immortal-js 1.0.0

package/packages/core/src/plugins/BuiltinPlugins.ts

@@ -0,0 +1,269 @@
+/**
+ * @file BuiltinPlugins.ts
+ * @description Built-in Immortal.js plugins
+ *
+ * These plugins demonstrate the Plugin API and provide ready-to-use integrations:
+ * 1. ConsoleLogPlugin — structured console logging of all events
+ * 2. AnomalyDetectionPlugin — detects unusual patterns and alerts
+ * 3. RequestTracingPlugin — adds trace IDs to all requests
+ * 4. SlackAlertPlugin — posts critical alerts to a Slack webhook
+ */
+
+import type { ImmortalPlugin, PluginContext, SystemMetrics, ImmortalEvent } from "../types.js";
+
+// ─────────────────────────────────────────────────────────────────────────────
+// 1. CONSOLE LOG PLUGIN
+// ─────────────────────────────────────────────────────────────────────────────
+
+export const ConsoleLogPlugin: ImmortalPlugin = {
+  name: "console-log",
+  version: "1.0.0",
+  description: "Logs all Immortal.js events to console with structured formatting",
+
+  onInit: (ctx: PluginContext) => {
+    ctx.logger.info("[ConsoleLogPlugin] Initialized — all events will be logged");
+  },
+
+  onError: (error: unknown, context) => {
+    console.error("[IMMORTAL ERROR]", {
+      requestId: context.requestId,
+      route: context.route,
+      error: error instanceof Error ? { name: error.name, message: error.message } : String(error),
+    });
+  },
+
+  onMetrics: (metrics: SystemMetrics) => {
+    if (metrics.eventLoopLag.p99 > 200) {
+      console.warn("[IMMORTAL METRICS] High event loop lag:", {
+        p99: `${metrics.eventLoopLag.p99.toFixed(1)}ms`,
+      });
+    }
+    if (metrics.memory.heapUsedMb > 500) {
+      console.warn("[IMMORTAL METRICS] High heap usage:", {
+        heap: `${metrics.memory.heapUsedMb.toFixed(1)}MB`,
+      });
+    }
+  },
+};
+
+// ─────────────────────────────────────────────────────────────────────────────
+// 2. ANOMALY DETECTION PLUGIN
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface AnomalyDetectionOptions {
+  /** Error rate threshold to trigger anomaly (default: 0.1 = 10%) */
+  errorRateThreshold?: number;
+  /** Request rate drop % that triggers anomaly (default: 50%) */
+  requestRateDropPercent?: number;
+  /** Event loop lag spike threshold in ms (default: 300) */
+  eventLoopLagSpikeMs?: number;
+  /** Custom alert handler */
+  onAnomaly?: (type: string, details: Record<string, unknown>) => void;
+}
+
+export function createAnomalyDetectionPlugin(
+  options: AnomalyDetectionOptions = {}
+): ImmortalPlugin {
+  const errorRateThreshold = options.errorRateThreshold ?? 0.1;
+  const lagSpike = options.eventLoopLagSpikeMs ?? 300;
+  let previousRequestRate = 0;
+  let initialized = false;
+
+  return {
+    name: "anomaly-detection",
+    version: "1.0.0",
+    description: "Detects unusual patterns in system metrics and fires alerts",
+
+    onInit: (ctx: PluginContext) => {
+      ctx.logger.info("[AnomalyDetectionPlugin] Watching for anomalies", {
+        errorRateThreshold: `${errorRateThreshold * 100}%`,
+        lagSpikeThreshold: `${lagSpike}ms`,
+      });
+    },
+
+    onMetrics: (metrics: SystemMetrics) => {
+      const anomalies: Array<{ type: string; details: Record<string, unknown> }> = [];
+
+      // Anomaly 1: Error rate spike
+      if (metrics.requests.errorRate > errorRateThreshold) {
+        anomalies.push({
+          type: "error-rate-spike",
+          details: {
+            errorRate: `${(metrics.requests.errorRate * 100).toFixed(2)}%`,
+            threshold: `${errorRateThreshold * 100}%`,
+            totalRequests: metrics.requests.total,
+          },
+        });
+      }
+
+      // Anomaly 2: Event loop lag spike
+      if (metrics.eventLoopLag.p99 > lagSpike) {
+        anomalies.push({
+          type: "event-loop-lag-spike",
+          details: {
+            p99: `${metrics.eventLoopLag.p99.toFixed(1)}ms`,
+            threshold: `${lagSpike}ms`,
+          },
+        });
+      }
+
+      // Anomaly 3: Request rate drop (only after initialized)
+      if (initialized && previousRequestRate > 0) {
+        const dropPercent =
+          ((previousRequestRate - metrics.requests.perSecond) / previousRequestRate) * 100;
+        if (dropPercent > (options.requestRateDropPercent ?? 50)) {
+          anomalies.push({
+            type: "request-rate-drop",
+            details: {
+              previous: `${previousRequestRate.toFixed(1)} req/s`,
+              current: `${metrics.requests.perSecond.toFixed(1)} req/s`,
+              dropPercent: `${dropPercent.toFixed(1)}%`,
+            },
+          });
+        }
+      }
+
+      previousRequestRate = metrics.requests.perSecond;
+      initialized = true;
+
+      for (const anomaly of anomalies) {
+        console.error(`[IMMORTAL ANOMALY] ${anomaly.type}`, anomaly.details);
+        options.onAnomaly?.(anomaly.type, anomaly.details);
+      }
+    },
+  };
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// 3. REQUEST TRACING PLUGIN
+// ─────────────────────────────────────────────────────────────────────────────
+
+export const RequestTracingPlugin: ImmortalPlugin = {
+  name: "request-tracing",
+  version: "1.0.0",
+  description: "Adds distributed trace IDs to requests for APM correlation",
+
+  onInit: (ctx: PluginContext) => {
+    ctx.logger.info("[RequestTracingPlugin] Request tracing enabled");
+  },
+
+  onRequest: (context) => {
+    // Trace ID is already set by AsyncBoundary (requestId)
+    // This plugin can extend it with parent trace correlation
+    const traceId = context.requestId;
+    const spanId = Math.random().toString(36).substring(2, 18);
+
+    context.custom = {
+      ...context.custom,
+      traceId,
+      spanId,
+      traceFlags: "01", // sampled
+    };
+  },
+
+  onError: (error: unknown, context) => {
+    // Log with trace context for APM correlation
+    const traceContext = context.custom ?? {};
+    console.error("[TRACE]", {
+      traceId: traceContext["traceId"],
+      spanId: traceContext["spanId"],
+      error: (error as Error).message,
+      route: context.route,
+    });
+  },
+};
+
+// ─────────────────────────────────────────────────────────────────────────────
+// 4. SLACK ALERT PLUGIN
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface SlackAlertOptions {
+  webhookUrl: string;
+  /** Only send alerts for these event types */
+  eventTypes?: string[];
+  /** Minimum severity to alert on */
+  minSeverity?: "warn" | "error" | "critical";
+  /** Application name in alert messages */
+  appName?: string;
+}
+
+export function createSlackAlertPlugin(options: SlackAlertOptions): ImmortalPlugin {
+  const criticalEvents = [
+    "worker:escalated",
+    "memory:proactive-restart",
+    "memory:critical",
+    "eventloop:lag-critical",
+    "circuit:opened",
+    "error:escalated",
+  ];
+
+  const targetEvents = options.eventTypes ?? criticalEvents;
+  let lastAlertTime = 0;
+  const alertThrottleMs = 60_000; // Don't spam — max 1 alert/minute per event type
+
+  return {
+    name: "slack-alerts",
+    version: "1.0.0",
+    description: "Posts critical Immortal.js events to a Slack webhook",
+
+    onInit: (ctx: PluginContext) => {
+      ctx.logger.info("[SlackAlertPlugin] Slack alerts configured", {
+        webhookUrl: options.webhookUrl.substring(0, 30) + "...",
+        events: targetEvents,
+      });
+    },
+
+    onMetrics: async (metrics: SystemMetrics) => {
+      const now = Date.now();
+      if (now - lastAlertTime < alertThrottleMs) return;
+
+      // Critical metrics that warrant a Slack alert
+      const alerts: string[] = [];
+
+      if (metrics.eventLoopLag.p99 > 1000) {
+        alerts.push(`🔴 Event loop critically lagging: *${metrics.eventLoopLag.p99.toFixed(0)}ms p99*`);
+      }
+      if (metrics.memory.heapUsedMb > 1400) {
+        alerts.push(`🔴 Memory critical: *${metrics.memory.heapUsedMb.toFixed(0)}MB* heap`);
+      }
+      if (metrics.requests.errorRate > 0.3) {
+        alerts.push(`🔴 Error rate spike: *${(metrics.requests.errorRate * 100).toFixed(1)}%*`);
+      }
+
+      if (alerts.length === 0) return;
+
+      lastAlertTime = now;
+
+      try {
+        await fetch(options.webhookUrl, {
+          method: "POST",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify({
+            text: `*${options.appName ?? "Immortal.js"} Critical Alert*\n${alerts.join("\n")}`,
+            attachments: [{
+              color: "danger",
+              fields: [
+                { title: "Environment", value: process.env["NODE_ENV"] ?? "unknown", short: true },
+                { title: "PID", value: String(process.pid), short: true },
+                { title: "Time", value: new Date().toISOString(), short: false },
+              ],
+            }],
+          }),
+        });
+      } catch {
+        // Don't let Slack failures affect the application
+      }
+    },
+  };
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// PLUGIN REGISTRY
+// ─────────────────────────────────────────────────────────────────────────────
+
+export const BUILTIN_PLUGINS = {
+  ConsoleLog: ConsoleLogPlugin,
+  RequestTracing: RequestTracingPlugin,
+  createAnomalyDetection: createAnomalyDetectionPlugin,
+  createSlackAlert: createSlackAlertPlugin,
+} as const;

package/packages/core/src/recovery/CircuitBreaker.ts

@@ -0,0 +1,334 @@
+/**
+ * @file CircuitBreaker.ts
+ * @description Production-grade Circuit Breaker with Sliding Window metrics
+ *
+ * State Machine:
+ *   CLOSED → OPEN: When failure rate in sliding window exceeds threshold
+ *   OPEN → HALF_OPEN: After cooldown period (one probe request allowed)
+ *   HALF_OPEN → CLOSED: If probe succeeds (reset failure count)
+ *   HALF_OPEN → OPEN: If probe fails (restart cooldown)
+ *
+ * Key improvement over basic implementations:
+ * - Sliding window (not cumulative counters) for accurate recent failure rate
+ * - Minimum request threshold before circuit can open (prevents premature opening)
+ * - Named circuits (one per external service)
+ * - Full telemetry via event bus
+ */
+
+import type { CircuitBreakerConfig, CircuitBreakerStatus, FallbackGenerator } from "../types.js";
+import { CircuitOpenError } from "../types.js";
+import { DEFAULT_CIRCUIT_BREAKER } from "../config/defaults.js";
+import { getEventBus } from "../event-bus.js";
+import { getLogger } from "../logger.js";
+
+// ─────────────────────────────────────────────────────────────────────────────
+// SLIDING WINDOW
+// ─────────────────────────────────────────────────────────────────────────────
+
+type RequestOutcome = "success" | "failure";
+
+class SlidingWindow {
+  private readonly size: number;
+  private window: RequestOutcome[] = [];
+  private failures = 0;
+  private successes = 0;
+
+  constructor(size: number) {
+    this.size = size;
+  }
+
+  record(outcome: RequestOutcome): void {
+    if (this.window.length >= this.size) {
+      const evicted = this.window.shift()!;
+      if (evicted === "failure") this.failures--;
+      else this.successes--;
+    }
+
+    this.window.push(outcome);
+    if (outcome === "failure") this.failures++;
+    else this.successes++;
+  }
+
+  getFailureRate(): number {
+    if (this.window.length === 0) return 0;
+    return (this.failures / this.window.length) * 100;
+  }
+
+  getTotalRequests(): number {
+    return this.window.length;
+  }
+
+  getFailures(): number {
+    return this.failures;
+  }
+
+  getSuccesses(): number {
+    return this.successes;
+  }
+
+  reset(): void {
+    this.window = [];
+    this.failures = 0;
+    this.successes = 0;
+  }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// CIRCUIT BREAKER CLASS
+// ─────────────────────────────────────────────────────────────────────────────
+
+export type CircuitState = "CLOSED" | "OPEN" | "HALF_OPEN";
+
+export class CircuitBreaker {
+  private state: CircuitState = "CLOSED";
+  private readonly window: SlidingWindow;
+  private lastStateChange: number = Date.now();
+  private lastFailureTime: number = 0;
+  private halfOpenProbeActive = false;
+  private readonly config: Required<CircuitBreakerConfig>;
+
+  constructor(
+    public readonly name: string,
+    config: CircuitBreakerConfig = {}
+  ) {
+    this.config = { ...DEFAULT_CIRCUIT_BREAKER, ...config };
+    this.window = new SlidingWindow(this.config.slidingWindowSize);
+  }
+
+  /**
+   * Execute a function through the circuit breaker.
+   * If circuit is OPEN, throws CircuitOpenError immediately (fail-fast).
+   * If circuit is HALF_OPEN, allows one probe request.
+   *
+   * @param fn - The operation to attempt
+   * @param fallback - Optional fallback value generator (if provided, never throws CircuitOpenError)
+   */
+  async execute<T>(
+    fn: () => Promise<T>,
+    fallback?: () => T | Promise<T>
+  ): Promise<T> {
+    const bus = getEventBus();
+
+    // Check if we should transition from OPEN to HALF_OPEN
+    if (this.state === "OPEN") {
+      const timeSinceLastFailure = Date.now() - this.lastFailureTime;
+
+      if (timeSinceLastFailure >= this.config.cooldownMs) {
+        this.transitionTo("HALF_OPEN");
+      } else {
+        // Still open — fail fast
+        bus.emit("circuit:rejected", {
+          service: this.name,
+          data: {
+            state: this.state,
+            cooldownRemainingMs: this.config.cooldownMs - timeSinceLastFailure,
+          },
+        });
+
+        if (fallback) {
+          return await Promise.resolve(fallback());
+        }
+        throw new CircuitOpenError(this.name);
+      }
+    }
+
+    // HALF_OPEN: only allow one probe at a time
+    if (this.state === "HALF_OPEN" && this.halfOpenProbeActive) {
+      if (fallback) {
+        return await Promise.resolve(fallback());
+      }
+      throw new CircuitOpenError(this.name);
+    }
+
+    if (this.state === "HALF_OPEN") {
+      this.halfOpenProbeActive = true;
+    }
+
+    // CLOSED or HALF_OPEN probe: attempt the operation
+    try {
+      const result = await fn();
+      this.onSuccess();
+      return result;
+    } catch (err) {
+      this.onFailure();
+
+      if (fallback && this.state === "OPEN") {
+        getLogger().debug(`Circuit '${this.name}' opened — using fallback`);
+        return await Promise.resolve(fallback());
+      }
+
+      throw err;
+    } finally {
+      this.halfOpenProbeActive = false;
+    }
+  }
+
+  /**
+   * Get current circuit status for monitoring
+   */
+  getStatus(): CircuitBreakerStatus {
+    return {
+      name: this.name,
+      state: this.state,
+      failures: this.window.getFailures(),
+      successes: this.window.getSuccesses(),
+      totalRequests: this.window.getTotalRequests(),
+      failureRate: this.window.getFailureRate(),
+      lastFailureTime: this.lastFailureTime,
+      lastStateChange: this.lastStateChange,
+    };
+  }
+
+  /**
+   * Force the circuit to a specific state (for testing/admin operations)
+   */
+  forceState(state: CircuitState): void {
+    getLogger().warn(`Circuit '${this.name}' state forced to ${state}`);
+    this.transitionTo(state);
+  }
+
+  /**
+   * Reset circuit to initial CLOSED state
+   */
+  reset(): void {
+    this.window.reset();
+    this.transitionTo("CLOSED");
+    this.lastFailureTime = 0;
+  }
+
+  getCurrentState(): CircuitState {
+    return this.state;
+  }
+
+  // ─────────────────────────────────────────────────────────────────────────
+  // PRIVATE STATE MACHINE
+  // ─────────────────────────────────────────────────────────────────────────
+
+  private onSuccess(): void {
+    this.window.record("success");
+
+    if (this.state === "HALF_OPEN") {
+      // Probe succeeded — close the circuit
+      this.window.reset();
+      this.transitionTo("CLOSED");
+      getLogger().info(`Circuit '${this.name}' recovered — transitioning HALF_OPEN → CLOSED`);
+    }
+  }
+
+  private onFailure(): void {
+    this.window.record("failure");
+    this.lastFailureTime = Date.now();
+
+    if (this.state === "HALF_OPEN") {
+      // Probe failed — reopen circuit
+      this.transitionTo("OPEN");
+      getLogger().warn(`Circuit '${this.name}' probe failed — returning to OPEN`);
+      return;
+    }
+
+    if (this.state === "CLOSED") {
+      const totalReqs = this.window.getTotalRequests();
+      const failureRate = this.window.getFailureRate();
+
+      const shouldOpen =
+        totalReqs >= this.config.minimumRequests &&
+        (failureRate >= this.config.failureRateThreshold ||
+          this.window.getFailures() >= this.config.threshold);
+
+      if (shouldOpen) {
+        this.transitionTo("OPEN");
+      }
+    }
+  }
+
+  private transitionTo(newState: CircuitState): void {
+    if (this.state === newState) return;
+
+    const bus = getEventBus();
+    const previousState = this.state;
+    this.state = newState;
+    this.lastStateChange = Date.now();
+
+    const status = this.getStatus();
+
+    switch (newState) {
+      case "OPEN":
+        bus.emit("circuit:opened", { service: this.name, data: { previousState, ...status } });
+        getLogger().warn(`Circuit '${this.name}' OPENED`, {
+          failureRate: `${status.failureRate.toFixed(1)}%`,
+          failures: status.failures,
+          totalRequests: status.totalRequests,
+        });
+        break;
+
+      case "HALF_OPEN":
+        bus.emit("circuit:half-open", { service: this.name, data: { previousState, ...status } });
+        getLogger().info(`Circuit '${this.name}' → HALF_OPEN (probing...)`);
+        break;
+
+      case "CLOSED":
+        bus.emit("circuit:closed", { service: this.name, data: { previousState, ...status } });
+        getLogger().info(`Circuit '${this.name}' CLOSED (recovered)`);
+        break;
+    }
+  }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// CIRCUIT BREAKER REGISTRY
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Registry of named circuit breakers — one per external service.
+ * Enables getting/monitoring all circuits in one place.
+ */
+export class CircuitBreakerRegistry {
+  private circuits = new Map<string, CircuitBreaker>();
+  private readonly defaultConfig: CircuitBreakerConfig;
+
+  constructor(defaultConfig: CircuitBreakerConfig = {}) {
+    this.defaultConfig = defaultConfig;
+  }
+
+  /**
+   * Get or create a circuit breaker for a named service
+   */
+  get(name: string, config?: CircuitBreakerConfig): CircuitBreaker {
+    if (!this.circuits.has(name)) {
+      this.circuits.set(
+        name,
+        new CircuitBreaker(name, { ...this.defaultConfig, ...config })
+      );
+    }
+    return this.circuits.get(name)!;
+  }
+
+  /**
+   * Get statuses of all registered circuits
+   */
+  getAllStatuses(): Record<string, CircuitBreakerStatus> {
+    const result: Record<string, CircuitBreakerStatus> = {};
+    for (const [name, circuit] of this.circuits) {
+      result[name] = circuit.getStatus();
+    }
+    return result;
+  }
+
+  /**
+   * Reset all circuits (for testing)
+   */
+  resetAll(): void {
+    for (const circuit of this.circuits.values()) {
+      circuit.reset();
+    }
+  }
+
+  /**
+   * Get list of currently open circuits
+   */
+  getOpenCircuits(): string[] {
+    return Array.from(this.circuits.entries())
+      .filter(([, c]) => c.getCurrentState() === "OPEN")
+      .map(([name]) => name);
+  }
+}