@objectstack/core 0.9.0 → 0.9.2

package/src/health-monitor.test.ts

@@ -0,0 +1,81 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+import { PluginHealthMonitor } from './health-monitor.js';
+import { createLogger } from './logger.js';
+import type { PluginHealthCheck } from '@objectstack/spec/system';
+
+describe('PluginHealthMonitor', () => {
+  let monitor: PluginHealthMonitor;
+  let logger: ReturnType<typeof createLogger>;
+
+  beforeEach(() => {
+    logger = createLogger({ level: 'silent' });
+    monitor = new PluginHealthMonitor(logger);
+  });
+
+  it('should register plugin for health monitoring', () => {
+    const config: PluginHealthCheck = {
+      interval: 5000,
+      timeout: 1000,
+      failureThreshold: 3,
+      successThreshold: 1,
+      autoRestart: false,
+      maxRestartAttempts: 3,
+      restartBackoff: 'exponential',
+    };
+
+    monitor.registerPlugin('test-plugin', config);
+    expect(monitor.getHealthStatus('test-plugin')).toBe('unknown');
+  });
+
+  it('should report healthy status initially', () => {
+    const config: PluginHealthCheck = {
+      interval: 5000,
+      timeout: 1000,
+      failureThreshold: 3,
+      successThreshold: 1,
+      autoRestart: false,
+      maxRestartAttempts: 3,
+      restartBackoff: 'fixed',
+    };
+
+    monitor.registerPlugin('test-plugin', config);
+    expect(monitor.getHealthStatus('test-plugin')).toBe('unknown');
+  });
+
+  it('should get all health statuses', () => {
+    const config: PluginHealthCheck = {
+      interval: 5000,
+      timeout: 1000,
+      failureThreshold: 3,
+      successThreshold: 1,
+      autoRestart: false,
+      maxRestartAttempts: 3,
+      restartBackoff: 'linear',
+    };
+
+    monitor.registerPlugin('plugin1', config);
+    monitor.registerPlugin('plugin2', config);
+    
+    const statuses = monitor.getAllHealthStatuses();
+    expect(statuses.size).toBe(2);
+    expect(statuses.has('plugin1')).toBe(true);
+    expect(statuses.has('plugin2')).toBe(true);
+  });
+
+  it('should shutdown cleanly', () => {
+    const config: PluginHealthCheck = {
+      interval: 5000,
+      timeout: 1000,
+      failureThreshold: 3,
+      successThreshold: 1,
+      autoRestart: false,
+      maxRestartAttempts: 3,
+      restartBackoff: 'exponential',
+    };
+
+    monitor.registerPlugin('test-plugin', config);
+    monitor.shutdown();
+    
+    expect(monitor.getAllHealthStatuses().size).toBe(0);
+  });
+});

package/src/health-monitor.ts

@@ -0,0 +1,316 @@
+import type { 
+  PluginHealthStatus, 
+  PluginHealthCheck, 
+  PluginHealthReport 
+} from '@objectstack/spec/system';
+import type { ObjectLogger } from './logger.js';
+import type { Plugin } from './types.js';
+
+/**
+ * Plugin Health Monitor
+ * 
+ * Monitors plugin health status and performs automatic recovery actions.
+ * Implements the advanced lifecycle health monitoring protocol.
+ */
+export class PluginHealthMonitor {
+  private logger: ObjectLogger;
+  private healthChecks = new Map<string, PluginHealthCheck>();
+  private healthStatus = new Map<string, PluginHealthStatus>();
+  private healthReports = new Map<string, PluginHealthReport>();
+  private checkIntervals = new Map<string, NodeJS.Timeout>();
+  private failureCounters = new Map<string, number>();
+  private successCounters = new Map<string, number>();
+  private restartAttempts = new Map<string, number>();
+
+  constructor(logger: ObjectLogger) {
+    this.logger = logger.child({ component: 'HealthMonitor' });
+  }
+
+  /**
+   * Register a plugin for health monitoring
+   */
+  registerPlugin(pluginName: string, config: PluginHealthCheck): void {
+    this.healthChecks.set(pluginName, config);
+    this.healthStatus.set(pluginName, 'unknown');
+    this.failureCounters.set(pluginName, 0);
+    this.successCounters.set(pluginName, 0);
+    this.restartAttempts.set(pluginName, 0);
+
+    this.logger.info('Plugin registered for health monitoring', { 
+      plugin: pluginName,
+      interval: config.interval 
+    });
+  }
+
+  /**
+   * Start monitoring a plugin
+   */
+  startMonitoring(pluginName: string, plugin: Plugin): void {
+    const config = this.healthChecks.get(pluginName);
+    if (!config) {
+      this.logger.warn('Cannot start monitoring - plugin not registered', { plugin: pluginName });
+      return;
+    }
+
+    // Clear any existing interval
+    this.stopMonitoring(pluginName);
+
+    // Set up periodic health checks
+    const interval = setInterval(() => {
+      this.performHealthCheck(pluginName, plugin, config).catch(error => {
+        this.logger.error('Health check failed with error', { 
+          plugin: pluginName, 
+          error 
+        });
+      });
+    }, config.interval);
+
+    this.checkIntervals.set(pluginName, interval);
+    this.logger.info('Health monitoring started', { plugin: pluginName });
+
+    // Perform initial health check
+    this.performHealthCheck(pluginName, plugin, config).catch(error => {
+      this.logger.error('Initial health check failed', { 
+        plugin: pluginName, 
+        error 
+      });
+    });
+  }
+
+  /**
+   * Stop monitoring a plugin
+   */
+  stopMonitoring(pluginName: string): void {
+    const interval = this.checkIntervals.get(pluginName);
+    if (interval) {
+      clearInterval(interval);
+      this.checkIntervals.delete(pluginName);
+      this.logger.info('Health monitoring stopped', { plugin: pluginName });
+    }
+  }
+
+  /**
+   * Perform a health check on a plugin
+   */
+  private async performHealthCheck(
+    pluginName: string,
+    plugin: Plugin,
+    config: PluginHealthCheck
+  ): Promise<void> {
+    const startTime = Date.now();
+    let status: PluginHealthStatus = 'healthy';
+    let message: string | undefined;
+    const checks: Array<{ name: string; status: 'passed' | 'failed' | 'warning'; message?: string }> = [];
+
+    try {
+      // Check if plugin has a custom health check method
+      if (config.checkMethod && typeof (plugin as any)[config.checkMethod] === 'function') {
+        const checkResult = await Promise.race([
+          (plugin as any)[config.checkMethod](),
+          this.timeout(config.timeout, `Health check timeout after ${config.timeout}ms`)
+        ]);
+
+        if (checkResult === false || (checkResult && checkResult.status === 'unhealthy')) {
+          status = 'unhealthy';
+          message = checkResult?.message || 'Custom health check failed';
+          checks.push({ name: config.checkMethod, status: 'failed', message });
+        } else {
+          checks.push({ name: config.checkMethod, status: 'passed' });
+        }
+      } else {
+        // Default health check - just verify plugin is loaded
+        checks.push({ name: 'plugin-loaded', status: 'passed' });
+      }
+
+      // Update counters based on result
+      if (status === 'healthy') {
+        this.successCounters.set(pluginName, (this.successCounters.get(pluginName) || 0) + 1);
+        this.failureCounters.set(pluginName, 0);
+
+        // Recover from unhealthy state if we have enough successes
+        const currentStatus = this.healthStatus.get(pluginName);
+        if (currentStatus === 'unhealthy' || currentStatus === 'degraded') {
+          const successCount = this.successCounters.get(pluginName) || 0;
+          if (successCount >= config.successThreshold) {
+            this.healthStatus.set(pluginName, 'healthy');
+            this.logger.info('Plugin recovered to healthy state', { plugin: pluginName });
+          } else {
+            this.healthStatus.set(pluginName, 'recovering');
+          }
+        } else {
+          this.healthStatus.set(pluginName, 'healthy');
+        }
+      } else {
+        this.failureCounters.set(pluginName, (this.failureCounters.get(pluginName) || 0) + 1);
+        this.successCounters.set(pluginName, 0);
+
+        const failureCount = this.failureCounters.get(pluginName) || 0;
+        if (failureCount >= config.failureThreshold) {
+          this.healthStatus.set(pluginName, 'unhealthy');
+          this.logger.warn('Plugin marked as unhealthy', { 
+            plugin: pluginName, 
+            failures: failureCount 
+          });
+
+          // Attempt auto-restart if configured
+          if (config.autoRestart) {
+            await this.attemptRestart(pluginName, plugin, config);
+          }
+        } else {
+          this.healthStatus.set(pluginName, 'degraded');
+        }
+      }
+    } catch (error) {
+      status = 'failed';
+      message = error instanceof Error ? error.message : 'Unknown error';
+      this.failureCounters.set(pluginName, (this.failureCounters.get(pluginName) || 0) + 1);
+      this.healthStatus.set(pluginName, 'failed');
+      
+      checks.push({ 
+        name: 'health-check', 
+        status: 'failed', 
+        message: message 
+      });
+
+      this.logger.error('Health check exception', { 
+        plugin: pluginName, 
+        error 
+      });
+    }
+
+    // Create health report
+    const report: PluginHealthReport = {
+      status: this.healthStatus.get(pluginName) || 'unknown',
+      timestamp: new Date().toISOString(),
+      message,
+      metrics: {
+        uptime: Date.now() - startTime,
+      },
+      checks: checks.length > 0 ? checks : undefined,
+    };
+
+    this.healthReports.set(pluginName, report);
+  }
+
+  /**
+   * Attempt to restart a plugin
+   */
+  private async attemptRestart(
+    pluginName: string,
+    plugin: Plugin,
+    config: PluginHealthCheck
+  ): Promise<void> {
+    const attempts = this.restartAttempts.get(pluginName) || 0;
+    
+    if (attempts >= config.maxRestartAttempts) {
+      this.logger.error('Max restart attempts reached, giving up', { 
+        plugin: pluginName, 
+        attempts 
+      });
+      this.healthStatus.set(pluginName, 'failed');
+      return;
+    }
+
+    this.restartAttempts.set(pluginName, attempts + 1);
+    
+    // Calculate backoff delay
+    const delay = this.calculateBackoff(attempts, config.restartBackoff);
+    
+    this.logger.info('Scheduling plugin restart', { 
+      plugin: pluginName, 
+      attempt: attempts + 1, 
+      delay 
+    });
+
+    await new Promise(resolve => setTimeout(resolve, delay));
+
+    try {
+      // Call destroy and init to restart
+      if (plugin.destroy) {
+        await plugin.destroy();
+      }
+      
+      // Note: Full restart would require kernel context
+      // This is a simplified version - actual implementation would need kernel integration
+      this.logger.info('Plugin restarted', { plugin: pluginName });
+      
+      // Reset counters on successful restart
+      this.failureCounters.set(pluginName, 0);
+      this.successCounters.set(pluginName, 0);
+      this.healthStatus.set(pluginName, 'recovering');
+    } catch (error) {
+      this.logger.error('Plugin restart failed', { 
+        plugin: pluginName, 
+        error 
+      });
+      this.healthStatus.set(pluginName, 'failed');
+    }
+  }
+
+  /**
+   * Calculate backoff delay for restarts
+   */
+  private calculateBackoff(attempt: number, strategy: 'fixed' | 'linear' | 'exponential'): number {
+    const baseDelay = 1000; // 1 second base
+
+    switch (strategy) {
+      case 'fixed':
+        return baseDelay;
+      case 'linear':
+        return baseDelay * (attempt + 1);
+      case 'exponential':
+        return baseDelay * Math.pow(2, attempt);
+      default:
+        return baseDelay;
+    }
+  }
+
+  /**
+   * Get current health status of a plugin
+   */
+  getHealthStatus(pluginName: string): PluginHealthStatus | undefined {
+    return this.healthStatus.get(pluginName);
+  }
+
+  /**
+   * Get latest health report for a plugin
+   */
+  getHealthReport(pluginName: string): PluginHealthReport | undefined {
+    return this.healthReports.get(pluginName);
+  }
+
+  /**
+   * Get all health statuses
+   */
+  getAllHealthStatuses(): Map<string, PluginHealthStatus> {
+    return new Map(this.healthStatus);
+  }
+
+  /**
+   * Shutdown health monitor
+   */
+  shutdown(): void {
+    // Stop all monitoring intervals
+    for (const pluginName of this.checkIntervals.keys()) {
+      this.stopMonitoring(pluginName);
+    }
+    
+    this.healthChecks.clear();
+    this.healthStatus.clear();
+    this.healthReports.clear();
+    this.failureCounters.clear();
+    this.successCounters.clear();
+    this.restartAttempts.clear();
+    
+    this.logger.info('Health monitor shutdown complete');
+  }
+
+  /**
+   * Timeout helper
+   */
+  private timeout<T>(ms: number, message: string): Promise<T> {
+    return new Promise((_, reject) => {
+      setTimeout(() => reject(new Error(message)), ms);
+    });
+  }
+}