@serve.zone/dcrouter 11.8.11 → 11.9.1

package/ts/classes.dcrouter.ts

@@ -252,6 +252,11 @@ export class DcRouter {
   // Certificate provisioning scheduler with per-domain backoff
   public certProvisionScheduler?: CertProvisionScheduler;
 
+  // Service lifecycle management
+  public serviceManager: plugins.taskbuffer.ServiceManager;
+  private serviceSubjectSubscription?: plugins.smartrx.rxjs.Subscription;
+  public smartAcmeReady = false;
+
   // TypedRouter for API endpoints
   public typedrouter = new plugins.typedrequest.TypedRouter();
 
@@ -279,67 +284,253 @@ export class DcRouter {
 
     // Initialize storage manager
     this.storageManager = new StorageManager(this.options.storage);
+
+    // Initialize service manager and register all services
+    this.serviceManager = new plugins.taskbuffer.ServiceManager({
+      name: 'dcrouter',
+      startupTimeoutMs: 120_000,
+      shutdownTimeoutMs: 30_000,
+    });
+    this.registerServices();
   }
 
-  public async start() {
-    logger.log('info', 'Starting DcRouter Services');
-    
+  /**
+   * Register all dcrouter services with the ServiceManager.
+   * Services are started in dependency order, with failure isolation for optional services.
+   */
+  private registerServices(): void {
+    // OpsServer: critical, no dependencies — provides visibility
+    this.serviceManager.addService(
+      new plugins.taskbuffer.Service('OpsServer')
+        .critical()
+        .withStart(async () => {
+          this.opsServer = new OpsServer(this);
+          await this.opsServer.start();
+        })
+        .withStop(async () => {
+          await this.opsServer?.stop();
+        })
+        .withRetry({ maxRetries: 0 }),
+    );
+
+    // CacheDb: optional, no dependencies
+    if (this.options.cacheConfig?.enabled !== false) {
+      this.serviceManager.addService(
+        new plugins.taskbuffer.Service('CacheDb')
+          .optional()
+          .withStart(async () => {
+            await this.setupCacheDb();
+          })
+          .withStop(async () => {
+            if (this.cacheCleaner) {
+              this.cacheCleaner.stop();
+              this.cacheCleaner = undefined;
+            }
+            if (this.cacheDb) {
+              await this.cacheDb.stop();
+              CacheDb.resetInstance();
+              this.cacheDb = undefined;
+            }
+          })
+          .withRetry({ maxRetries: 2, baseDelayMs: 1000, maxDelayMs: 5000 }),
+      );
+    }
 
-    this.opsServer = new OpsServer(this);
-    await this.opsServer.start();
+    // MetricsManager: optional, depends on OpsServer
+    this.serviceManager.addService(
+      new plugins.taskbuffer.Service('MetricsManager')
+        .optional()
+        .dependsOn('OpsServer')
+        .withStart(async () => {
+          this.metricsManager = new MetricsManager(this);
+          await this.metricsManager.start();
+        })
+        .withStop(async () => {
+          if (this.metricsManager) {
+            await this.metricsManager.stop();
+            this.metricsManager = undefined;
+          }
+        })
+        .withRetry({ maxRetries: 1, baseDelayMs: 1000 }),
+    );
 
-    try {
-      // Initialize cache database if enabled (default: enabled)
-      if (this.options.cacheConfig?.enabled !== false) {
-        await this.setupCacheDb();
-      }
+    // SmartProxy: critical, depends on CacheDb (if enabled)
+    const smartProxyDeps: string[] = [];
+    if (this.options.cacheConfig?.enabled !== false) {
+      smartProxyDeps.push('CacheDb');
+    }
+    this.serviceManager.addService(
+      new plugins.taskbuffer.Service('SmartProxy')
+        .critical()
+        .dependsOn(...smartProxyDeps)
+        .withStart(async () => {
+          await this.setupSmartProxy();
+        })
+        .withStop(async () => {
+          if (this.smartProxy) {
+            this.smartProxy.removeAllListeners();
+            await this.smartProxy.stop();
+            this.smartProxy = undefined;
+          }
+        })
+        .withRetry({ maxRetries: 0 }),
+    );
 
-      // Initialize MetricsManager
-      this.metricsManager = new MetricsManager(this);
-      await this.metricsManager.start();
-      
-      // Set up SmartProxy for HTTP/HTTPS and all traffic including email routes
-      await this.setupSmartProxy();
-
-      // Initialize programmatic config API managers
-      this.routeConfigManager = new RouteConfigManager(
-        this.storageManager,
-        () => this.getConstructorRoutes(),
-        () => this.smartProxy,
-        () => this.options.http3,
+    // SmartAcme: optional, depends on SmartProxy — aggressive retry for rate limits
+    // Only registered if DNS challenge is configured
+    if (this.options.dnsChallenge?.cloudflareApiKey) {
+      this.serviceManager.addService(
+        new plugins.taskbuffer.Service('SmartAcme')
+          .optional()
+          .dependsOn('SmartProxy')
+          .withStart(async () => {
+            if (this.smartAcme) {
+              await this.smartAcme.start();
+              this.smartAcmeReady = true;
+              logger.log('info', 'SmartAcme DNS-01 provider is now ready');
+            }
+          })
+          .withStop(async () => {
+            this.smartAcmeReady = false;
+            if (this.smartAcme) {
+              await this.smartAcme.stop();
+              this.smartAcme = undefined;
+            }
+          })
+          .withRetry({ maxRetries: 20, baseDelayMs: 5000, maxDelayMs: 3_600_000, backoffFactor: 2 }),
       );
-      this.apiTokenManager = new ApiTokenManager(this.storageManager);
-      await this.apiTokenManager.initialize();
-      await this.routeConfigManager.initialize();
+    }
 
-      // Set up unified email handling if configured
-      if (this.options.emailConfig) {
-        await this.setupUnifiedEmailHandling();
-      }
-      
-      // Set up DNS server if configured with nameservers and scopes
-      if (this.options.dnsNsDomains && this.options.dnsNsDomains.length > 0 &&
-          this.options.dnsScopes && this.options.dnsScopes.length > 0) {
-        await this.setupDnsWithSocketHandler();
-      }
+    // ConfigManagers: optional, depends on SmartProxy
+    this.serviceManager.addService(
+      new plugins.taskbuffer.Service('ConfigManagers')
+        .optional()
+        .dependsOn('SmartProxy')
+        .withStart(async () => {
+          this.routeConfigManager = new RouteConfigManager(
+            this.storageManager,
+            () => this.getConstructorRoutes(),
+            () => this.smartProxy,
+            () => this.options.http3,
+          );
+          this.apiTokenManager = new ApiTokenManager(this.storageManager);
+          await this.apiTokenManager.initialize();
+          await this.routeConfigManager.initialize();
+        })
+        .withStop(async () => {
+          this.routeConfigManager = undefined;
+          this.apiTokenManager = undefined;
+        })
+        .withRetry({ maxRetries: 2, baseDelayMs: 1000 }),
+    );
 
-      // Set up RADIUS server if configured
-      if (this.options.radiusConfig) {
-        await this.setupRadiusServer();
-      }
+    // Email Server: optional, depends on SmartProxy
+    if (this.options.emailConfig) {
+      this.serviceManager.addService(
+        new plugins.taskbuffer.Service('EmailServer')
+          .optional()
+          .dependsOn('SmartProxy')
+          .withStart(async () => {
+            await this.setupUnifiedEmailHandling();
+          })
+          .withStop(async () => {
+            if (this.emailServer) {
+              if ((this.emailServer as any).deliverySystem) {
+                (this.emailServer as any).deliverySystem.removeAllListeners();
+              }
+              this.emailServer.removeAllListeners();
+              await this.emailServer.stop();
+              this.emailServer = undefined;
+            }
+          })
+          .withRetry({ maxRetries: 3, baseDelayMs: 2000, maxDelayMs: 30_000 }),
+      );
+    }
 
-      // Set up Remote Ingress hub if configured
-      if (this.options.remoteIngressConfig?.enabled) {
-        await this.setupRemoteIngress();
-      }
+    // DNS Server: optional, depends on SmartProxy
+    if (this.options.dnsNsDomains?.length > 0 && this.options.dnsScopes?.length > 0) {
+      this.serviceManager.addService(
+        new plugins.taskbuffer.Service('DnsServer')
+          .optional()
+          .dependsOn('SmartProxy')
+          .withStart(async () => {
+            await this.setupDnsWithSocketHandler();
+          })
+          .withStop(async () => {
+            // Flush pending DNS batch log
+            if (this.dnsBatchTimer) {
+              clearTimeout(this.dnsBatchTimer);
+              if (this.dnsBatchCount > 0) {
+                logger.log('info', `DNS: ${this.dnsBatchCount} queries processed (final flush)`, { zone: 'dns' });
+              }
+              this.dnsBatchTimer = null;
+              this.dnsBatchCount = 0;
+              this.dnsLogWindowSecond = 0;
+              this.dnsLogWindowCount = 0;
+            }
+            if (this.dnsServer) {
+              this.dnsServer.removeAllListeners();
+              await this.dnsServer.stop();
+              this.dnsServer = undefined;
+            }
+          })
+          .withRetry({ maxRetries: 3, baseDelayMs: 2000, maxDelayMs: 30_000 }),
+      );
+    }
 
-      this.logStartupSummary();
-    } catch (error) {
-      logger.log('error', 'Error starting DcRouter', { error: String(error) });
-      // Try to clean up any services that may have started
-      await this.stop();
-      throw error;
+    // RADIUS Server: optional, no dependency on SmartProxy
+    if (this.options.radiusConfig) {
+      this.serviceManager.addService(
+        new plugins.taskbuffer.Service('RadiusServer')
+          .optional()
+          .withStart(async () => {
+            await this.setupRadiusServer();
+          })
+          .withStop(async () => {
+            if (this.radiusServer) {
+              await this.radiusServer.stop();
+              this.radiusServer = undefined;
+            }
+          })
+          .withRetry({ maxRetries: 3, baseDelayMs: 2000, maxDelayMs: 30_000 }),
+      );
     }
+
+    // Remote Ingress: optional, depends on SmartProxy
+    if (this.options.remoteIngressConfig?.enabled) {
+      this.serviceManager.addService(
+        new plugins.taskbuffer.Service('RemoteIngress')
+          .optional()
+          .dependsOn('SmartProxy')
+          .withStart(async () => {
+            await this.setupRemoteIngress();
+          })
+          .withStop(async () => {
+            if (this.tunnelManager) {
+              await this.tunnelManager.stop();
+              this.tunnelManager = undefined;
+            }
+            this.remoteIngressManager = undefined;
+          })
+          .withRetry({ maxRetries: 3, baseDelayMs: 2000, maxDelayMs: 30_000 }),
+      );
+    }
+
+    // Wire up aggregated events for logging
+    this.serviceSubjectSubscription = this.serviceManager.serviceSubject.subscribe((event) => {
+      const level = event.type === 'failed' ? 'error' : event.type === 'retrying' ? 'warn' : 'info';
+      logger.log(level as any, `Service '${event.serviceName}': ${event.type}`, {
+        state: event.state,
+        ...(event.error ? { error: event.error } : {}),
+        ...(event.attempt ? { attempt: event.attempt } : {}),
+      });
+    });
+  }
+
+  public async start() {
+    logger.log('info', 'Starting DcRouter Services');
+    await this.serviceManager.start();
+    this.logStartupSummary();
   }
   
   /**
@@ -399,7 +590,21 @@ export class DcRouter {
       logger.log('info', `Cache Database: storage=${this.cacheDb.getStoragePath()}, db=${this.cacheDb.getDbName()}, cleaner=${this.cacheCleaner?.isActive() ? 'active' : 'inactive'} (${(this.options.cacheConfig?.cleanupIntervalHours || 1)}h interval)`);
     }
 
-    logger.log('info', 'All services are running');
+    // Service status summary from ServiceManager
+    const health = this.serviceManager.getHealth();
+    const statuses = health.services;
+    const running = statuses.filter(s => s.state === 'running').length;
+    const failed = statuses.filter(s => s.state === 'failed').length;
+    const retrying = statuses.filter(s => s.state === 'starting' || s.state === 'degraded').length;
+
+    if (failed > 0) {
+      const failedNames = statuses.filter(s => s.state === 'failed').map(s => `${s.name}: ${s.lastError || 'unknown'}`);
+      logger.log('warn', `DcRouter started in degraded mode — ${running} running, ${failed} failed: ${failedNames.join('; ')}`);
+    } else if (retrying > 0) {
+      logger.log('info', `DcRouter started — ${running} running, ${retrying} still initializing`);
+    } else {
+      logger.log('info', `All ${running} services are running`);
+    }
   }
   
   /**
@@ -435,6 +640,13 @@ export class DcRouter {
    */
   private async setupSmartProxy(): Promise<void> {
     logger.log('info', 'Setting up SmartProxy...');
+
+    // Clean up any existing SmartProxy instance (e.g. from a retry)
+    if (this.smartProxy) {
+      this.smartProxy.removeAllListeners();
+      this.smartProxy = undefined;
+    }
+
     let routes: plugins.smartproxy.IRouteConfig[] = [];
     let acmeConfig: plugins.smartproxy.IAcmeOptions | undefined;
     
@@ -535,10 +747,13 @@ export class DcRouter {
       // Initialize cert provision scheduler
       this.certProvisionScheduler = new CertProvisionScheduler(this.storageManager);
 
-      // If we have DNS challenge handlers, create SmartAcme and wire to certProvisionFunction
+      // If we have DNS challenge handlers, create SmartAcme instance and wire certProvisionFunction
+      // Note: SmartAcme.start() is NOT called here — it runs as a separate optional service
+      // via the ServiceManager, with aggressive retry for rate-limit resilience.
       if (challengeHandlers.length > 0) {
         // Stop old SmartAcme if it exists (e.g., during updateSmartProxyConfig)
         if (this.smartAcme) {
+          this.smartAcmeReady = false;
           await this.smartAcme.stop().catch(err =>
             logger.log('error', 'Error stopping old SmartAcme', { error: String(err) })
           );
@@ -550,10 +765,15 @@ export class DcRouter {
           challengeHandlers: challengeHandlers,
           challengePriority: ['dns-01'],
         });
-        await this.smartAcme.start();
 
         const scheduler = this.certProvisionScheduler;
         smartProxyConfig.certProvisionFunction = async (domain, eventComms) => {
+          // If SmartAcme is not yet ready (still starting or retrying), fall back to HTTP-01
+          if (!this.smartAcmeReady) {
+            eventComms.warn(`SmartAcme not yet initialized, falling back to http-01 for ${domain}`);
+            return 'http01';
+          }
+
           // Check backoff before attempting provision
           if (await scheduler.isInBackoff(domain)) {
             const info = await scheduler.getBackoffInfo(domain);
@@ -914,105 +1134,29 @@ export class DcRouter {
   public async stop() {
     logger.log('info', 'Stopping DcRouter services...');
 
-    // Flush pending DNS batch log
-    if (this.dnsBatchTimer) {
-      clearTimeout(this.dnsBatchTimer);
-      if (this.dnsBatchCount > 0) {
-        logger.log('info', `DNS: ${this.dnsBatchCount} queries processed (rate limited, final flush)`, { zone: 'dns' });
-      }
-      this.dnsBatchTimer = null;
-      this.dnsBatchCount = 0;
-      this.dnsLogWindowSecond = 0;
-      this.dnsLogWindowCount = 0;
+    // Unsubscribe from service events before stopping services
+    if (this.serviceSubjectSubscription) {
+      this.serviceSubjectSubscription.unsubscribe();
+      this.serviceSubjectSubscription = undefined;
     }
 
-    await this.opsServer.stop();
-
-    try {
-      // Remove event listeners before stopping services to prevent leaks
-      if (this.smartProxy) {
-        this.smartProxy.removeAllListeners();
-      }
-      if (this.emailServer) {
-        if ((this.emailServer as any).deliverySystem) {
-          (this.emailServer as any).deliverySystem.removeAllListeners();
-        }
-        this.emailServer.removeAllListeners();
-      }
-      if (this.dnsServer) {
-        this.dnsServer.removeAllListeners();
-      }
-
-      // Stop all services in parallel for faster shutdown
-      await Promise.all([
-        // Stop cache cleaner if running
-        this.cacheCleaner ? Promise.resolve(this.cacheCleaner.stop()) : Promise.resolve(),
-
-        // Stop metrics manager if running
-        this.metricsManager ? this.metricsManager.stop().catch(err => logger.log('error', 'Error stopping MetricsManager', { error: String(err) })) : Promise.resolve(),
-
-        // Stop unified email server if running
-        this.emailServer ? this.emailServer.stop().catch(err => logger.log('error', 'Error stopping email server', { error: String(err) })) : Promise.resolve(),
-
-        // Stop SmartAcme if running
-        this.smartAcme ? this.smartAcme.stop().catch(err => logger.log('error', 'Error stopping SmartAcme', { error: String(err) })) : Promise.resolve(),
-
-        // Stop HTTP SmartProxy if running
-        this.smartProxy ? this.smartProxy.stop().catch(err => logger.log('error', 'Error stopping SmartProxy', { error: String(err) })) : Promise.resolve(),
-
-        // Stop DNS server if running
-        this.dnsServer ?
-          this.dnsServer.stop().catch(err => logger.log('error', 'Error stopping DNS server', { error: String(err) })) :
-          Promise.resolve(),
-
-        // Stop RADIUS server if running
-        this.radiusServer ?
-          this.radiusServer.stop().catch(err => logger.log('error', 'Error stopping RADIUS server', { error: String(err) })) :
-          Promise.resolve(),
+    // ServiceManager handles reverse-dependency-ordered shutdown
+    await this.serviceManager.stop();
 
-        // Stop Remote Ingress tunnel manager if running
-        this.tunnelManager ?
-          this.tunnelManager.stop().catch(err => logger.log('error', 'Error stopping TunnelManager', { error: String(err) })) :
-          Promise.resolve()
-      ]);
-
-      // Stop cache database after other services (they may need it during shutdown)
-      if (this.cacheDb) {
-        await this.cacheDb.stop().catch(err => logger.log('error', 'Error stopping CacheDb', { error: String(err) }));
-        CacheDb.resetInstance();
-      }
-
-      // Clear backoff cache in cert scheduler
-      if (this.certProvisionScheduler) {
-        this.certProvisionScheduler.clear();
-      }
-
-      // Allow GC of stopped services by nulling references
-      this.smartProxy = undefined;
-      this.emailServer = undefined;
-      this.dnsServer = undefined;
-      this.metricsManager = undefined;
-      this.cacheCleaner = undefined;
-      this.cacheDb = undefined;
-      this.tunnelManager = undefined;
-      this.radiusServer = undefined;
-      this.smartAcme = undefined;
+    // Clear backoff cache in cert scheduler
+    if (this.certProvisionScheduler) {
+      this.certProvisionScheduler.clear();
       this.certProvisionScheduler = undefined;
-      this.remoteIngressManager = undefined;
-      this.routeConfigManager = undefined;
-      this.apiTokenManager = undefined;
-      this.certificateStatusMap.clear();
+    }
 
-      // Reset security singletons to allow GC
-      SecurityLogger.resetInstance();
-      ContentScanner.resetInstance();
-      IPReputationChecker.resetInstance();
+    this.certificateStatusMap.clear();
 
-      logger.log('info', 'All DcRouter services stopped');
-    } catch (error) {
-      logger.log('error', 'Error during DcRouter shutdown', { error: String(error) });
-      throw error;
-    }
+    // Reset security singletons to allow GC
+    SecurityLogger.resetInstance();
+    ContentScanner.resetInstance();
+    IPReputationChecker.resetInstance();
+
+    logger.log('info', 'All DcRouter services stopped');
   }
 
   /**

package/ts/index.ts

@@ -35,6 +35,6 @@ export const runCli = async () => {
     await dcRouter.stop();
     process.exit(0);
   };
-  process.on('SIGINT', shutdown);
-  process.on('SIGTERM', shutdown);
+  process.once('SIGINT', shutdown);
+  process.once('SIGTERM', shutdown);
 };

package/ts/opsserver/handlers/stats.handler.ts

@@ -489,44 +489,41 @@ export class StatsHandler {
       message?: string;
     }>;
   }> {
-    const services: Array<{
-      name: string;
-      status: 'healthy' | 'degraded' | 'unhealthy';
-      message?: string;
-    }> = [];
-    
-    // Check HTTP Proxy
-    if (this.opsServerRef.dcRouterRef.smartProxy) {
-      services.push({
-        name: 'HTTP/HTTPS Proxy',
-        status: 'healthy',
-      });
-    }
-    
-    // Check Email Server
-    if (this.opsServerRef.dcRouterRef.emailServer) {
-      services.push({
-        name: 'Email Server',
-        status: 'healthy',
-      });
-    }
-    
-    // Check DNS Server
-    if (this.opsServerRef.dcRouterRef.dnsServer) {
-      services.push({
-        name: 'DNS Server',
-        status: 'healthy',
-      });
-    }
-    
-    // Check OpsServer
-    services.push({
-      name: 'OpsServer',
-      status: 'healthy',
+    const dcRouter = this.opsServerRef.dcRouterRef;
+    const health = dcRouter.serviceManager.getHealth();
+
+    const services = health.services.map((svc) => {
+      let status: 'healthy' | 'degraded' | 'unhealthy';
+      switch (svc.state) {
+        case 'running':
+          status = 'healthy';
+          break;
+        case 'starting':
+        case 'degraded':
+          status = 'degraded';
+          break;
+        case 'failed':
+          status = svc.criticality === 'critical' ? 'unhealthy' : 'degraded';
+          break;
+        case 'stopped':
+        case 'stopping':
+        default:
+          status = 'degraded';
+          break;
+      }
+
+      let message: string | undefined;
+      if (svc.state === 'failed' && svc.lastError) {
+        message = svc.lastError;
+      } else if (svc.retryCount > 0 && svc.state !== 'running') {
+        message = `Retry attempt ${svc.retryCount}`;
+      }
+
+      return { name: svc.name, status, message };
     });
-    
-    const healthy = services.every(s => s.status === 'healthy');
-    
+
+    const healthy = health.overall === 'healthy';
+
     return {
       healthy,
       services,

package/ts/plugins.ts

@@ -62,8 +62,9 @@ import * as smartradius from '@push.rocks/smartradius';
 import * as smartrequest from '@push.rocks/smartrequest';
 import * as smartrx from '@push.rocks/smartrx';
 import * as smartunique from '@push.rocks/smartunique';
+import * as taskbuffer from '@push.rocks/taskbuffer';
 
-export { projectinfo, qenv, smartacme, smartdata, smartdns, smartfile, smartguard, smartjwt, smartlog, smartmetrics, smartmongo, smartmta, smartnetwork, smartpath, smartproxy, smartpromise, smartradius, smartrequest, smartrx, smartunique };
+export { projectinfo, qenv, smartacme, smartdata, smartdns, smartfile, smartguard, smartjwt, smartlog, smartmetrics, smartmongo, smartmta, smartnetwork, smartpath, smartproxy, smartpromise, smartradius, smartrequest, smartrx, smartunique, taskbuffer };
 
 // Define SmartLog types for use in error handling
 export type TLogLevel = 'error' | 'warn' | 'info' | 'success' | 'debug';