@serve.zone/dcrouter 13.40.3 → 13.41.1

package/ts/classes.dcrouter.ts

@@ -33,6 +33,7 @@ import { DnsManager } from './dns/manager.dns.js';
 import { AcmeConfigManager } from './acme/manager.acme-config.js';
 import { EmailDomainManager, SmartMtaStorageManager, WorkAppMailManager, buildEmailDnsRecords } from './email/index.js';
 import type { IRoute } from '../ts_interfaces/data/route-management.js';
+import type { IDcRouterRouteConfig, IRemoteIngressHubSettings, IRemoteIngressPerformanceConfig } from '../ts_interfaces/data/remoteingress.js';
 import type { ISecurityCompiledPolicy } from '../ts_interfaces/data/security-policy.js';
 
 export interface IDcRouterOptions {
@@ -280,6 +281,9 @@ export class DcRouter {
   // Remote Ingress
   public remoteIngressManager?: RemoteIngressManager;
   public tunnelManager?: TunnelManager;
+  private remoteIngressHubLifecycleChain: Promise<void> = Promise.resolve();
+  private remoteIngressHubStopping = false;
+  private remoteIngressHubGeneration = 0;
 
   // VPN
   public vpnManager?: VpnManager;
@@ -326,6 +330,11 @@ export class DcRouter {
   public serviceManager: plugins.taskbuffer.ServiceManager;
   private serviceSubjectSubscription?: plugins.smartrx.rxjs.Subscription;
   public smartAcmeReady = false;
+  private smartAcmeServiceStarted = false;
+  private smartAcmeStartGeneration = 0;
+  private smartAcmeStartPromise?: Promise<void>;
+  private smartAcmeRetryTimer?: ReturnType<typeof setTimeout>;
+  private smartAcmeRetryAttempt = 0;
 
   // TypedRouter for API endpoints
   public typedrouter = new plugins.typedrequest.TypedRouter();
@@ -545,45 +554,14 @@ export class DcRouter {
           .optional()
           .dependsOn('SmartProxy')
           .withStart(async () => {
-            if (this.smartAcme) {
-              await this.smartAcme.start();
-              this.smartAcmeReady = true;
-              logger.log('info', 'SmartAcme DNS-01 provider is now ready');
-
-              // Re-trigger certificate provisioning for all auto-cert routes.
-              // During startup, certProvisionFunction returned 'http01' (SmartAcme not ready),
-              // but Rust ACME is disabled when certProvisionFunction is set — so all domains
-              // failed silently (SmartProxy doesn't emit certificate-failed for this path).
-              // Calling updateRoutes() re-triggers provisionCertificatesViaCallback internally,
-              // which calls certProvisionFunction again — now with smartAcmeReady === true.
-              if (this.routeConfigManager) {
-                // Go through RouteConfigManager to get the full merged route set
-                // and serialize via the route-update mutex (prevents stale overwrites)
-                logger.log('info', 'Re-triggering certificate provisioning via RouteConfigManager');
-                this.routeConfigManager.applyRoutes().catch((err: any) => {
-                  logger.log('warn', `Failed to re-trigger cert provisioning: ${err?.message || err}`);
-                });
-              } else if (this.smartProxy) {
-                // No RouteConfigManager (DB disabled) — re-send current routes to trigger cert provisioning
-                if (this.certProvisionScheduler) {
-                  this.certProvisionScheduler.clear();
-                }
-                const currentRoutes = this.smartProxy.routeManager.getRoutes();
-                logger.log('info', `Re-triggering certificate provisioning for ${currentRoutes.length} routes`);
-                this.smartProxy.updateRoutes(currentRoutes).catch((err: any) => {
-                  logger.log('warn', `Failed to re-trigger cert provisioning: ${err?.message || err}`);
-                });
-              }
-            }
+            this.smartAcmeServiceStarted = true;
+            this.startSmartAcmeInBackground();
           })
           .withStop(async () => {
-            this.smartAcmeReady = false;
-            if (this.smartAcme) {
-              await this.smartAcme.stop();
-              this.smartAcme = undefined;
-            }
+            this.smartAcmeServiceStarted = false;
+            await this.stopSmartAcme();
           })
-          .withRetry({ maxRetries: 20, baseDelayMs: 5000, maxDelayMs: 3_600_000, backoffFactor: 2 }),
+          .withRetry({ maxRetries: 0 }),
       );
     }
 
@@ -613,15 +591,10 @@ export class DcRouter {
               // Sync routes to RemoteIngressManager whenever routes change,
               // then push updated derived ports to the Rust hub binary
               async (routes) => {
-                if (this.remoteIngressManager) {
-                  this.remoteIngressManager.setRoutes(routes as any[]);
-                }
-                if (this.tunnelManager) {
-                  try {
-                    await this.tunnelManager.syncAllowedEdges();
-                  } catch (err: unknown) {
-                    logger.log('error', `Failed to sync Remote Ingress allowed edges: ${(err as Error).message}`);
-                  }
+                try {
+                  await this.updateRemoteIngressRoutes(routes as IDcRouterRouteConfig[]);
+                } catch (err: unknown) {
+                  logger.log('error', `Failed to sync Remote Ingress allowed edges: ${(err as Error).message}`);
                 }
               },
               undefined,
@@ -739,11 +712,7 @@ export class DcRouter {
             await this.setupRemoteIngress();
           })
           .withStop(async () => {
-            if (this.tunnelManager) {
-              await this.tunnelManager.stop();
-              this.tunnelManager = undefined;
-            }
-            this.remoteIngressManager = undefined;
+            await this.stopRemoteIngress();
           })
           .withRetry({ maxRetries: 3, baseDelayMs: 2000, maxDelayMs: 30_000 }),
       );
@@ -783,6 +752,138 @@ export class DcRouter {
     });
   }
 
+  private startSmartAcmeInBackground(): void {
+    if (!this.smartAcme) {
+      this.smartAcmeReady = false;
+      return;
+    }
+
+    const generation = ++this.smartAcmeStartGeneration;
+    this.smartAcmeReady = false;
+    this.smartAcmeRetryAttempt = 0;
+    this.clearSmartAcmeRetryTimer();
+    this.scheduleSmartAcmeStart(generation, 0);
+  }
+
+  private scheduleSmartAcmeStart(generation: number, delayMs: number): void {
+    this.clearSmartAcmeRetryTimer();
+    const retryTimer = setTimeout(() => {
+      this.smartAcmeRetryTimer = undefined;
+      this.runSmartAcmeStartAttempt(generation).catch((err) => {
+        logger.log('error', `Unexpected SmartAcme startup error: ${(err as Error).message}`);
+      });
+    }, delayMs);
+    this.smartAcmeRetryTimer = retryTimer;
+    const unrefableTimer = retryTimer as any;
+    if (typeof unrefableTimer?.unref === 'function') {
+      unrefableTimer.unref();
+    }
+  }
+
+  private async runSmartAcmeStartAttempt(generation: number): Promise<void> {
+    const smartAcme = this.smartAcme;
+    if (!smartAcme || generation !== this.smartAcmeStartGeneration) {
+      return;
+    }
+
+    const startPromise = smartAcme.start();
+    this.smartAcmeStartPromise = startPromise;
+
+    try {
+      await startPromise;
+      if (generation !== this.smartAcmeStartGeneration || this.smartAcme !== smartAcme) {
+        await smartAcme.stop().catch((err) => {
+          logger.log('warn', `Failed to stop stale SmartAcme instance: ${(err as Error).message}`);
+        });
+        return;
+      }
+
+      this.smartAcmeReady = true;
+      this.smartAcmeRetryAttempt = 0;
+      logger.log('info', 'SmartAcme DNS-01 provider is now ready');
+      this.retriggerCertificateProvisioningAfterSmartAcmeReady();
+    } catch (err) {
+      if (generation !== this.smartAcmeStartGeneration || this.smartAcme !== smartAcme) {
+        return;
+      }
+
+      this.smartAcmeReady = false;
+      await smartAcme.stop().catch((stopErr) => {
+        logger.log('warn', `Failed to clean up SmartAcme after startup failure: ${(stopErr as Error).message}`);
+      });
+      this.smartAcmeRetryAttempt++;
+      if (this.smartAcmeRetryAttempt > 20) {
+        logger.log('error', `SmartAcme DNS-01 provider failed after 20 startup attempts: ${(err as Error).message}`);
+        return;
+      }
+
+      const baseDelayMs = 5000;
+      const maxDelayMs = 3_600_000;
+      const delayMs = Math.min(baseDelayMs * Math.pow(2, this.smartAcmeRetryAttempt - 1), maxDelayMs);
+      const jitter = 0.8 + Math.random() * 0.4;
+      const actualDelayMs = Math.floor(delayMs * jitter);
+      logger.log('warn', `SmartAcme DNS-01 provider startup failed: ${(err as Error).message}; retrying in ${actualDelayMs}ms (attempt ${this.smartAcmeRetryAttempt}/20)`);
+      this.scheduleSmartAcmeStart(generation, actualDelayMs);
+    } finally {
+      if (this.smartAcmeStartPromise === startPromise) {
+        this.smartAcmeStartPromise = undefined;
+      }
+    }
+  }
+
+  private retriggerCertificateProvisioningAfterSmartAcmeReady(): void {
+    // During startup, certProvisionFunction returns 'http01' while SmartAcme is not ready,
+    // but Rust ACME is disabled when certProvisionFunction is set. Re-applying routes
+    // retries provisioning now that DNS-01 is available.
+    if (this.routeConfigManager) {
+      logger.log('info', 'Re-triggering certificate provisioning via RouteConfigManager');
+      this.routeConfigManager.applyRoutes().catch((err: any) => {
+        logger.log('warn', `Failed to re-trigger cert provisioning: ${err?.message || err}`);
+      });
+      return;
+    }
+
+    if (this.smartProxy) {
+      if (this.certProvisionScheduler) {
+        this.certProvisionScheduler.clear();
+      }
+      const currentRoutes = this.smartProxy.routeManager.getRoutes();
+      logger.log('info', `Re-triggering certificate provisioning for ${currentRoutes.length} routes`);
+      this.smartProxy.updateRoutes(currentRoutes).catch((err: any) => {
+        logger.log('warn', `Failed to re-trigger cert provisioning: ${err?.message || err}`);
+      });
+    }
+  }
+
+  private clearSmartAcmeRetryTimer(): void {
+    if (this.smartAcmeRetryTimer) {
+      clearTimeout(this.smartAcmeRetryTimer);
+      this.smartAcmeRetryTimer = undefined;
+    }
+  }
+
+  private async stopSmartAcme(): Promise<void> {
+    this.smartAcmeStartGeneration++;
+    this.smartAcmeReady = false;
+    this.smartAcmeRetryAttempt = 0;
+    this.clearSmartAcmeRetryTimer();
+
+    const smartAcme = this.smartAcme;
+    if (!smartAcme) {
+      return;
+    }
+
+    try {
+      await smartAcme.stop();
+    } catch (err) {
+      logger.log('error', 'Error stopping SmartAcme', { error: String(err) });
+    } finally {
+      if (this.smartAcme === smartAcme) {
+        this.smartAcme = undefined;
+      }
+    }
+  }
+
   public async start() {
     await this.checkSystemLimits();
     logger.log('info', 'Starting DcRouter Services');
@@ -1098,17 +1199,13 @@ export class DcRouter {
       // Initialize cert provision scheduler
       this.certProvisionScheduler = new CertProvisionScheduler();
 
-      // If we have DNS challenge handlers, create SmartAcme instance and wire certProvisionFunction
-      // Note: SmartAcme.start() is NOT called here — it runs as a separate optional service
-      // via the ServiceManager, with aggressive retry for rate-limit resilience.
+      // If we have DNS challenge handlers, create SmartAcme instance and wire certProvisionFunction.
+      // SmartAcme starts in the background because ACME account setup can be slow or rate-limited,
+      // and must not block dcrouter's global startup timeout.
+      if (this.smartAcme) {
+        await this.stopSmartAcme();
+      }
       if (challengeHandlers.length > 0) {
-        // Stop old SmartAcme if it exists (e.g., during updateSmartProxyConfig)
-        if (this.smartAcme) {
-          this.smartAcmeReady = false;
-          await this.smartAcme.stop().catch(err =>
-            logger.log('error', 'Error stopping old SmartAcme', { error: String(err) })
-          );
-        }
         // Safe non-null: challengeHandlers.length > 0 implies both dnsManager
         // and acmeConfig exist (enforced above).
         this.smartAcme = new plugins.smartacme.SmartAcme({
@@ -1118,6 +1215,9 @@ export class DcRouter {
           challengeHandlers: challengeHandlers,
           challengePriority: ['dns-01'],
         });
+        if (this.smartAcmeServiceStarted) {
+          this.startSmartAcmeInBackground();
+        }
 
         const scheduler = this.certProvisionScheduler;
         smartProxyConfig.certProvisionFallbackToAcme = false;
@@ -1319,12 +1419,15 @@ export class DcRouter {
     }
 
     const firewallConfig = await this.securityPolicyManager.compileRemoteIngressFirewall();
-    if (this.remoteIngressManager) {
-      (this.remoteIngressManager as any).setFirewallConfig?.(firewallConfig);
-    }
-    if (this.tunnelManager) {
-      await this.tunnelManager.syncAllowedEdges();
-    }
+    await this.queueRemoteIngressHubTask(async () => {
+      if (this.remoteIngressHubStopping) return;
+      if (this.remoteIngressManager) {
+        this.remoteIngressManager.setFirewallConfig(firewallConfig);
+      }
+      if (this.tunnelManager) {
+        await this.tunnelManager.syncAllowedEdges();
+      }
+    });
   }
 
   private mergeSecurityPolicies(
@@ -2340,28 +2443,180 @@ export class DcRouter {
     }
 
     logger.log('info', 'Setting up Remote Ingress hub...');
+    this.remoteIngressHubStopping = false;
+    const generation = ++this.remoteIngressHubGeneration;
 
     // Initialize the edge registration manager
-    this.remoteIngressManager = new RemoteIngressManager();
-    await this.remoteIngressManager.initialize();
-    this.remoteIngressManager.setFirewallConfig(
-      await this.securityPolicyManager?.compileRemoteIngressFirewall(),
-    );
+    const remoteIngressManager = new RemoteIngressManager(this.options.remoteIngressConfig.performance);
+    this.remoteIngressManager = remoteIngressManager;
+    await remoteIngressManager.initialize();
+    if (!this.isRemoteIngressHubGenerationCurrent(generation, remoteIngressManager)) {
+      return;
+    }
+
+    const firewallConfig = await this.securityPolicyManager?.compileRemoteIngressFirewall();
+    if (!this.isRemoteIngressHubGenerationCurrent(generation, remoteIngressManager)) {
+      return;
+    }
+    remoteIngressManager.setFirewallConfig(firewallConfig);
 
     // Pass current bootstrap routes so the manager can derive edge ports initially.
     // Once RouteConfigManager applies the full DB set, the onRoutesApplied callback
     // will push the complete merged routes here.
     const bootstrapRoutes = [...this.seedConfigRoutes, ...this.seedEmailRoutes, ...this.runtimeDnsRoutes];
-    this.remoteIngressManager.setRoutes(bootstrapRoutes as any[]);
+    remoteIngressManager.setRoutes(bootstrapRoutes as any[]);
 
     // If ConfigManagers finished before us, re-apply routes
     // so the callback delivers the full DB set to our newly-created remoteIngressManager.
     if (this.routeConfigManager) {
       await this.routeConfigManager.applyRoutes();
     }
+    if (!this.isRemoteIngressHubGenerationCurrent(generation, remoteIngressManager)) {
+      return;
+    }
 
-    // Resolve TLS certs for tunnel: explicit paths > ACME for hubDomain > self-signed (Rust default)
+    await this.queueRemoteIngressHubTask(async () => {
+      await this.startRemoteIngressTunnelHubLocked(generation);
+    });
+    if (!this.isRemoteIngressHubGenerationCurrent(generation, remoteIngressManager)) {
+      return;
+    }
+
+    const edgeCount = remoteIngressManager.getAllEdges().length;
+    logger.log('info', `Remote Ingress hub started on port ${this.options.remoteIngressConfig.tunnelPort || 8443} with ${edgeCount} registered edge(s)`);
+  }
+
+  private isRemoteIngressHubGenerationCurrent(generation: number, manager: RemoteIngressManager): boolean {
+    return !this.remoteIngressHubStopping
+      && generation === this.remoteIngressHubGeneration
+      && this.remoteIngressManager === manager;
+  }
+
+  private queueRemoteIngressHubTask<T>(task: () => Promise<T>): Promise<T> {
+    const run = this.remoteIngressHubLifecycleChain.then(task);
+    this.remoteIngressHubLifecycleChain = run.then(() => undefined, () => undefined);
+    return run;
+  }
+
+  private async stopRemoteIngress(): Promise<void> {
+    this.remoteIngressHubStopping = true;
+    this.remoteIngressHubGeneration++;
+    await this.queueRemoteIngressHubTask(async () => {
+      const currentTunnelManager = this.tunnelManager;
+      this.tunnelManager = undefined;
+      if (currentTunnelManager) {
+        await currentTunnelManager.stop();
+      }
+    });
+    this.remoteIngressManager = undefined;
+  }
+
+  public async mutateRemoteIngressEdges<T>(
+    mutation: (manager: RemoteIngressManager) => Promise<T>,
+    syncAllowedEdges = true,
+  ): Promise<T> {
+    return await this.queueRemoteIngressHubTask(async () => {
+      if (this.remoteIngressHubStopping) {
+        throw new Error('RemoteIngress is stopping');
+      }
+      const manager = this.remoteIngressManager;
+      if (!manager) {
+        throw new Error('RemoteIngress not configured');
+      }
+      const result = await mutation(manager);
+      if (syncAllowedEdges && this.tunnelManager) {
+        await this.tunnelManager.syncAllowedEdges();
+      }
+      return result;
+    });
+  }
+
+  private async updateRemoteIngressRoutes(routes: IDcRouterRouteConfig[]): Promise<void> {
+    await this.queueRemoteIngressHubTask(async () => {
+      if (this.remoteIngressHubStopping) return;
+      if (this.remoteIngressManager) {
+        this.remoteIngressManager.setRoutes(routes);
+      }
+      if (this.tunnelManager) {
+        await this.tunnelManager.syncAllowedEdges();
+      }
+    });
+  }
+
+  public async updateRemoteIngressHubSettings(
+    updates: { performance?: IRemoteIngressPerformanceConfig },
+    updatedBy: string,
+  ): Promise<IRemoteIngressHubSettings> {
+    return await this.queueRemoteIngressHubTask(async () => {
+      if (this.remoteIngressHubStopping) {
+        throw new Error('RemoteIngress is stopping');
+      }
+      if (!this.remoteIngressManager) {
+        throw new Error('RemoteIngress is not configured');
+      }
+
+      const settings = await this.remoteIngressManager.updateHubSettings(updates, updatedBy);
+      if (this.options.remoteIngressConfig?.enabled) {
+        await this.restartRemoteIngressTunnelHubLocked();
+      }
+      return settings;
+    });
+  }
+
+  private async restartRemoteIngressTunnelHubLocked(): Promise<void> {
+    const generation = ++this.remoteIngressHubGeneration;
+    if (!this.remoteIngressManager || !this.options.remoteIngressConfig?.enabled || this.remoteIngressHubStopping) {
+      return;
+    }
+
+    const currentTunnelManager = this.tunnelManager;
+    this.tunnelManager = undefined;
+    if (currentTunnelManager) {
+      await currentTunnelManager.stop();
+    }
+
+    if (this.remoteIngressHubStopping || generation !== this.remoteIngressHubGeneration) {
+      return;
+    }
+    await this.startRemoteIngressTunnelHubLocked(generation);
+  }
+
+  private async startRemoteIngressTunnelHubLocked(generation: number): Promise<void> {
     const riCfg = this.options.remoteIngressConfig;
+    const manager = this.remoteIngressManager;
+    if (!riCfg?.enabled || !manager || this.remoteIngressHubStopping || generation !== this.remoteIngressHubGeneration) {
+      return;
+    }
+
+    const tlsConfig = await this.resolveRemoteIngressTlsConfig(riCfg);
+    if (this.remoteIngressHubStopping || generation !== this.remoteIngressHubGeneration || this.remoteIngressManager !== manager) {
+      return;
+    }
+
+    const tunnelManager = new TunnelManager(manager, {
+      tunnelPort: riCfg.tunnelPort ?? 8443,
+      targetHost: '127.0.0.1',
+      tls: tlsConfig,
+      performance: manager.getHubPerformanceConfig(),
+    });
+    try {
+      await tunnelManager.start();
+    } catch (err) {
+      await tunnelManager.stop().catch(() => {});
+      throw err;
+    }
+
+    if (this.remoteIngressHubStopping || generation !== this.remoteIngressHubGeneration || this.remoteIngressManager !== manager) {
+      await tunnelManager.stop();
+      return;
+    }
+    this.tunnelManager = tunnelManager;
+  }
+
+  private async resolveRemoteIngressTlsConfig(
+    riCfg: NonNullable<IDcRouterOptions['remoteIngressConfig']>,
+  ): Promise<{ certPem: string; keyPem: string } | undefined> {
+    // Resolve TLS certs for tunnel: explicit paths > ACME for hubDomain > self-signed (Rust default)
     let tlsConfig: { certPem: string; keyPem: string } | undefined;
 
     // Priority 1: Explicit cert/key file paths
@@ -2391,17 +2646,7 @@ export class DcRouter {
       logger.log('info', 'No TLS cert configured for RemoteIngress tunnel — using auto-generated self-signed');
     }
 
-    // Create and start the tunnel manager
-    this.tunnelManager = new TunnelManager(this.remoteIngressManager, {
-      tunnelPort: riCfg.tunnelPort ?? 8443,
-      targetHost: '127.0.0.1',
-      tls: tlsConfig,
-      performance: riCfg.performance,
-    });
-    await this.tunnelManager.start();
-
-    const edgeCount = this.remoteIngressManager.getAllEdges().length;
-    logger.log('info', `Remote Ingress hub started on port ${this.options.remoteIngressConfig.tunnelPort || 8443} with ${edgeCount} registered edge(s)`);
+    return tlsConfig;
   }
 
   /**

package/ts/db/documents/classes.remote-ingress-hub-settings.doc.ts

@@ -0,0 +1,29 @@
+import * as plugins from '../../plugins.js';
+import { DcRouterDb } from '../classes.dcrouter-db.js';
+import type { IRemoteIngressPerformanceConfig } from '../../../ts_interfaces/data/remoteingress.js';
+
+const getDb = () => DcRouterDb.getInstance().getDb();
+
+@plugins.smartdata.Collection(() => getDb())
+export class RemoteIngressHubSettingsDoc extends plugins.smartdata.SmartDataDbDoc<RemoteIngressHubSettingsDoc, RemoteIngressHubSettingsDoc> {
+  @plugins.smartdata.unI()
+  @plugins.smartdata.svDb()
+  public settingsId: string = 'remote-ingress-hub-settings';
+
+  @plugins.smartdata.svDb()
+  public performance?: IRemoteIngressPerformanceConfig;
+
+  @plugins.smartdata.svDb()
+  public updatedAt: number = 0;
+
+  @plugins.smartdata.svDb()
+  public updatedBy: string = '';
+
+  constructor() {
+    super();
+  }
+
+  public static async load(): Promise<RemoteIngressHubSettingsDoc | null> {
+    return await RemoteIngressHubSettingsDoc.getInstance({ settingsId: 'remote-ingress-hub-settings' });
+  }
+}

package/ts/db/documents/index.ts

@@ -24,6 +24,7 @@ export * from './classes.cert-backoff.doc.js';
 
 // Remote ingress document classes
 export * from './classes.remote-ingress-edge.doc.js';
+export * from './classes.remote-ingress-hub-settings.doc.js';
 
 // RADIUS document classes
 export * from './classes.vlan-mappings.doc.js';

package/ts/opsserver/handlers/config.handler.ts

@@ -208,7 +208,7 @@ export class ConfigHandler {
       hubDomain: riCfg?.hubDomain || null,
       tlsMode,
       connectedEdgeIps,
-      performance: riCfg?.performance,
+      performance: dcRouter.remoteIngressManager?.getHubPerformanceConfig() || riCfg?.performance,
     };
 
     return {