@serve.zone/dcrouter 13.17.9 → 13.19.0

package/ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@serve.zone/dcrouter',
-  version: '13.17.9',
+  version: '13.19.0',
   description: 'A multifaceted routing service handling mail and SMS delivery functions.'
 }

package/ts/classes.dcrouter.ts

@@ -9,6 +9,7 @@ import {
   type IUnifiedEmailServerOptions,
   type IEmailRoute,
   type IEmailDomainConfig,
+  type IStorageManagerLike,
 } from '@push.rocks/smartmta';
 import { logger } from './logger.js';
 import { StorageBackedCertManager } from './classes.storage-cert-manager.js';
@@ -29,7 +30,8 @@ import { SecurityLogger, ContentScanner, IPReputationChecker } from './security/
 import { type IHttp3Config, augmentRoutesWithHttp3 } from './http3/index.js';
 import { DnsManager } from './dns/manager.dns.js';
 import { AcmeConfigManager } from './acme/manager.acme-config.js';
-import { EmailDomainManager } from './email/classes.email-domain.manager.js';
+import { EmailDomainManager, SmartMtaStorageManager, buildEmailDnsRecords } from './email/index.js';
+import type { IRoute } from '../ts_interfaces/data/route-management.js';
 
 export interface IDcRouterOptions {
   /** Base directory for all dcrouter data. Defaults to ~/.serve.zone/dcrouter */
@@ -248,15 +250,13 @@ export class DcRouter {
   public radiusServer?: RadiusServer;
   public opsServer!: OpsServer;
   public metricsManager?: MetricsManager;
+  private emailEventSubscriptions: Array<{
+    emitter: { off(eventName: string, listener: (...args: any[]) => void): void };
+    eventName: string;
+    listener: (...args: any[]) => void;
+  }> = [];
 
-  // Compatibility shim for smartmta's DkimManager which calls dcRouter.storageManager.set()
-  public storageManager: any = {
-    get: async (_key: string) => null,
-    set: async (_key: string, _value: string) => {
-      // DKIM keys from smartmta — logged but not yet migrated to smartdata
-      logger.log('debug', `storageManager.set() called (compat shim) for key: ${_key}`);
-    },
-  };
+  public storageManager: IStorageManagerLike;
 
   // Unified database (smartdata + LocalSmartDb or external MongoDB)
   public dcRouterDb?: DcRouterDb;
@@ -315,7 +315,8 @@ export class DcRouter {
   // Seed routes assembled during setupSmartProxy, passed to RouteConfigManager for DB seeding
   private seedConfigRoutes: plugins.smartproxy.IRouteConfig[] = [];
   private seedEmailRoutes: plugins.smartproxy.IRouteConfig[] = [];
-  // Runtime-only DoH routes. These carry live socket handlers and must never be persisted.
+  private seedDnsRoutes: plugins.smartproxy.IRouteConfig[] = [];
+  // Live DoH routes used during SmartProxy bootstrap before RouteConfigManager re-applies stored routes.
   private runtimeDnsRoutes: plugins.smartproxy.IRouteConfig[] = [];
 
   // Environment access
@@ -329,6 +330,10 @@ export class DcRouter {
 
     // Resolve all data paths from baseDir
     this.resolvedPaths = paths.resolvePaths(this.options.baseDir);
+    paths.ensureDataDirectories(this.resolvedPaths);
+    this.storageManager = new SmartMtaStorageManager(
+      plugins.path.join(this.resolvedPaths.dataDir, 'smartmta-storage')
+    );
 
     // Initialize service manager and register all services
     this.serviceManager = new plugins.taskbuffer.ServiceManager({
@@ -452,9 +457,13 @@ export class DcRouter {
           .dependsOn('DcRouterDb')
           .withStart(async () => {
             this.emailDomainManager = new EmailDomainManager(this);
+            await this.emailDomainManager.start();
           })
           .withStop(async () => {
-            this.emailDomainManager = undefined;
+            if (this.emailDomainManager) {
+              await this.emailDomainManager.stop();
+              this.emailDomainManager = undefined;
+            }
           }),
       );
     }
@@ -581,13 +590,15 @@ export class DcRouter {
                   this.tunnelManager.syncAllowedEdges();
                 }
               },
-              () => this.runtimeDnsRoutes,
+              undefined,
+              (storedRoute: IRoute) => this.hydrateStoredRouteForRuntime(storedRoute),
             );
             this.apiTokenManager = new ApiTokenManager();
             await this.apiTokenManager.initialize();
             await this.routeConfigManager.initialize(
               this.seedConfigRoutes as import('../ts_interfaces/data/remoteingress.js').IDcRouterRouteConfig[],
               this.seedEmailRoutes as import('../ts_interfaces/data/remoteingress.js').IDcRouterRouteConfig[],
+              this.seedDnsRoutes as import('../ts_interfaces/data/remoteingress.js').IDcRouterRouteConfig[],
             );
             await this.targetProfileManager.normalizeAllRouteRefs();
 
@@ -610,19 +621,20 @@ export class DcRouter {
 
     // Email Server: optional, depends on SmartProxy
     if (this.options.emailConfig) {
+      const emailServiceDeps = ['SmartProxy', 'MetricsManager'];
+      if (this.options.dbConfig?.enabled !== false) {
+        emailServiceDeps.push('EmailDomainManager');
+      }
       this.serviceManager.addService(
         new plugins.taskbuffer.Service('EmailServer')
           .optional()
-          .dependsOn('SmartProxy')
+          .dependsOn(...emailServiceDeps)
           .withStart(async () => {
             await this.setupUnifiedEmailHandling();
           })
           .withStop(async () => {
             if (this.emailServer) {
-              if ((this.emailServer as any).deliverySystem) {
-                (this.emailServer as any).deliverySystem.removeAllListeners();
-              }
-              this.emailServer.removeAllListeners();
+              this.clearEmailEventSubscriptions();
               await this.emailServer.stop();
               this.emailServer = undefined;
             }
@@ -636,7 +648,7 @@ export class DcRouter {
       this.serviceManager.addService(
         new plugins.taskbuffer.Service('DnsServer')
           .optional()
-          .dependsOn('SmartProxy')
+          .dependsOn('SmartProxy', ...(this.options.emailConfig ? ['EmailServer'] : []))
           .withStart(async () => {
             await this.setupDnsWithSocketHandler();
           })
@@ -904,10 +916,12 @@ export class DcRouter {
       logger.log('debug', 'Email routes generated', { routes: JSON.stringify(this.seedEmailRoutes) });
     }
 
+    this.seedDnsRoutes = [];
     this.runtimeDnsRoutes = [];
     if (this.options.dnsNsDomains && this.options.dnsNsDomains.length > 0) {
-      this.runtimeDnsRoutes = this.generateDnsRoutes();
-      logger.log('debug', `DNS routes for nameservers ${this.options.dnsNsDomains.join(', ')}`, { routes: JSON.stringify(this.runtimeDnsRoutes) });
+      this.seedDnsRoutes = this.generateDnsRoutes({ includeSocketHandler: false });
+      this.runtimeDnsRoutes = this.generateDnsRoutes({ includeSocketHandler: true });
+      logger.log('debug', `DNS routes for nameservers ${this.options.dnsNsDomains.join(', ')}`, { routes: JSON.stringify(this.seedDnsRoutes) });
     }
 
     // Combined routes for SmartProxy bootstrap (before DB routes are loaded)
@@ -1330,19 +1344,20 @@ export class DcRouter {
   /**
    * Generate SmartProxy routes for DNS configuration
    */
-  private generateDnsRoutes(): plugins.smartproxy.IRouteConfig[] {
+  private generateDnsRoutes(options?: { includeSocketHandler?: boolean }): plugins.smartproxy.IRouteConfig[] {
     if (!this.options.dnsNsDomains || this.options.dnsNsDomains.length === 0) {
       return [];
     }
-    
+
+    const includeSocketHandler = options?.includeSocketHandler !== false;
     const dnsRoutes: plugins.smartproxy.IRouteConfig[] = [];
-    
+
     // Create routes for DNS-over-HTTPS paths
     const dohPaths = ['/dns-query', '/resolve'];
-    
+
     // Use the first nameserver domain for DoH routes
     const primaryNameserver = this.options.dnsNsDomains[0];
-    
+
     for (const path of dohPaths) {
       const dohRoute: plugins.smartproxy.IRouteConfig = {
         name: `dns-over-https-${path.replace('/', '')}`,
@@ -1351,18 +1366,42 @@ export class DcRouter {
           domains: [primaryNameserver],
           path: path
         },
-        action: {
-          type: 'socket-handler' as any,
-          socketHandler: this.createDnsSocketHandler()
-        } as any
+        action: includeSocketHandler
+          ? {
+              type: 'socket-handler' as any,
+              socketHandler: this.createDnsSocketHandler()
+            } as any
+          : {
+              type: 'socket-handler' as any,
+            } as any
       };
-      
+
       dnsRoutes.push(dohRoute);
     }
-    
+
     return dnsRoutes;
   }
 
+  private hydrateStoredRouteForRuntime(storedRoute: IRoute): plugins.smartproxy.IRouteConfig | undefined {
+    const routeName = storedRoute.route.name || '';
+    const isDohRoute = storedRoute.origin === 'dns'
+      && storedRoute.route.action?.type === 'socket-handler'
+      && routeName.startsWith('dns-over-https-');
+
+    if (!isDohRoute) {
+      return undefined;
+    }
+
+    return {
+      ...storedRoute.route,
+      action: {
+        ...storedRoute.route.action,
+        type: 'socket-handler' as any,
+        socketHandler: this.createDnsSocketHandler(),
+      } as any,
+    };
+  }
+
   /**
    * Check if a domain matches a pattern (including wildcard support)
    * @param domain The domain to check
@@ -1511,40 +1550,74 @@ export class DcRouter {
       ...this.options.emailConfig,
       domains: transformedDomains,
       ports: this.options.emailConfig.ports.map(port => portMapping[port] || port + 10000),
-      hostname: 'localhost' // Listen on localhost for SmartProxy forwarding
+      persistRoutes: this.options.emailConfig.persistRoutes ?? false,
+      queue: {
+        storageType: 'disk',
+        persistentPath: plugins.path.join(this.resolvedPaths.dataDir, 'smartmta-queue'),
+        ...this.options.emailConfig.queue,
+      },
     };
     
     // Create unified email server
     this.emailServer = new UnifiedEmailServer(this, emailConfig);
+    this.clearEmailEventSubscriptions();
     
     // Set up error handling
-    this.emailServer.on('error', (err: Error) => {
+    this.addEmailEventSubscription(this.emailServer, 'error', (err: Error) => {
       logger.log('error', `UnifiedEmailServer error: ${err.message}`);
     });
     
     // Start the server
     await this.emailServer.start();
 
-    // Wire delivery events to MetricsManager and logger
-    if (this.metricsManager && this.emailServer.deliverySystem) {
-      this.emailServer.deliverySystem.on('deliveryStart', (item: any) => {
-        this.metricsManager!.trackEmailReceived(item?.from);
-        logger.log('info', `Email delivery started: ${item?.from} → ${item?.to}`, { zone: 'email' });
+    // Wire delivery events to MetricsManager and logger using smartmta's public queue APIs.
+    if (this.metricsManager && this.emailServer) {
+      const getEnvelope = (item: { processingResult?: any; lastError?: string }) => {
+        const emailLike = item?.processingResult;
+        const from = emailLike?.from || emailLike?.email?.from || '';
+        const recipients = Array.isArray(emailLike?.to)
+          ? emailLike.to
+          : Array.isArray(emailLike?.email?.to)
+            ? emailLike.email.to
+            : [];
+        return {
+          from,
+          recipients: recipients.filter(Boolean),
+        };
+      };
+      const updateQueueSize = () => {
+        this.metricsManager!.updateQueueSize(this.emailServer!.getQueueStats().queueSize);
+      };
+
+      this.addEmailEventSubscription(this.emailServer.deliveryQueue, 'itemEnqueued', (item: any) => {
+        const envelope = getEnvelope(item);
+        this.metricsManager!.trackEmailReceived(envelope.from);
+        updateQueueSize();
+        logger.log('info', `Email queued: ${envelope.from} → ${envelope.recipients.join(', ') || 'unknown'}`, { zone: 'email' });
       });
-      this.emailServer.deliverySystem.on('deliverySuccess', (item: any) => {
-        this.metricsManager!.trackEmailSent(item?.to);
-        logger.log('info', `Email delivered to ${item?.to}`, { zone: 'email' });
+      this.addEmailEventSubscription(this.emailServer.deliveryQueue, 'itemDelivered', (item: any) => {
+        const envelope = getEnvelope(item);
+        this.metricsManager!.trackEmailSent(envelope.recipients[0]);
+        updateQueueSize();
+        logger.log('info', `Email delivered to ${envelope.recipients.join(', ') || 'unknown'}`, { zone: 'email' });
       });
-      this.emailServer.deliverySystem.on('deliveryFailed', (item: any, error: any) => {
-        this.metricsManager!.trackEmailFailed(item?.to, error?.message);
-        logger.log('warn', `Email delivery failed to ${item?.to}: ${error?.message}`, { zone: 'email' });
+      this.addEmailEventSubscription(this.emailServer.deliveryQueue, 'itemFailed', (item: any) => {
+        const envelope = getEnvelope(item);
+        this.metricsManager!.trackEmailFailed(envelope.recipients[0], item?.lastError);
+        updateQueueSize();
+        logger.log('warn', `Email delivery failed to ${envelope.recipients.join(', ') || 'unknown'}: ${item?.lastError || 'unknown error'}`, { zone: 'email' });
       });
-    }
-    if (this.metricsManager && this.emailServer) {
-      this.emailServer.on('bounceProcessed', () => {
+      this.addEmailEventSubscription(this.emailServer.deliveryQueue, 'itemDeferred', () => {
+        updateQueueSize();
+      });
+      this.addEmailEventSubscription(this.emailServer.deliveryQueue, 'itemRemoved', () => {
+        updateQueueSize();
+      });
+      this.addEmailEventSubscription(this.emailServer, 'bounceProcessed', () => {
         this.metricsManager!.trackEmailBounced();
         logger.log('warn', 'Email bounce processed', { zone: 'email' });
       });
+      updateQueueSize();
     }
 
     logger.log('info', `Email server started on ports: ${emailConfig.ports.join(', ')}`);
@@ -1574,11 +1647,7 @@ export class DcRouter {
     try {
       // Stop the unified email server which contains all components
       if (this.emailServer) {
-        // Remove listeners before stopping to prevent leaks on config update cycles
-        if ((this.emailServer as any).deliverySystem) {
-          (this.emailServer as any).deliverySystem.removeAllListeners();
-        }
-        this.emailServer.removeAllListeners();
+        this.clearEmailEventSubscriptions();
         await this.emailServer.stop();
         logger.log('info', 'Unified email server stopped');
         this.emailServer = undefined;
@@ -1783,14 +1852,14 @@ export class DcRouter {
     // Generate and register authoritative records
     const authoritativeRecords = await this.generateAuthoritativeRecords();
     
-    // Generate email DNS records
-    const emailDnsRecords = await this.generateEmailDnsRecords();
-    
-    // Initialize DKIM for all email domains
-    await this.initializeDkimForEmailDomains();
-    
-    // Load DKIM records from JSON files (they should now exist)
-    const dkimRecords = await this.loadDkimRecords();
+     // Generate email DNS records
+     const emailDnsRecords = await this.generateEmailDnsRecords();
+     
+     // Ensure DKIM keys exist for internal-dns domains before generating records.
+     await this.initializeDkimForEmailDomains();
+     
+     // Generate DKIM records directly from smartmta instead of scanning legacy JSON files.
+     const dkimRecords = await this.loadDkimRecords();
     
     // Combine all records: authoritative, email, DKIM, and user-defined
     const allRecords = [...authoritativeRecords, ...emailDnsRecords, ...dkimRecords];
@@ -1901,37 +1970,20 @@ export class DcRouter {
     for (const domainConfig of internalDnsDomains) {
       const domain = domainConfig.domain;
       const ttl = domainConfig.dns?.internal?.ttl || 3600;
-      const mxPriority = domainConfig.dns?.internal?.mxPriority || 10;
-      
-      // MX record - points to the domain itself for email handling
-      records.push({
-        name: domain,
-        type: 'MX',
-        value: `${mxPriority} ${domain}`,
-        ttl
-      });
-      
-      // SPF record - using sensible defaults
-      const spfRecord = 'v=spf1 a mx ~all';
-      records.push({
-        name: domain,
-        type: 'TXT',
-        value: spfRecord,
-        ttl
-      });
-      
-      // DMARC record - using sensible defaults
-      const dmarcPolicy = 'none'; // Start with 'none' policy for monitoring
-      const dmarcEmail = `dmarc@${domain}`;
-      records.push({
-        name: `_dmarc.${domain}`,
-        type: 'TXT',
-        value: `v=DMARC1; p=${dmarcPolicy}; rua=mailto:${dmarcEmail}`,
-        ttl
-      });
-      
-      // Note: DKIM records will be generated later when DKIM keys are available
-      // They require the DKIMCreator which is part of the email server
+      const requiredRecords = buildEmailDnsRecords({
+        domain,
+        hostname: this.options.emailConfig.hostname,
+        mxPriority: domainConfig.dns?.internal?.mxPriority,
+      }).filter((record) => !record.name.includes('._domainkey.'));
+
+      for (const record of requiredRecords) {
+        records.push({
+          name: record.name,
+          type: record.type,
+          value: record.value,
+          ttl,
+        });
+      }
     }
     
     logger.log('info', `Generated ${records.length} email DNS records for ${internalDnsDomains.length} internal-dns domains`);
@@ -1939,54 +1991,30 @@ export class DcRouter {
   }
   
   /**
-   * Load DKIM records from JSON files
-   * Reads all *.dkimrecord.json files from the DNS records directory
+   * Generate DKIM DNS records for internal-dns domains from smartmta's selector-aware DKIM state.
    */
   private async loadDkimRecords(): Promise<Array<{name: string; type: string; value: string; ttl?: number}>> {
     const records: Array<{name: string; type: string; value: string; ttl?: number}> = [];
-    
-    try {
-      // Ensure paths are imported
-      const dnsDir = this.resolvedPaths.dnsRecordsDir;
-      
-      // Check if directory exists
-      if (!plugins.fs.existsSync(dnsDir)) {
-        logger.log('debug', 'No DNS records directory found, skipping DKIM record loading');
-        return records;
+    if (!this.options.emailConfig?.domains || !this.emailServer?.dkimCreator) {
+      return records;
+    }
+
+    for (const domainConfig of this.options.emailConfig.domains) {
+      if (domainConfig.dnsMode !== 'internal-dns') {
+        continue;
       }
-      
-      // Read all files in the directory
-      const files = plugins.fs.readdirSync(dnsDir);
-      const dkimFiles = files.filter(f => f.endsWith('.dkimrecord.json'));
-      
-      logger.log('info', `Found ${dkimFiles.length} DKIM record files`);
-      
-      // Load each DKIM record
-      for (const file of dkimFiles) {
-        try {
-          const filePath = plugins.path.join(dnsDir, file);
-          const fileContent = plugins.fs.readFileSync(filePath, 'utf8');
-          const dkimRecord = JSON.parse(fileContent);
-          
-          // Validate record structure
-          if (dkimRecord.name && dkimRecord.type === 'TXT' && dkimRecord.value) {
-            records.push({
-              name: dkimRecord.name,
-              type: 'TXT',
-              value: dkimRecord.value,
-              ttl: 3600 // Standard DKIM TTL
-            });
-            
-            logger.log('info', `Loaded DKIM record for ${dkimRecord.name}`);
-          } else {
-            logger.log('warn', `Invalid DKIM record structure in ${file}`);
-          }
-        } catch (error: unknown) {
-          logger.log('error', `Failed to load DKIM record from ${file}: ${(error as Error).message}`);
-        }
+      const selector = domainConfig.dkim?.selector || 'default';
+      try {
+        const dkimRecord = await this.emailServer.dkimCreator.getDNSRecordForDomain(domainConfig.domain, selector);
+        records.push({
+          name: dkimRecord.name,
+          type: 'TXT',
+          value: dkimRecord.value,
+          ttl: domainConfig.dns?.internal?.ttl || 3600,
+        });
+      } catch (error: unknown) {
+        logger.log('error', `Failed to generate DKIM record for ${domainConfig.domain}: ${(error as Error).message}`);
       }
-    } catch (error: unknown) {
-      logger.log('error', `Failed to load DKIM records: ${(error as Error).message}`);
     }
 
     return records;
@@ -2013,12 +2041,17 @@ export class DcRouter {
     // Ensure necessary directories exist
     paths.ensureDataDirectories(this.resolvedPaths);
     
-    // Generate DKIM keys for each email domain
+    // Generate DKIM keys for each internal-dns email domain using the configured selector.
     for (const domainConfig of this.options.emailConfig.domains) {
+      if (domainConfig.dnsMode !== 'internal-dns') {
+        continue;
+      }
       try {
-        // Generate DKIM keys for all domains, regardless of DNS mode
-        // This ensures keys are ready even if DNS mode changes later
-        await dkimCreator.handleDKIMKeysForDomain(domainConfig.domain);
+        await dkimCreator.handleDKIMKeysForSelector(
+          domainConfig.domain,
+          domainConfig.dkim?.selector || 'default',
+          domainConfig.dkim?.keySize || 2048,
+        );
         logger.log('info', `DKIM keys initialized for ${domainConfig.domain}`);
       } catch (error: unknown) {
         logger.log('error', `Failed to initialize DKIM for ${domainConfig.domain}: ${(error as Error).message}`);
@@ -2148,6 +2181,25 @@ export class DcRouter {
       }
     }
   }
+
+  private addEmailEventSubscription(
+    emitter: {
+      on(eventName: string, listener: (...args: any[]) => void): void;
+      off(eventName: string, listener: (...args: any[]) => void): void;
+    },
+    eventName: string,
+    listener: (...args: any[]) => void,
+  ): void {
+    emitter.on(eventName, listener);
+    this.emailEventSubscriptions.push({ emitter, eventName, listener });
+  }
+
+  private clearEmailEventSubscriptions(): void {
+    for (const subscription of this.emailEventSubscriptions) {
+      subscription.emitter.off(subscription.eventName, subscription.listener);
+    }
+    this.emailEventSubscriptions = [];
+  }
   
   /**
    * Detect the server's public IP address