@serve.zone/dcrouter 15.11.0 → 16.0.0

package/ts/classes.dcrouter.ts

@@ -14,8 +14,6 @@ import {
   type IExtendedSmtpSession,
   type IMessageAcceptanceContext,
   type IMessageAcceptanceDecision,
-  type IOutboundConnectionProxy,
-  type IOutboundConnectionProxyContext,
 } from '@push.rocks/smartmta';
 import { logger } from './logger.js';
 import { StorageBackedCertManager } from './classes.storage-cert-manager.js';
@@ -29,7 +27,7 @@ import { commitinfo } from './00_commitinfo_data.js';
 import { OpsServer } from './opsserver/index.js';
 import { MetricsManager } from './monitoring/index.js';
 import { RadiusServer, type IRadiusServerConfig } from './radius/index.js';
-import { RemoteIngressHubLifecycle, RemoteIngressManager, TunnelManager } from './remoteingress/index.js';
+import { RemoteIngressHubLifecycle, RemoteIngressManager, TunnelManager, buildEdgeOnlyDerivationRoutes } from './remoteingress/index.js';
 import { VpnAccessResolver, VpnManager, type IVpnManagerConfig } from './vpn/index.js';
 import { RouteConfigManager, ApiTokenManager, GatewayClientManager, ReferenceResolver, DbSeeder, TargetProfileManager, buildHttpRedirectRuntimeRoutes } from './config/index.js';
 import type { TVpnClientAllowEntry } from './config/classes.route-config-manager.js';
@@ -39,9 +37,9 @@ import { DnsManager } from './dns/manager.dns.js';
 import { DnsServerRuntime } from './dns/classes.dns-server-runtime.js';
 import { AcmeConfigManager } from './acme/manager.acme-config.js';
 import { SmartAcmeLifecycle } from './acme/classes.smartacme-lifecycle.js';
-import { AcceptedEmailSpool, EmailDomainManager, EmailRouteBuilder, EmailSettingsManager, SmartMtaStorageManager, WorkAppMailManager, type TSmartMtaQueueItemLike } from './email/index.js';
+import { AcceptedEmailSpool, EmailDomainManager, EmailRouteBuilder, EmailSettingsManager, MailDnsSync, MailEgressCoordinator, SmartMtaStorageManager, WorkAppMailManager, type TSmartMtaQueueItemLike } from './email/index.js';
 import type { IRoute } from '../ts_interfaces/data/route-management.js';
-import type { IEmailPortConfig, IEmailServerSettings, IEmailServerSettingsSeed, TEmailOutboundMode, TEmailServerSettingsUpdate } from '../ts_interfaces/data/email-settings.js';
+import type { IEmailOutboundEgressStatus, IEmailPortConfig, IEmailServerSettings, IEmailServerSettingsSeed, TEmailOutboundMode, TEmailServerSettingsUpdate } from '../ts_interfaces/data/email-settings.js';
 import type { IDcRouterRouteConfig, IRemoteIngressHubSettings, IRemoteIngressPerformanceConfig, TRemoteIngressHubSettingsUpdate } from '../ts_interfaces/data/remoteingress.js';
 import type { ISecurityCompiledPolicy } from '../ts_interfaces/data/security-policy.js';
 
@@ -311,6 +309,8 @@ export class DcRouter {
   public emailDomainManager?: EmailDomainManager;
   public workAppMailManager: WorkAppMailManager;
   public acceptedEmailSpool: AcceptedEmailSpool;
+  public mailEgressCoordinator: MailEgressCoordinator;
+  public mailDnsSync: MailDnsSync;
   public securityPolicyManager?: SecurityPolicyManager;
 
   // Auto-discovered public IP (populated by generateAuthoritativeRecords)
@@ -372,6 +372,8 @@ export class DcRouter {
     );
     this.workAppMailManager = new WorkAppMailManager(this);
     this.acceptedEmailSpool = new AcceptedEmailSpool(this);
+    this.mailEgressCoordinator = new MailEgressCoordinator(this);
+    this.mailDnsSync = new MailDnsSync(this);
     this.smartAcmeLifecycle = new SmartAcmeLifecycle(this);
     this.vpnAccessResolver = new VpnAccessResolver(this);
     this.remoteIngressHubLifecycle = new RemoteIngressHubLifecycle(this);
@@ -521,7 +523,14 @@ export class DcRouter {
           .dependsOn('DcRouterDb')
           .withStart(async () => {
             this.emailSettingsManager = new EmailSettingsManager(this.options);
-            await this.emailSettingsManager.start();
+            try {
+              await this.emailSettingsManager.start();
+            } catch (error: unknown) {
+              // Loud failure: without this manager, outbound mail defers (strict mode
+              // resolution) — this log is the operator's pointer to the root cause.
+              logger.log('error', `EmailSettingsManager failed to start — DB-backed email settings unavailable, outbound mail will defer: ${(error as Error).message}`);
+              throw error;
+            }
           })
           .withStop(async () => {
             if (this.emailSettingsManager) {
@@ -566,6 +575,8 @@ export class DcRouter {
           .dependsOn('DcRouterDb')
           .withStart(async () => {
             this.remoteIngressManager = new RemoteIngressManager();
+            this.remoteIngressManager.setHubPublicIps([this.options.publicIp]);
+            this.remoteIngressManager.setOnEdgesChanged((reason) => this.mailDnsSync.requestSync(reason));
             await this.remoteIngressManager.initialize();
           })
           .withStop(async () => {
@@ -659,7 +670,10 @@ export class DcRouter {
               // then push updated derived ports to the Rust hub binary
               async (routes) => {
                 try {
-                  await this.remoteIngressHubLifecycle.updateRoutes(routes as IDcRouterRouteConfig[]);
+                  await this.remoteIngressHubLifecycle.updateRoutes([
+                    ...(routes as IDcRouterRouteConfig[]),
+                    ...this.getEdgeOnlyDerivationRoutes(),
+                  ]);
                 } catch (err: unknown) {
                   logger.log('error', `Failed to sync Remote Ingress allowed edges: ${(err as Error).message}`);
                 }
@@ -701,6 +715,10 @@ export class DcRouter {
     if (this.options.dbConfig?.enabled !== false) {
       const emailServiceDeps = ['SmartProxy', 'MetricsManager'];
       emailServiceDeps.push('EmailDomainManager');
+      // Explicit dependency: the outbound mode must come from the DB-backed settings;
+      // without it, deliveries defer via the coordinator's strict mode resolution
+      // instead of going direct.
+      emailServiceDeps.push('EmailSettingsManager');
       this.serviceManager.addService(
         new plugins.taskbuffer.Service('EmailServer')
           .optional()
@@ -1599,9 +1617,6 @@ export class DcRouter {
     // Create config with mapped ports
     const baseEmailConfig = this.options.emailConfig;
     const configuredMessageDataHook = baseEmailConfig.hooks?.onMessageData;
-    const outboundMode = this.emailSettingsManager?.getOutboundMode()
-      || this.options.emailOutboundMode
-      || 'direct';
     const mappedEmailPorts = baseEmailConfig.ports.map(port => portMapping[port] || port + 10000);
     const configuredSecurePort = baseEmailConfig.smtp?.securePort;
     const mappedSecurePort = configuredSecurePort === undefined
@@ -1619,9 +1634,7 @@ export class DcRouter {
       },
       outbound: {
         ...baseEmailConfig.outbound,
-        ...(outboundMode === 'remoteIngress'
-          ? { connectionProxyProvider: async (context) => this.createRemoteIngressEmailConnectionProxy(context) }
-          : {}),
+        connectionProxyProvider: (context) => this.mailEgressCoordinator.provideConnectionProxy(context),
       },
       smtp: {
         ...baseEmailConfig.smtp,
@@ -1770,44 +1783,19 @@ export class DcRouter {
     }
 
     logger.log('info', `Email server started on ports: ${emailConfig.ports.join(', ')}`);
+
+    this.mailEgressCoordinator.logEgressPreflight();
+    this.mailDnsSync.requestSync('email server start');
   }
 
-  private async createRemoteIngressEmailConnectionProxy(
-    context: IOutboundConnectionProxyContext,
-  ): Promise<IOutboundConnectionProxy> {
-    const tunnelManager = this.tunnelManager;
-    if (!tunnelManager) {
-      throw new Error('RemoteIngress outbound SMTP egress is enabled, but the hub is not running');
-    }
-
-    const connectionTimeoutMs = this.options.emailConfig?.outbound?.connectionTimeout ?? 30_000;
-    const proxy = await tunnelManager.startEgressTcpProxy({
-      logicalHost: context.host,
-      port: context.port,
-      serverFirst: !context.secure,
-      connectTimeoutMs: connectionTimeoutMs,
-      acceptTimeoutMs: connectionTimeoutMs,
-    });
-    let closed = false;
-    return {
-      connectHost: proxy.listenHost,
-      connectPort: proxy.listenPort,
-      poolKey: `remoteIngress:${proxy.edgeId}:${context.host}:${context.port}`,
-      close: async () => {
-        if (closed) {
-          return;
-        }
-        closed = true;
-        try {
-          await tunnelManager.stopEgressTcpProxy(proxy.proxyId);
-        } catch (err: unknown) {
-          logger.log('warn', `Failed to stop RemoteIngress egress proxy ${proxy.proxyId}: ${(err as Error).message}`);
-        }
-      },
-    };
+  /**
+   * Readiness of the RemoteIngress outbound mail egress path (mail-tagged edges).
+   */
+  public getOutboundEgressStatus(): IEmailOutboundEgressStatus {
+    return this.mailEgressCoordinator.getOutboundEgressStatus();
   }
 
-  
+
   /**
    * Update the unified email configuration
    * @param config New email configuration
@@ -2017,7 +2005,12 @@ export class DcRouter {
 
   /** Bootstrap routes the RemoteIngress hub uses to derive edge ports before the DB route set is applied. */
   public getRemoteIngressBootstrapRoutes(): plugins.smartproxy.IRouteConfig[] {
-    return [...this.seedConfigRoutes, ...this.seedEmailRoutes, ...this.runtimeDnsRoutes];
+    return [...this.seedConfigRoutes, ...this.seedEmailRoutes, ...this.runtimeDnsRoutes, ...this.getEdgeOnlyDerivationRoutes()];
+  }
+
+  /** Edge-derivation-only routes (see buildEdgeOnlyDerivationRoutes) — never passed to SmartProxy. */
+  public getEdgeOnlyDerivationRoutes(): IDcRouterRouteConfig[] {
+    return buildEdgeOnlyDerivationRoutes(this.options);
   }
 
   /**

package/ts/db/documents/classes.cached.email.ts

@@ -8,6 +8,12 @@ const TTL = plugins.smartdata.smartdataTtlValues;
  */
 export type TCachedEmailStatus = 'pending' | 'processing' | 'queued' | 'delivered' | 'failed' | 'deferred';
 
+/**
+ * Direction of an accepted email: 'inbound' = received from a remote peer,
+ * 'outbound' = locally submitted for delivery.
+ */
+export type TCachedEmailDirection = 'inbound' | 'outbound';
+
 /**
  * Helper to get the smartdata database instance
  */
@@ -112,6 +118,24 @@ export class CachedEmail extends plugins.smartdata.SmartdataCachedDocument<Cache
   @plugins.smartdata.svDb()
   public senderDomain!: string;
 
+  /**
+   * Direction of the accepted message (inbound receipt vs local submission)
+   */
+  @plugins.smartdata.svDb()
+  public direction: TCachedEmailDirection = 'outbound';
+
+  /**
+   * Lowercased recipient domains (for querying/filtering)
+   */
+  @plugins.smartdata.svDb()
+  public recipientDomains: string[] = [];
+
+  /**
+   * Epoch ms when the message was accepted (for newest-first listings)
+   */
+  @plugins.smartdata.svDb()
+  public acceptedAt: number = 0;
+
   /**
    * Priority level (higher = more important)
    */
@@ -183,7 +207,11 @@ export class CachedEmail extends plugins.smartdata.SmartdataCachedDocument<Cache
     }, limit);
   }
 
-  private static async findLimited(filter: Record<string, any>, limit: number): Promise<CachedEmail[]> {
+  private static async findLimited(
+    filter: Record<string, any>,
+    limit: number,
+    sort?: Record<string, 1 | -1>,
+  ): Promise<CachedEmail[]> {
     const safeLimit = Number.isFinite(limit) ? Math.max(0, Math.floor(limit)) : 25;
     if (safeLimit === 0) {
       return [];
@@ -195,6 +223,9 @@ export class CachedEmail extends plugins.smartdata.SmartdataCachedDocument<Cache
       filter,
       CachedEmail as unknown as typeof plugins.smartdata.SmartDataDbDoc,
     );
+    if (sort) {
+      cursor.mongodbCursor.sort(sort);
+    }
     cursor.mongodbCursor.limit(safeLimit);
     try {
       return await cursor.toArray();
@@ -203,6 +234,32 @@ export class CachedEmail extends plugins.smartdata.SmartdataCachedDocument<Cache
     }
   }
 
+  /**
+   * Find the most recently accepted emails, newest first.
+   */
+  public static async findRecent(
+    limit = 200,
+    filter: { direction?: TCachedEmailDirection; recipientDomain?: string } = {},
+  ): Promise<CachedEmail[]> {
+    const query: Record<string, any> = {};
+    if (filter.direction) {
+      query.direction = filter.direction;
+    }
+    if (filter.recipientDomain) {
+      query.recipientDomains = filter.recipientDomain.toLowerCase();
+    }
+    return await CachedEmail.findLimited(query, limit, { acceptedAt: -1 });
+  }
+
+  /**
+   * Find emails addressed to a recipient domain
+   */
+  public static async findByRecipientDomain(domain: string): Promise<CachedEmail[]> {
+    return await CachedEmail.getInstances({
+      recipientDomains: { $all: [domain.toLowerCase()] },
+    });
+  }
+
   /**
    * Find emails by sender domain
    */
@@ -254,4 +311,18 @@ export class CachedEmail extends plugins.smartdata.SmartdataCachedDocument<Cache
       }
     }
   }
+
+  /**
+   * Derive recipient domains from the recipient address list
+   */
+  public updateRecipientDomains(): void {
+    const domains = new Set<string>();
+    for (const address of this.to || []) {
+      const match = address.match(/@([^>]+)>?$/);
+      if (match) {
+        domains.add(match[1].toLowerCase());
+      }
+    }
+    this.recipientDomains = [...domains];
+  }
 }

package/ts/db/documents/classes.email-domain.doc.ts

@@ -4,6 +4,7 @@ import type {
   IEmailDomainDkim,
   IEmailDomainRateLimits,
   IEmailDomainDnsStatus,
+  IEmailDomainRemoteIngress,
 } from '../../../ts_interfaces/data/email-domain.js';
 
 const getDb = () => DcRouterDb.getInstance().getDb();
@@ -29,6 +30,9 @@ export class EmailDomainDoc extends plugins.smartdata.SmartDataDbDoc<EmailDomain
   @plugins.smartdata.svDb()
   public rateLimits?: IEmailDomainRateLimits;
 
+  @plugins.smartdata.svDb()
+  public remoteIngress?: IEmailDomainRemoteIngress;
+
   @plugins.smartdata.svDb()
   public dnsStatus!: IEmailDomainDnsStatus;
 

package/ts/db/documents/classes.email-server-settings.doc.ts

@@ -18,7 +18,7 @@ export class EmailServerSettingsDoc extends plugins.smartdata.SmartDataDbDoc<Ema
   public emailConfig?: IUnifiedEmailServerOptions;
 
   @plugins.smartdata.svDb()
-  public outboundMode: TEmailOutboundMode = 'direct';
+  public outboundMode: TEmailOutboundMode = 'remoteIngress';
 
   @plugins.smartdata.svDb()
   public emailPortConfig?: IEmailPortConfig;

package/ts/db/documents/classes.remote-ingress-edge.doc.ts

@@ -37,6 +37,15 @@ export class RemoteIngressEdgeDoc extends plugins.smartdata.SmartDataDbDoc<Remot
   @plugins.smartdata.svDb()
   public tags!: string[];
 
+  @plugins.smartdata.svDb()
+  public publicIp?: string;
+
+  @plugins.smartdata.svDb()
+  public publicIpV6?: string;
+
+  @plugins.smartdata.svDb()
+  public mailHostname?: string;
+
   @plugins.smartdata.svDb()
   public createdAt!: number;
 

package/ts/dns/classes.dns-server-runtime.ts

@@ -446,6 +446,7 @@ export class DnsServerRuntime {
       // Use explicitly configured public IP
       publicIp = options.publicIp;
       this.dcRouterRef.detectedPublicIp = publicIp;
+      this.dcRouterRef.remoteIngressManager?.setHubPublicIps([options.publicIp, publicIp]);
       logger.log('info', `Using configured public IP for nameserver A records: ${publicIp}`);
     } else {
       // Auto-discover public IP using smartnetwork
@@ -457,6 +458,7 @@ export class DnsServerRuntime {
         if (publicIps.v4) {
           publicIp = publicIps.v4;
           this.dcRouterRef.detectedPublicIp = publicIp;
+          this.dcRouterRef.remoteIngressManager?.setHubPublicIps([options.publicIp, publicIp]);
           logger.log('info', `Auto-discovered public IPv4: ${publicIp}`);
         } else {
           logger.log('warn', 'Could not auto-discover public IPv4 address');

package/ts/email/classes.accepted-email-spool.ts

@@ -1,6 +1,7 @@
 import * as plugins from '../plugins.js';
 import { logger } from '../logger.js';
 import { CachedEmail } from '../db/index.js';
+import type { TCachedEmailDirection } from '../db/documents/classes.cached.email.js';
 import type {
   Email,
   IExtendedSmtpSession,
@@ -70,6 +71,8 @@ export interface IAcceptRawEmailOptions {
   messageId?: string;
   subject?: string;
   processAfterAccept?: boolean;
+  /** Explicit direction override for programmatic submitters. Defaults to the session-authenticated rule. */
+  direction?: TCachedEmailDirection;
 }
 
 /**
@@ -118,6 +121,8 @@ export class AcceptedEmailSpool {
     cachedEmail.rawContent = this.setDcRouterCacheIdHeader(rawMessage.toString('utf8'), cachedEmail.id);
     cachedEmail.status = 'pending';
     cachedEmail.nextAttempt = new Date();
+    cachedEmail.direction = session.authenticated ? 'outbound' : 'inbound';
+    cachedEmail.acceptedAt = Date.now();
     cachedEmail.routeData = JSON.stringify({
       acceptedAt: new Date().toISOString(),
       session: {
@@ -131,6 +136,7 @@ export class AcceptedEmailSpool {
       },
     });
     cachedEmail.updateSenderDomain();
+    cachedEmail.updateRecipientDomains();
     await cachedEmail.save();
     if (context.abortSignal?.aborted) {
       cachedEmail.markFailed('Message acceptance aborted before SMTP success');
@@ -171,6 +177,9 @@ export class AcceptedEmailSpool {
     cachedEmail.rawContent = this.setDcRouterCacheIdHeader(rawMessage, cachedEmail.id);
     cachedEmail.status = 'pending';
     cachedEmail.nextAttempt = new Date();
+    cachedEmail.direction = optionsArg.direction
+      ?? (optionsArg.session.authenticated ? 'outbound' : 'inbound');
+    cachedEmail.acceptedAt = Date.now();
     cachedEmail.routeData = JSON.stringify({
       acceptedAt: new Date().toISOString(),
       session: {
@@ -184,6 +193,7 @@ export class AcceptedEmailSpool {
       },
     });
     cachedEmail.updateSenderDomain();
+    cachedEmail.updateRecipientDomains();
     await cachedEmail.save();
 
     if (optionsArg.processAfterAccept !== false) {
@@ -506,7 +516,8 @@ export class AcceptedEmailSpool {
     return cachedEmail.status === 'delivered' || cachedEmail.status === 'failed';
   }
 
-  private getCachedEmailIdFromQueueItem(item: TSmartMtaQueueItemLike): string | undefined {
+  /** The CachedEmail row a live queue item was spooled from (via the cache-id header). */
+  public getCachedEmailIdFromQueueItem(item: TSmartMtaQueueItemLike): string | undefined {
     return this.getHeaderValue(item.processingResult?.headers, DCROUTER_CACHE_ID_HEADER)
       || this.getHeaderValue(item.processingResult?.email?.headers, DCROUTER_CACHE_ID_HEADER);
   }