@push.rocks/smartmta 9.3.0 → 9.4.1

package/ts/mail/delivery/classes.delivery.system.ts

@@ -1,6 +1,7 @@
 import * as plugins from '../../plugins.js';
 import { EventEmitter } from 'node:events';
 import { logger } from '../../logger.js';
+import { getErrorMessage } from '../../functions.errors.js';
 import {
   SecurityLogger,
   SecurityLogLevel,
@@ -16,6 +17,7 @@ import { Email } from '../core/classes.email.js';
 import type { UnifiedEmailServer } from '../routing/classes.unified.email.server.js';
 import { SmtpDeliveryError } from '../../security/classes.rustsecuritybridge.js';
 import type {
+  IOutboundAuthenticationCheck,
   ISmtpRecipientResult,
   ISmtpSendResult,
 } from '../../security/classes.rustsecuritybridge.js';
@@ -146,6 +148,15 @@ export class MultiModeDeliverySystem extends EventEmitter {
   private emailServer?: UnifiedEmailServer;
   private bounceDrainTimer?: NodeJS.Timeout;
   private bounceDrainPromise?: Promise<void>;
+  private readonly queueItemsReadyHandler = (items: IQueueItem[]): Promise<void> =>
+    this.processItems(items);
+  private queueListenerRegistered = false;
+  private lifecycleGeneration = 0;
+  private pendingSmtpAuthenticationUpdates = new Map<string, {
+    itemId: string;
+    transactionId: string;
+    transaction: ISmtpTransactionAttempt;
+  }>();
   
   /**
    * Create a new multi-mode delivery system
@@ -218,8 +229,18 @@ export class MultiModeDeliverySystem extends EventEmitter {
       }
     };
     
-    // Set up event listeners
-    this.queue.on('itemsReady', this.processItems.bind(this));
+  }
+
+  private registerQueueListener(): void {
+    if (this.queueListenerRegistered) return;
+    this.queue.on('itemsReady', this.queueItemsReadyHandler);
+    this.queueListenerRegistered = true;
+  }
+
+  private unregisterQueueListener(): void {
+    if (!this.queueListenerRegistered) return;
+    this.queue.off('itemsReady', this.queueItemsReadyHandler);
+    this.queueListenerRegistered = false;
   }
 
   private createEmailForRecipients(email: Email, recipients: string[]): Email {
@@ -250,9 +271,43 @@ export class MultiModeDeliverySystem extends EventEmitter {
       recipients,
       recipientResults: error.recipientResults,
       ...(recipientDomain ? { recipientDomain } : {}),
+      ...(error.outboundAuthenticationHandle ? { outboundAuthenticationHandle: error.outboundAuthenticationHandle } : {}),
     }, error);
   }
 
+  private smtpAuthenticationUpdateKey(itemIdArg: string, transactionIdArg: string): string {
+    return `${itemIdArg}\u0000${transactionIdArg}`;
+  }
+
+  private async persistSmtpAuthenticationUpdate(updateArg: {
+    itemId: string;
+    transactionId: string;
+    transaction: ISmtpTransactionAttempt;
+  }): Promise<void> {
+    const updated = await this.queue.updateSmtpTransaction(
+      updateArg.itemId,
+      updateArg.transactionId,
+      updateArg.transaction,
+    );
+    if (updated) await emitSafely(this, 'smtpTransactionUpdated', updateArg.transaction);
+  }
+
+  private async flushPendingSmtpAuthenticationUpdates(itemIdArg: string): Promise<void> {
+    const updates = [...this.pendingSmtpAuthenticationUpdates.entries()]
+      .filter(([, update]) => update.itemId === itemIdArg);
+    for (const [key, update] of updates) {
+      this.pendingSmtpAuthenticationUpdates.delete(key);
+      try {
+        await this.persistSmtpAuthenticationUpdate(update);
+      } catch (error) {
+        logger.log(
+          'warn',
+          `Failed to persist outbound authentication self-check: ${getErrorMessage(error)}`,
+        );
+      }
+    }
+  }
+
   private async persistSmtpAttempt(optionsArg: {
     item: IQueueItem;
     targetHost: string;
@@ -265,6 +320,7 @@ export class MultiModeDeliverySystem extends EventEmitter {
   }): Promise<ISmtpTransactionAttempt> {
     const completedAt = new Date();
     const error = optionsArg.error;
+    const authenticationHandle = error?.outboundAuthenticationHandle || optionsArg.result?.outboundAuthenticationHandle;
     const attempt: ISmtpTransactionAttempt = {
       id: plugins.uuid.v4(),
       queueItemId: optionsArg.item.id,
@@ -285,9 +341,48 @@ export class MultiModeDeliverySystem extends EventEmitter {
       recipientResults: error?.recipientResults || optionsArg.result?.recipientResults,
       ...(error ? { failurePhase: error.phase, error: error.message } : {}),
       transcript: error?.transactionLog || optionsArg.result?.transactionLog || [],
+      ...(authenticationHandle ? { outboundAuthentication: authenticationHandle.pending } : {}),
     };
     await this.queue.appendSmtpTransaction(optionsArg.item.id, attempt);
     await emitSafely(this, 'smtpTransactionCompleted', attempt);
+    if (authenticationHandle) {
+      const persistAuthentication = async (authenticationArg: IOutboundAuthenticationCheck): Promise<void> => {
+        const updatedAttempt: ISmtpTransactionAttempt = {
+          ...attempt,
+          outboundAuthentication: authenticationArg,
+        };
+        const update = {
+          itemId: optionsArg.item.id,
+          transactionId: attempt.id,
+          transaction: updatedAttempt,
+        };
+        // Delivery status and recipient checkpoints use whole-item durable
+        // mutations. Defer this diagnostic replacement until that lifecycle
+        // has settled so it cannot race and restore an earlier queue status.
+        if (this.activeDeliveries.has(optionsArg.item.id)) {
+          this.pendingSmtpAuthenticationUpdates.set(
+            this.smtpAuthenticationUpdateKey(optionsArg.item.id, attempt.id),
+            update,
+          );
+          return;
+        }
+        await this.persistSmtpAuthenticationUpdate(update);
+      };
+      void authenticationHandle.result.then(
+        persistAuthentication,
+        (failureArg) => persistAuthentication({
+          state: 'indeterminate',
+          overall: 'indeterminate',
+          checkedAt: new Date().toISOString(),
+          error: String(failureArg instanceof Error ? failureArg.message : failureArg).slice(0, 512),
+        }),
+      ).catch((error) => {
+        logger.log(
+          'warn',
+          `Failed to persist outbound authentication self-check: ${getErrorMessage(error)}`,
+        );
+      });
+    }
     return attempt;
   }
 
@@ -429,14 +524,33 @@ export class MultiModeDeliverySystem extends EventEmitter {
       return;
     }
     
+    const lifecycleGeneration = ++this.lifecycleGeneration;
     this.running = true;
-    await this.drainBounceOutbox();
-    this.bounceDrainTimer = setInterval(() => {
-      void this.drainBounceOutbox();
-    }, 30_000);
+    this.registerQueueListener();
+    try {
+      await this.drainBounceOutbox();
+      if (!this.running || lifecycleGeneration !== this.lifecycleGeneration) return;
+
+      this.bounceDrainTimer = setInterval(() => {
+        void this.drainBounceOutbox();
+      }, 30_000);
+
+      await emitSafely(this, 'started');
+      if (!this.running || lifecycleGeneration !== this.lifecycleGeneration) return;
 
-    await emitSafely(this, 'started');
-    logger.log('info', 'MultiModeDeliverySystem started successfully');
+      this.queue.wake();
+      logger.log('info', 'MultiModeDeliverySystem started successfully');
+    } catch (error) {
+      if (lifecycleGeneration === this.lifecycleGeneration) {
+        this.running = false;
+        this.unregisterQueueListener();
+        if (this.bounceDrainTimer) {
+          clearInterval(this.bounceDrainTimer);
+          this.bounceDrainTimer = undefined;
+        }
+      }
+      throw error;
+    }
   }
   
   /**
@@ -446,11 +560,15 @@ export class MultiModeDeliverySystem extends EventEmitter {
     logger.log('info', 'Stopping MultiModeDeliverySystem');
     
     if (!this.running) {
+      this.unregisterQueueListener();
       logger.log('warn', 'MultiModeDeliverySystem is already stopped');
       return;
     }
+
+    const lifecycleGeneration = ++this.lifecycleGeneration;
     
     this.running = false;
+    this.unregisterQueueListener();
     if (this.bounceDrainTimer) {
       clearInterval(this.bounceDrainTimer);
       this.bounceDrainTimer = undefined;
@@ -461,6 +579,10 @@ export class MultiModeDeliverySystem extends EventEmitter {
       clearTimeout(this.throttleResetTimer);
       this.throttleResetTimer = null;
     }
+    this.throttled = false;
+    this.rateLimitCounter = 0;
+    this.rateLimitLastCheck = Date.now();
+    this.stats.rateLimiting.currentRate = 0;
 
     // Wait for active deliveries to complete
     if (this.activeDeliveries.size > 0) {
@@ -485,6 +607,8 @@ export class MultiModeDeliverySystem extends EventEmitter {
     }
     
     await this.bounceDrainPromise;
+    if (lifecycleGeneration !== this.lifecycleGeneration) return;
+
     await emitSafely(this, 'stopped');
     logger.log('info', 'MultiModeDeliverySystem stopped successfully');
   }
@@ -504,17 +628,15 @@ export class MultiModeDeliverySystem extends EventEmitter {
       return;
     }
     
-    // Check rate limiting
-    if (this.checkRateLimit()) {
-      logger.log('debug', 'Rate limit exceeded, throttling deliveries');
-      return;
-    }
-    
     let startedDeliveries = 0;
 
     for (const item of items) {
       if (this.activeDeliveries.size >= this.options.concurrentDeliveries) break;
       if (this.activeDeliveries.has(item.id)) continue;
+      if (this.checkRateLimit()) {
+        logger.log('debug', 'Rate limit exceeded, throttling deliveries');
+        break;
+      }
 
       // Reserve synchronously before the first await so overlapping queue events
       // cannot dispatch the same item while its durable claim is in flight.
@@ -733,9 +855,11 @@ export class MultiModeDeliverySystem extends EventEmitter {
       // Remove from active deliveries
       this.activeDeliveries.delete(item.id);
       this.stats.activeDeliveries = this.activeDeliveries.size;
+      await this.flushPendingSmtpAuthenticationUpdates(item.id);
       
       // Update statistics
       await emitSafely(this, 'statsUpdated', this.stats);
+      this.queue.wake();
     }
   }
   
@@ -1321,7 +1445,6 @@ export class MultiModeDeliverySystem extends EventEmitter {
       this.rateLimitCounter = 0;
       this.throttled = false;
       this.stats.rateLimiting.currentRate = 0;
-      return false;
     }
     
     // Check if we're already throttled
@@ -1329,15 +1452,9 @@ export class MultiModeDeliverySystem extends EventEmitter {
       return true;
     }
     
-    // Increment counter
-    this.rateLimitCounter++;
-    
-    // Calculate current rate (emails per minute)
-    const rate = (this.rateLimitCounter / elapsed) * 60000;
-    this.stats.rateLimiting.currentRate = rate;
-    
-    // Check if rate limit is exceeded
-    if (rate > this.options.globalRateLimit) {
+    // Use a fixed one-minute window. Extrapolating the first delivery from a
+    // few elapsed milliseconds falsely throttles every freshly started queue.
+    if (this.rateLimitCounter >= this.options.globalRateLimit) {
       this.throttled = true;
       this.stats.rateLimiting.throttled++;
       
@@ -1352,10 +1469,16 @@ export class MultiModeDeliverySystem extends EventEmitter {
         this.rateLimitLastCheck = Date.now();
         this.rateLimitCounter = 0;
         this.stats.rateLimiting.currentRate = 0;
+        if (this.running) {
+          this.queue.wake();
+        }
       }, resetDelay);
       
       return true;
     }
+
+    this.rateLimitCounter++;
+    this.stats.rateLimiting.currentRate = this.rateLimitCounter;
     
     return false;
   }

package/ts/mail/delivery/interfaces.ts

@@ -3,6 +3,7 @@
  */
 
 import type {
+  IOutboundAuthenticationCheck,
   ISmtpRecipientResult,
   ISmtpTranscriptEntry,
   TSmtpDeliveryPhase,
@@ -27,6 +28,7 @@ export interface ISmtpTransactionAttempt {
   failurePhase?: TSmtpDeliveryPhase;
   error?: string;
   transcript: ISmtpTranscriptEntry[];
+  outboundAuthentication?: IOutboundAuthenticationCheck;
 }
 
 /**

package/ts/mail/routing/classes.unified.email.server.ts

@@ -9,8 +9,10 @@ import {
 } from '../../security/index.js';
 import { DKIMCreator } from '../security/classes.dkimcreator.js';
 import type { IStorageManager } from '../interfaces.storage.js';
-import { RustSecurityBridge } from '../../security/classes.rustsecuritybridge.js';
+import { RustSecurityBridge, SmtpDeliveryError } from '../../security/classes.rustsecuritybridge.js';
 import type {
+  IOutboundAuthenticationCheck,
+  IOutboundAuthenticationHandle,
   IAuthRequestEvent,
   IEmailReceivedEvent,
   IEmailSecurityResult,
@@ -258,6 +260,12 @@ export interface IOutboundConnectionProxyContext {
   senderDomain?: string;
 }
 
+export interface IOutboundConnectionSourceEvidence {
+  sourceIp: string;
+  addressFamily: 'ipv4' | 'ipv6';
+}
+
+
 export interface IOutboundConnectionProxy {
   connectHost: string;
   connectPort: number;
@@ -265,6 +273,8 @@ export interface IOutboundConnectionProxy {
   /** Per-connection EHLO identity override (e.g. the selected egress edge's PTR hostname). */
   ehloHostname?: string;
   close?: () => Promise<void> | void;
+  /** Actual source identity observed after the proxied connection is established. */
+  sourceEvidence?: Promise<IOutboundConnectionSourceEvidence | null>;
 }
 
 export interface IOutboundTopologyIdentity {
@@ -937,8 +947,28 @@ export class UnifiedEmailServer extends EventEmitter {
       senderDomain,
     });
 
+    const captureId = proxy?.sourceEvidence ? plugins.uuid.v4() : undefined;
+    const attachAuthenticationHandle = <T extends ISmtpSendResult | SmtpDeliveryError>(valueArg: T): T => {
+      if (!captureId || !proxy?.sourceEvidence) return valueArg;
+      const handle = this.createOutboundAuthenticationHandle({
+        captureId,
+        sourceEvidence: proxy.sourceEvidence,
+        heloDomain: proxy.ehloHostname || this.getOutboundHostname(),
+        receiverHostname: host,
+        envelopeFrom: email.from,
+        dkimDomain: dkim?.domain,
+        dkimSelector: dkim?.selector,
+      });
+      Object.defineProperty(valueArg, 'outboundAuthenticationHandle', {
+        value: handle,
+        enumerable: false,
+        configurable: true,
+      });
+      return valueArg;
+    };
+
     try {
-      return await this.rustBridge.sendOutboundEmail({
+      const result = await this.rustBridge.sendOutboundEmail({
         host,
         port,
         ...(proxy ? { connectHost: proxy.connectHost, connectPort: proxy.connectPort } : {}),
@@ -953,12 +983,99 @@ export class UnifiedEmailServer extends EventEmitter {
         maxPoolConnections: this.options.outbound?.maxConnections || 10,
         tlsOpportunistic,
         ...(proxy ? { disablePooling: true } : {}),
+        ...(captureId ? { outboundAuthenticationCaptureId: captureId } : {}),
       });
+      return attachAuthenticationHandle(result);
+    } catch (error) {
+      throw attachAuthenticationHandle(SmtpDeliveryError.from(error));
     } finally {
       await proxy?.close?.();
     }
   }
 
+  private createOutboundAuthenticationHandle(
+    optionsArg: {
+      captureId: string;
+      sourceEvidence: Promise<IOutboundConnectionSourceEvidence | null>;
+      heloDomain: string;
+      receiverHostname: string;
+      envelopeFrom: string;
+      dkimDomain?: string;
+      dkimSelector?: string;
+    },
+  ): IOutboundAuthenticationHandle {
+    const pending: IOutboundAuthenticationCheck = {
+      state: 'pending',
+      overall: 'indeterminate',
+      checkedAt: new Date().toISOString(),
+    };
+    const indeterminate = (errorArg: string): IOutboundAuthenticationCheck => ({
+      state: 'indeterminate',
+      overall: 'indeterminate',
+      checkedAt: new Date().toISOString(),
+      error: errorArg,
+    });
+    const result = (async (): Promise<IOutboundAuthenticationCheck> => {
+      let timeout: ReturnType<typeof setTimeout> | undefined;
+      try {
+        const sourceEvidence = await new Promise<IOutboundConnectionSourceEvidence | null>((resolve, reject) => {
+          timeout = setTimeout(() => {
+            reject(new Error('Timed out waiting for the actual outbound source identity'));
+          }, 40_000);
+          timeout.unref?.();
+          optionsArg.sourceEvidence.then(resolve, reject);
+        });
+        if (!sourceEvidence) {
+          await this.rustBridge.discardCapturedOutboundEmail(optionsArg.captureId).catch(() => undefined);
+          return indeterminate('The outbound proxy did not provide source identity evidence');
+        }
+        const verification = await this.rustBridge.verifyCapturedOutboundEmail({
+          captureId: optionsArg.captureId,
+          ip: sourceEvidence.sourceIp,
+          heloDomain: optionsArg.heloDomain,
+          hostname: optionsArg.receiverHostname,
+          mailFrom: optionsArg.envelopeFrom,
+        });
+        if (verification.state !== 'completed' || !verification.result) {
+          return indeterminate(verification.error || 'The outbound authentication self-check was not completed');
+        }
+        const security = verification.result;
+        const dkimPass = !optionsArg.dkimDomain || security.dkim.some((resultArg) => (
+          resultArg.is_valid
+          && resultArg.domain?.toLowerCase() === optionsArg.dkimDomain?.toLowerCase()
+          && (!optionsArg.dkimSelector || resultArg.selector === optionsArg.dkimSelector)
+        ));
+        const spfPass = security.spf?.result.toLowerCase() === 'pass';
+        const dmarcPass = security.dmarc?.passed === true;
+        return {
+          state: 'completed',
+          overall: dkimPass && spfPass && dmarcPass ? 'pass' : 'fail',
+          checkedAt: new Date().toISOString(),
+          signedMessageSha256: verification.signedMessageSha256,
+          identity: {
+            sourceIp: sourceEvidence.sourceIp,
+            addressFamily: sourceEvidence.addressFamily,
+            heloDomain: optionsArg.heloDomain,
+            receiverHostname: optionsArg.receiverHostname,
+            envelopeFrom: optionsArg.envelopeFrom,
+            ...(optionsArg.dkimDomain ? { dkimDomain: optionsArg.dkimDomain } : {}),
+            ...(optionsArg.dkimSelector ? { dkimSelector: optionsArg.dkimSelector } : {}),
+          },
+          dkim: security.dkim,
+          spf: security.spf,
+          dmarc: security.dmarc,
+        };
+      } catch (error) {
+        await this.rustBridge.discardCapturedOutboundEmail(optionsArg.captureId).catch(() => undefined);
+        return indeterminate(getErrorMessage(error).slice(0, 512));
+      } finally {
+        if (timeout) clearTimeout(timeout);
+      }
+    })();
+    return { pending, result };
+  }
+
+
   /**
    * Start the unified email server
    */

package/ts/security/classes.rustsecuritybridge.ts

@@ -242,6 +242,7 @@ export interface ISmtpDeliveryErrorDetails {
   recipients?: string[];
   recipientDomain?: string;
   recipientResults?: ISmtpRecipientResult[];
+  outboundAuthenticationHandle?: IOutboundAuthenticationHandle;
 }
 
 interface ISmtpRecipientResult {
@@ -259,6 +260,7 @@ export class SmtpDeliveryError extends Error implements ISmtpDeliveryErrorDetail
   public readonly recipients: string[];
   public readonly recipientDomain?: string;
   public readonly recipientResults: ISmtpRecipientResult[];
+  public readonly outboundAuthenticationHandle?: IOutboundAuthenticationHandle;
 
   constructor(details: ISmtpDeliveryErrorDetails, cause?: unknown) {
     const transactionLog = normalizeSmtpTranscript(details.transactionLog);
@@ -277,6 +279,7 @@ export class SmtpDeliveryError extends Error implements ISmtpDeliveryErrorDetail
     this.recipients = details.recipients || [];
     this.recipientDomain = details.recipientDomain;
     this.recipientResults = (details.recipientResults || []).map((result) => ({ ...result }));
+    this.outboundAuthenticationHandle = details.outboundAuthenticationHandle;
     if (cause !== undefined && !authFailure) {
       (this as Error & { cause?: unknown }).cause = cause;
     }
@@ -370,6 +373,31 @@ interface IOutboundEmail {
   }>;
 }
 
+export interface IOutboundAuthenticationCheck {
+  state: 'pending' | 'completed' | 'indeterminate' | 'not-evaluated';
+  overall: 'pass' | 'fail' | 'indeterminate';
+  checkedAt: string;
+  signedMessageSha256?: string;
+  error?: string;
+  identity?: {
+    sourceIp: string;
+    addressFamily: 'ipv4' | 'ipv6';
+    heloDomain: string;
+    receiverHostname: string;
+    envelopeFrom: string;
+    dkimDomain?: string;
+    dkimSelector?: string;
+  };
+  dkim?: IDkimVerificationResult[];
+  spf?: ISpfResult | null;
+  dmarc?: IDmarcResult | null;
+}
+
+export interface IOutboundAuthenticationHandle {
+  pending: IOutboundAuthenticationCheck;
+  result: Promise<IOutboundAuthenticationCheck>;
+}
+
 interface ISmtpSendResult {
   accepted: string[];
   rejected: string[];
@@ -378,6 +406,7 @@ interface ISmtpSendResult {
   response: string;
   envelope: { from: string; to: string[] };
   transactionLog: ISmtpTranscriptEntry[];
+  outboundAuthenticationHandle?: IOutboundAuthenticationHandle;
 }
 
 interface ISmtpSendOptions {
@@ -396,6 +425,14 @@ interface ISmtpSendOptions {
   maxPoolConnections?: number;
   tlsOpportunistic?: boolean;
   disablePooling?: boolean;
+  outboundAuthenticationCaptureId?: string;
+}
+
+interface IVerifyCapturedOutboundEmailResult {
+  state: 'completed' | 'indeterminate';
+  signedMessageSha256?: string;
+  error?: string;
+  result?: IEmailSecurityResult;
 }
 
 interface ISmtpSendRawOptions {
@@ -634,6 +671,20 @@ type TMailerCommands = {
     params: ISmtpSendRawOptions;
     result: ISmtpSendResult;
   };
+  verifyCapturedOutboundEmail: {
+    params: {
+      captureId: string;
+      ip: string;
+      heloDomain: string;
+      hostname: string;
+      mailFrom: string;
+    };
+    result: IVerifyCapturedOutboundEmailResult;
+  };
+  discardCapturedOutboundEmail: {
+    params: { captureId: string };
+    result: { discarded: boolean };
+  };
   verifySmtpConnection: {
     params: ISmtpVerifyOptions;
     result: ISmtpVerifyResult;
@@ -1189,6 +1240,7 @@ export class RustSecurityBridge extends EventEmitter {
     this.ensureRunning();
     try {
       const result = await this.bridge.sendCommand('sendRawEmail', opts);
+
       return {
         ...result,
         transactionLog: normalizeSmtpTranscript(result.transactionLog),
@@ -1198,6 +1250,22 @@ export class RustSecurityBridge extends EventEmitter {
     }
   }
 
+  public async verifyCapturedOutboundEmail(opts: {
+    captureId: string;
+    ip: string;
+    heloDomain: string;
+    hostname: string;
+    mailFrom: string;
+  }): Promise<IVerifyCapturedOutboundEmailResult> {
+    this.ensureRunning();
+    return this.bridge.sendCommand('verifyCapturedOutboundEmail', opts);
+  }
+
+  public async discardCapturedOutboundEmail(captureId: string): Promise<void> {
+    this.ensureRunning();
+    await this.bridge.sendCommand('discardCapturedOutboundEmail', { captureId });
+  }
+
   /** Verify connectivity to an SMTP server. */
   public async verifySmtpConnection(opts: ISmtpVerifyOptions): Promise<ISmtpVerifyResult> {
     this.ensureRunning();