@push.rocks/smartmta 9.4.0 → 9.5.0

package/ts/mail/index.ts

@@ -3,6 +3,7 @@ export * from './interfaces.storage.js';
 export * from './routing/index.js';
 export * from './security/index.js';
 export type { ISmtpTransactionAttempt } from './delivery/interfaces.js';
+export type { IRawSmtpEnvelope } from './delivery/functions.raw-email.js';
 
 // Make the core and delivery modules accessible
 import * as Core from './core/index.js';

package/ts/mail/routing/classes.unified.email.server.ts

@@ -38,6 +38,7 @@ import type { ISmtpSendResult, IOutboundEmail } from '../../security/classes.rus
 import { MultiModeDeliverySystem, type IDeliveryStats, type IMultiModeDeliveryOptions } from '../delivery/classes.delivery.system.js';
 import {
   ACCEPTED_ENVELOPE_IDEMPOTENCY_RETENTION_MS,
+  DEFAULT_MAX_RAW_MESSAGE_BYTES,
   UnifiedDeliveryQueue,
   type IQueueItem,
   type IQueueOptions,
@@ -49,6 +50,20 @@ import type { EmailProcessingMode, ISmtpSession as IBaseSmtpSession } from '../d
 import { EmailActionExecutor } from './classes.email.action.executor.js';
 import { DkimManager } from './classes.dkim.manager.js';
 import { callObserverSafely, emitSafely } from '../delivery/functions.safe-observers.js';
+import {
+  assertCanonicalRawRfc822Message,
+  normalizeRawSmtpEnvelope,
+  type IRawSmtpEnvelope,
+} from '../delivery/functions.raw-email.js';
+import {
+  kQueueCloseAdmission,
+  kQueueEnqueueRaw,
+  kQueueReserveRawAccess,
+} from '../delivery/symbols.delivery.queue.js';
+import {
+  kSendOutboundEmailInternal,
+  kSendRawOutboundEmailInternal,
+} from './symbols.unified.email.server.js';
 
 
 /** External DcRouter interface shape used by UnifiedEmailServer */
@@ -282,6 +297,30 @@ export interface IOutboundTopologyIdentity {
   ipAddresses?: string[];
 }
 
+interface IOutboundTransportParameters {
+  host: string;
+  port: number;
+  connectHost?: string;
+  connectPort?: number;
+  secure: boolean;
+  domain: string;
+  auth?: { user: string; pass: string };
+  connectionTimeoutSecs: number;
+  socketTimeoutSecs: number;
+  poolKey: string;
+  maxPoolConnections: number;
+  tlsOpportunistic: boolean;
+  disablePooling?: boolean;
+  outboundAuthenticationCaptureId?: string;
+}
+
+interface IOutboundDkimMaterial {
+  domain: string;
+  selector: string;
+  privateKey: string;
+  keyType: 'rsa' | 'ed25519';
+}
+
 export interface ISelfMxIdentityMatch {
   isSelf: boolean;
   source: 'explicit' | 'dns-derived' | 'dynamic' | null;
@@ -499,6 +538,10 @@ export class UnifiedEmailServer extends EventEmitter {
   private smtpListenerActive = false;
   private smtpListenerShutdown = false;
   private smtpListenerRestartPromise?: Promise<void>;
+  private acceptingOutboundWork = true;
+  private readonly outboundWorkFlights = new Set<Promise<unknown>>();
+  private readonly outboundAuthenticationFlights = new Set<Promise<IOutboundAuthenticationCheck>>();
+  private stopFlight?: Promise<void>;
   private static readonly VERIFIER_DERIVATION_CONCURRENCY = 4;
   private static readonly VERIFIER_DERIVATION_MAX_QUEUE = 32;
   private verifierDerivationActive = 0;
@@ -634,6 +677,23 @@ export class UnifiedEmailServer extends EventEmitter {
     // We'll create the SMTP servers during the start() method
   }
 
+  private assertAcceptingOutboundWork(): void {
+    if (!this.acceptingOutboundWork) {
+      throw new Error('UnifiedEmailServer is not accepting outbound work');
+    }
+  }
+
+  private trackOutboundWork<T>(operationArg: () => Promise<T>): Promise<T> {
+    let flight: Promise<T>;
+    flight = Promise.resolve()
+      .then(operationArg)
+      .finally(() => {
+        this.outboundWorkFlights.delete(flight);
+      });
+    this.outboundWorkFlights.add(flight);
+    return flight;
+  }
+
   public onRecipientRejected(
     listener: (event: IRecipientRejectedEvent) => Promise<void> | void,
   ): this {
@@ -890,32 +950,30 @@ export class UnifiedEmailServer extends EventEmitter {
    * Send an outbound email via the Rust SMTP client.
    * Uses connection pooling in the Rust binary for efficiency.
    */
-  public async sendOutboundEmail(host: string, port: number, email: Email, options?: {
+  public sendOutboundEmail(host: string, port: number, email: Email, options?: {
     auth?: { user: string; pass: string };
     dkimDomain?: string;
     dkimSelector?: string;
     tlsOpportunistic?: boolean;
   }): Promise<ISmtpSendResult> {
-    // Build DKIM config if domain has keys
-    let dkim: {
-      domain: string;
-      selector: string;
-      privateKey: string;
-      keyType: 'rsa' | 'ed25519';
-    } | undefined;
-    if (options?.dkimDomain) {
-      const selector = options.dkimSelector || 'default';
-      const material = await this.dkimCreator.readValidatedDKIMKeysForSelector(
-        options.dkimDomain,
-        selector,
-      );
-      dkim = {
-        domain: options.dkimDomain,
-        selector,
-        privateKey: material.privateKey,
-        keyType: material.keyType,
-      };
-    }
+    this.assertAcceptingOutboundWork();
+    return this.trackOutboundWork(() => this[kSendOutboundEmailInternal](
+      host,
+      port,
+      email,
+      options,
+    ));
+  }
+
+  public async [kSendOutboundEmailInternal](host: string, port: number, email: Email, options?: {
+    auth?: { user: string; pass: string };
+    dkimDomain?: string;
+    dkimSelector?: string;
+    tlsOpportunistic?: boolean;
+  }): Promise<ISmtpSendResult> {
+    const dkim = options?.dkimDomain
+      ? await this.resolveOutboundDkim(options.dkimDomain, options.dkimSelector || 'default')
+      : undefined;
 
     // Serialize the Email to the outbound format
     const outboundEmail: IOutboundEmail = {
@@ -935,18 +993,170 @@ export class UnifiedEmailServer extends EventEmitter {
       })),
     };
 
-    const secure = port === 465;
-    const tlsOpportunistic = options?.tlsOpportunistic ?? (port === 25);
     const senderDomain = email.getFromDomain()?.toLowerCase() || undefined;
-    const proxy = await this.options.outbound?.connectionProxyProvider?.({
+    return this.sendWithOutboundTransport({
       host,
       port,
+      envelopeFrom: email.from,
+      senderDomain,
+      auth: options?.auth,
+      tlsOpportunistic: options?.tlsOpportunistic,
+      dkim,
+      send: (transport) => this.rustBridge.sendOutboundEmail({
+        ...transport,
+        email: outboundEmail,
+        dkim,
+      }),
+    });
+  }
+
+  /** Send exact RFC822 bytes through the same outbound transport path as structured mail. */
+  public sendRawOutboundEmail(
+    host: string,
+    port: number,
+    rawMessage: Buffer,
+    envelopeArg: IRawSmtpEnvelope,
+    options: {
+      auth?: { user: string; pass: string };
+      dkimDomain: string;
+      dkimSelector: string;
+      tlsOpportunistic?: boolean;
+    },
+  ): Promise<ISmtpSendResult> {
+    this.assertAcceptingOutboundWork();
+    assertCanonicalRawRfc822Message(rawMessage);
+    const maxRawMessageBytes = this.options.queue.maxRawMessageBytes
+      ?? DEFAULT_MAX_RAW_MESSAGE_BYTES;
+    if (rawMessage.length > maxRawMessageBytes) {
+      throw new Error(`Raw outbound message exceeds ${maxRawMessageBytes} bytes`);
+    }
+    const releaseRawReservation = this.deliveryQueue[kQueueReserveRawAccess](rawMessage.length);
+    let exactRawMessage: Buffer;
+    let envelope: IRawSmtpEnvelope;
+    try {
+      exactRawMessage = Buffer.from(rawMessage);
+      assertCanonicalRawRfc822Message(exactRawMessage);
+      envelope = normalizeRawSmtpEnvelope(envelopeArg);
+    } catch (error) {
+      releaseRawReservation();
+      throw error;
+    }
+    const exactOptions = {
+      ...options,
+      ...(options.auth ? { auth: { ...options.auth } } : {}),
+    };
+    return this.trackOutboundWork(async () => {
+      try {
+        return await this[kSendRawOutboundEmailInternal](
+          host,
+          port,
+          exactRawMessage,
+          envelope,
+          exactOptions,
+        );
+      } finally {
+        releaseRawReservation();
+      }
+    });
+  }
+
+  public async [kSendRawOutboundEmailInternal](
+    host: string,
+    port: number,
+    rawMessage: Buffer,
+    envelopeArg: IRawSmtpEnvelope,
+    options: {
+      auth?: { user: string; pass: string };
+      dkimDomain: string;
+      dkimSelector: string;
+      tlsOpportunistic?: boolean;
+    },
+  ): Promise<ISmtpSendResult> {
+    assertCanonicalRawRfc822Message(rawMessage);
+    const maxRawMessageBytes = this.options.queue.maxRawMessageBytes
+      ?? DEFAULT_MAX_RAW_MESSAGE_BYTES;
+    if (rawMessage.length > maxRawMessageBytes) {
+      throw new Error(`Raw outbound message exceeds ${maxRawMessageBytes} bytes`);
+    }
+    const envelope = normalizeRawSmtpEnvelope(envelopeArg);
+    const senderDomain = envelope.from.split('@')[1];
+    const dkimDomain = options.dkimDomain.toLowerCase();
+    if (dkimDomain !== senderDomain) {
+      throw new SmtpDeliveryError({
+        message: `Raw outbound DKIM domain ${dkimDomain} does not match envelope sender domain ${senderDomain}`,
+        errorType: 'dkim',
+        retryable: false,
+        phase: 'message_body',
+        transactionLog: [],
+        recipients: envelope.to,
+      });
+    }
+    const dkim = await this.resolveOutboundDkim(dkimDomain, options.dkimSelector);
+    return this.sendWithOutboundTransport({
+      host,
+      port,
+      envelopeFrom: envelope.from,
+      senderDomain,
+      auth: options.auth,
+      tlsOpportunistic: options.tlsOpportunistic,
+      dkim,
+      send: (transport) => this.rustBridge.sendRawEmail({
+        ...transport,
+        envelopeFrom: envelope.from,
+        envelopeTo: [...envelope.to],
+        rawMessageBase64: rawMessage.toString('base64'),
+        dkim,
+      }),
+    });
+  }
+
+  private async resolveOutboundDkim(
+    domainArg: string,
+    selectorArg: string,
+  ): Promise<IOutboundDkimMaterial> {
+    try {
+      const material = await this.dkimCreator.readValidatedDKIMKeysForSelector(
+        domainArg,
+        selectorArg,
+      );
+      return {
+        domain: domainArg,
+        selector: selectorArg,
+        privateKey: material.privateKey,
+        keyType: material.keyType,
+      };
+    } catch (error) {
+      throw new SmtpDeliveryError({
+        message: getErrorMessage(error),
+        errorType: 'dkim',
+        retryable: false,
+        phase: 'message_body',
+        transactionLog: [],
+        recipients: [],
+      }, error);
+    }
+  }
+
+  private async sendWithOutboundTransport(optionsArg: {
+    host: string;
+    port: number;
+    envelopeFrom: string;
+    senderDomain?: string;
+    auth?: { user: string; pass: string };
+    tlsOpportunistic?: boolean;
+    dkim?: IOutboundDkimMaterial;
+    send: (transport: IOutboundTransportParameters) => Promise<ISmtpSendResult>;
+  }): Promise<ISmtpSendResult> {
+    const secure = optionsArg.port === 465;
+    const tlsOpportunistic = optionsArg.tlsOpportunistic ?? (optionsArg.port === 25);
+    const proxy = await this.options.outbound?.connectionProxyProvider?.({
+      host: optionsArg.host,
+      port: optionsArg.port,
       secure,
       ehloDomain: this.getOutboundHostname(),
       tlsOpportunistic,
-      senderDomain,
+      senderDomain: optionsArg.senderDomain,
     });
-
     const captureId = proxy?.sourceEvidence ? plugins.uuid.v4() : undefined;
     const attachAuthenticationHandle = <T extends ISmtpSendResult | SmtpDeliveryError>(valueArg: T): T => {
       if (!captureId || !proxy?.sourceEvidence) return valueArg;
@@ -954,10 +1164,10 @@ export class UnifiedEmailServer extends EventEmitter {
         captureId,
         sourceEvidence: proxy.sourceEvidence,
         heloDomain: proxy.ehloHostname || this.getOutboundHostname(),
-        receiverHostname: host,
-        envelopeFrom: email.from,
-        dkimDomain: dkim?.domain,
-        dkimSelector: dkim?.selector,
+        receiverHostname: optionsArg.host,
+        envelopeFrom: optionsArg.envelopeFrom,
+        dkimDomain: optionsArg.dkim?.domain,
+        dkimSelector: optionsArg.dkim?.selector,
       });
       Object.defineProperty(valueArg, 'outboundAuthenticationHandle', {
         value: handle,
@@ -968,18 +1178,16 @@ export class UnifiedEmailServer extends EventEmitter {
     };
 
     try {
-      const result = await this.rustBridge.sendOutboundEmail({
-        host,
-        port,
+      const result = await optionsArg.send({
+        host: optionsArg.host,
+        port: optionsArg.port,
         ...(proxy ? { connectHost: proxy.connectHost, connectPort: proxy.connectPort } : {}),
         secure,
         domain: proxy?.ehloHostname || this.getOutboundHostname(),
-        auth: options?.auth,
-        email: outboundEmail,
-        dkim,
+        auth: optionsArg.auth,
         connectionTimeoutSecs: Math.floor((this.options.outbound?.connectionTimeout || 30000) / 1000),
         socketTimeoutSecs: Math.floor((this.options.outbound?.socketTimeout || 120000) / 1000),
-        poolKey: proxy?.poolKey || `${host}:${port}`,
+        poolKey: proxy?.poolKey || `${optionsArg.host}:${optionsArg.port}`,
         maxPoolConnections: this.options.outbound?.maxConnections || 10,
         tlsOpportunistic,
         ...(proxy ? { disablePooling: true } : {}),
@@ -1015,7 +1223,8 @@ export class UnifiedEmailServer extends EventEmitter {
       checkedAt: new Date().toISOString(),
       error: errorArg,
     });
-    const result = (async (): Promise<IOutboundAuthenticationCheck> => {
+    let result: Promise<IOutboundAuthenticationCheck>;
+    result = (async (): Promise<IOutboundAuthenticationCheck> => {
       let timeout: ReturnType<typeof setTimeout> | undefined;
       try {
         const sourceEvidence = await new Promise<IOutboundConnectionSourceEvidence | null>((resolve, reject) => {
@@ -1071,7 +1280,10 @@ export class UnifiedEmailServer extends EventEmitter {
       } finally {
         if (timeout) clearTimeout(timeout);
       }
-    })();
+    })().finally(() => {
+      this.outboundAuthenticationFlights.delete(result);
+    });
+    this.outboundAuthenticationFlights.add(result);
     return { pending, result };
   }
 
@@ -1442,18 +1654,46 @@ export class UnifiedEmailServer extends EventEmitter {
   /**
    * Stop the unified email server
    */
-  public async stop(): Promise<void> {
+  public stop(): Promise<void> {
+    if (this.stopFlight) return this.stopFlight;
     logger.log('info', 'Stopping UnifiedEmailServer');
+    this.acceptingOutboundWork = false;
+    this.deliveryQueue[kQueueCloseAdmission]();
+    this.stopFlight = this.stopOnce();
+    return this.stopFlight;
+  }
 
-    try {
+  private async stopOnce(): Promise<void> {
+    const errors: Error[] = [];
+    const runStep = async (nameArg: string, stepArg: () => Promise<void> | void): Promise<void> => {
+      try {
+        await stepArg();
+      } catch (error) {
+        const wrapped = new Error(`${nameArg}: ${getErrorMessage(error)}`, { cause: error });
+        errors.push(wrapped);
+        logger.log('error', wrapped.message);
+      }
+    };
+    const runBestEffortStep = async (
+      nameArg: string,
+      stepArg: () => Promise<void> | void,
+    ): Promise<void> => {
+      try {
+        await stepArg();
+      } catch (error) {
+        logger.log('warn', `${nameArg}: ${getErrorMessage(error)}`);
+      }
+    };
+
+    await runStep('accepted-envelope cleanup drain failed', async () => {
       if (this.acceptedEnvelopeDispatchCleanupTimer) {
         clearInterval(this.acceptedEnvelopeDispatchCleanupTimer);
         this.acceptedEnvelopeDispatchCleanupTimer = undefined;
       }
       await this.acceptedEnvelopeDispatchCleanupPromise;
-      // Prevent new listener restarts from being scheduled or executed
-      // during shutdown, then drain any in-flight reconfiguration before
-      // tearing down (the chain may be extended while awaited).
+    });
+
+    await runStep('SMTP listener reconfiguration drain failed', async () => {
       this.smtpListenerShutdown = true;
       this.smtpListenerActive = false;
       while (this.smtpListenerRestartPromise) {
@@ -1463,55 +1703,49 @@ export class UnifiedEmailServer extends EventEmitter {
           this.smtpListenerRestartPromise = undefined;
         }
       }
-      await this.stopDnsValidation();
-
-      // Stop the Rust SMTP server first
-      try {
-        await this.rustBridge.stopSmtpServer();
-        logger.log('info', 'Rust SMTP server stopped');
-      } catch (err) {
-        logger.log('warn', `Error stopping Rust SMTP server: ${(err as Error).message}`);
-      }
-      this.smtpListenerActive = false;
-
-      // Clear the servers array - servers will be garbage collected
-      this.servers = [];
+    });
+    await runStep('DNS validation shutdown failed', () => this.stopDnsValidation());
 
-      this.unregisterBridgeEventHandlers();
-      this.unregisterBridgeStateHandler();
-      await this.rustBridge.stop();
+    await runBestEffortStep(
+      'Rust SMTP listener shutdown skipped or failed',
+      () => this.rustBridge.stopSmtpServer(),
+    );
+    this.smtpListenerActive = false;
+    this.servers = [];
 
-      // Stop the delivery system
-      if (this.deliverySystem) {
-        await this.deliverySystem.stop();
-        logger.log('info', 'Email delivery system stopped');
-      }
+    await runStep('delivery-system drain failed', async () => {
+      await this.deliverySystem?.stop();
+    });
 
-      // Shut down the delivery queue
-      if (this.deliveryQueue) {
-        await this.deliveryQueue.shutdown();
-        logger.log('info', 'Email delivery queue shut down');
-      }
+    while (this.outboundWorkFlights.size > 0) {
+      await Promise.allSettled([...this.outboundWorkFlights]);
+    }
+    while (this.outboundAuthenticationFlights.size > 0) {
+      await Promise.allSettled([...this.outboundAuthenticationFlights]);
+    }
 
-      this.bounceManager.stop();
-      logger.log('info', 'Bounce manager stopped');
+    await runStep('delivery-queue shutdown failed', async () => {
+      await this.deliveryQueue?.shutdown();
+    });
 
-      this.rateLimiter.stop();
-      logger.log('info', 'Rate limiter stopped');
+    await runStep('bounce-manager shutdown failed', () => this.bounceManager.stop());
+    await runStep('rate-limiter shutdown failed', () => this.rateLimiter.stop());
+    await runBestEffortStep(
+      'SMTP pool shutdown skipped or failed',
+      () => this.rustBridge.closeSmtpPool(),
+    );
 
-      // Close all Rust SMTP client connection pools
-      try {
-        await this.rustBridge.closeSmtpPool();
-      } catch {
-        // Bridge may already be stopped
-      }
+    await runStep('Rust bridge shutdown failed', async () => {
+      this.unregisterBridgeEventHandlers();
+      this.unregisterBridgeStateHandler();
+      await this.rustBridge.stop();
+    });
 
-      logger.log('info', 'UnifiedEmailServer stopped successfully');
-      await emitSafely(this, 'stopped');
-    } catch (error) {
-      logger.log('error', `Error stopping UnifiedEmailServer: ${getErrorMessage(error)}`);
-      throw error;
+    await emitSafely(this, 'stopped');
+    if (errors.length > 0) {
+      throw new AggregateError(errors, 'UnifiedEmailServer shutdown completed with errors');
     }
+    logger.log('info', 'UnifiedEmailServer stopped successfully');
   }
 
   // -----------------------------------------------------------------------
@@ -3663,6 +3897,15 @@ export class UnifiedEmailServer extends EventEmitter {
   /**
    * Send an email through the delivery system
    */
+  public async enqueueRawEmail(
+    rawMessage: Buffer,
+    envelope: IRawSmtpEnvelope,
+    idempotencyKey: string,
+  ): Promise<string> {
+    this.assertAcceptingOutboundWork();
+    return this.deliveryQueue[kQueueEnqueueRaw]({ rawMessage, envelope, idempotencyKey });
+  }
+
   public async sendEmail(
     email: Email,
     mode: EmailProcessingMode = 'mta',
@@ -3673,6 +3916,7 @@ export class UnifiedEmailServer extends EventEmitter {
       isTransactional?: boolean;
     }
   ): Promise<string> {
+    this.assertAcceptingOutboundWork();
     logger.log('info', `Sending email: ${email.subject} to ${email.to.join(', ')}`);
 
     try {

package/ts/mail/routing/symbols.unified.email.server.ts

@@ -0,0 +1,6 @@
+export const kSendOutboundEmailInternal: unique symbol = Symbol(
+  'smartmta.server.sendOutboundEmailInternal',
+);
+export const kSendRawOutboundEmailInternal: unique symbol = Symbol(
+  'smartmta.server.sendRawOutboundEmailInternal',
+);

package/ts/security/classes.rustsecuritybridge.ts

@@ -446,8 +446,14 @@ interface ISmtpSendRawOptions {
   envelopeFrom: string;
   envelopeTo: string[];
   rawMessageBase64: string;
+  dkim: { domain: string; selector: string; privateKey: string; keyType?: string };
+  connectionTimeoutSecs?: number;
+  socketTimeoutSecs?: number;
   poolKey?: string;
+  maxPoolConnections?: number;
+  tlsOpportunistic?: boolean;
   disablePooling?: boolean;
+  outboundAuthenticationCaptureId?: string;
 }
 
 interface ISmtpVerifyOptions {