@push.rocks/smartmta 9.4.1 → 9.5.0

package/ts/mail/delivery/classes.delivery.system.ts

@@ -23,6 +23,15 @@ import type {
 } from '../../security/classes.rustsecuritybridge.js';
 import type { ISmtpTransactionAttempt } from './interfaces.js';
 import { callObserverSafely, emitSafely } from './functions.safe-observers.js';
+import {
+  kQueueCreateItemSnapshot,
+  kQueueSetReadyItemsHandler,
+  kQueueWithRawMessage,
+} from './symbols.delivery.queue.js';
+import {
+  kSendOutboundEmailInternal,
+  kSendRawOutboundEmailInternal,
+} from '../routing/symbols.unified.email.server.js';
 
 const dns = plugins.dns;
 
@@ -148,8 +157,20 @@ export class MultiModeDeliverySystem extends EventEmitter {
   private emailServer?: UnifiedEmailServer;
   private bounceDrainTimer?: NodeJS.Timeout;
   private bounceDrainPromise?: Promise<void>;
-  private readonly queueItemsReadyHandler = (items: IQueueItem[]): Promise<void> =>
-    this.processItems(items);
+  private readonly dispatchFlights = new Set<Promise<void>>();
+  private readonly deliveryFlights = new Set<Promise<void>>();
+  private readonly authenticationUpdateFlights = new Set<Promise<void>>();
+  private stopFlight?: Promise<void>;
+  private readonly queueItemsReadyHandler = (itemIds: string[]): Promise<void> => {
+    let flight: Promise<void>;
+    flight = Promise.resolve()
+      .then(() => this.processItems(itemIds))
+      .finally(() => {
+        this.dispatchFlights.delete(flight);
+      });
+    this.dispatchFlights.add(flight);
+    return flight;
+  };
   private queueListenerRegistered = false;
   private lifecycleGeneration = 0;
   private pendingSmtpAuthenticationUpdates = new Map<string, {
@@ -233,16 +254,36 @@ export class MultiModeDeliverySystem extends EventEmitter {
 
   private registerQueueListener(): void {
     if (this.queueListenerRegistered) return;
-    this.queue.on('itemsReady', this.queueItemsReadyHandler);
+    this.queue[kQueueSetReadyItemsHandler](this.queueItemsReadyHandler);
     this.queueListenerRegistered = true;
   }
 
   private unregisterQueueListener(): void {
     if (!this.queueListenerRegistered) return;
-    this.queue.off('itemsReady', this.queueItemsReadyHandler);
+    this.queue[kQueueSetReadyItemsHandler](undefined);
     this.queueListenerRegistered = false;
   }
 
+  private async emitItemEventSafely(
+    eventNameArg: string,
+    itemArg: IQueueItem,
+    ...additionalArgsArg: unknown[]
+  ): Promise<void> {
+    for (const rawListener of this.rawListeners(eventNameArg)) {
+      try {
+        await Reflect.apply(rawListener, this, [
+          this.queue[kQueueCreateItemSnapshot](itemArg),
+          ...additionalArgsArg,
+        ]);
+      } catch (error) {
+        logger.log(
+          'error',
+          `Observer for ${eventNameArg} failed: ${getErrorMessage(error)}`,
+        );
+      }
+    }
+  }
+
   private createEmailForRecipients(email: Email, recipients: string[]): Email {
     const recipientSet = new Set(recipients.map((recipient) => recipient.toLowerCase()));
     return new Email({
@@ -368,20 +409,27 @@ export class MultiModeDeliverySystem extends EventEmitter {
         }
         await this.persistSmtpAuthenticationUpdate(update);
       };
-      void authenticationHandle.result.then(
-        persistAuthentication,
-        (failureArg) => persistAuthentication({
-          state: 'indeterminate',
-          overall: 'indeterminate',
-          checkedAt: new Date().toISOString(),
-          error: String(failureArg instanceof Error ? failureArg.message : failureArg).slice(0, 512),
-        }),
-      ).catch((error) => {
-        logger.log(
-          'warn',
-          `Failed to persist outbound authentication self-check: ${getErrorMessage(error)}`,
-        );
-      });
+      let authenticationUpdateFlight: Promise<void>;
+      authenticationUpdateFlight = authenticationHandle.result
+        .then(
+          persistAuthentication,
+          (failureArg) => persistAuthentication({
+            state: 'indeterminate',
+            overall: 'indeterminate',
+            checkedAt: new Date().toISOString(),
+            error: String(failureArg instanceof Error ? failureArg.message : failureArg).slice(0, 512),
+          }),
+        )
+        .catch((error) => {
+          logger.log(
+            'warn',
+            `Failed to persist outbound authentication self-check: ${getErrorMessage(error)}`,
+          );
+        })
+        .finally(() => {
+          this.authenticationUpdateFlights.delete(authenticationUpdateFlight);
+        });
+      this.authenticationUpdateFlights.add(authenticationUpdateFlight);
     }
     return attempt;
   }
@@ -556,7 +604,19 @@ export class MultiModeDeliverySystem extends EventEmitter {
   /**
    * Stop the delivery system
    */
-  public async stop(): Promise<void> {
+  public stop(): Promise<void> {
+    if (this.stopFlight) return this.stopFlight;
+    let stopFlight: Promise<void>;
+    stopFlight = this.stopOnce().finally(() => {
+      if (this.stopFlight === stopFlight) {
+        this.stopFlight = undefined;
+      }
+    });
+    this.stopFlight = stopFlight;
+    return stopFlight;
+  }
+
+  private async stopOnce(): Promise<void> {
     logger.log('info', 'Stopping MultiModeDeliverySystem');
     
     if (!this.running) {
@@ -584,26 +644,17 @@ export class MultiModeDeliverySystem extends EventEmitter {
     this.rateLimitLastCheck = Date.now();
     this.stats.rateLimiting.currentRate = 0;
 
-    // Wait for active deliveries to complete
-    if (this.activeDeliveries.size > 0) {
-      logger.log('info', `Waiting for ${this.activeDeliveries.size} active deliveries to complete`);
-      
-      // Wait for a maximum of 30 seconds
-      await new Promise<void>(resolve => {
-        const checkInterval = setInterval(() => {
-          if (this.activeDeliveries.size === 0) {
-            clearInterval(checkInterval);
-            clearTimeout(forceTimeout);
-            resolve();
-          }
-        }, 1000);
-        
-        // Force resolve after 30 seconds
-        const forceTimeout = setTimeout(() => {
-          clearInterval(checkInterval);
-          resolve();
-        }, 30000);
-      });
+    while (this.dispatchFlights.size > 0) {
+      await Promise.allSettled([...this.dispatchFlights]);
+    }
+    if (this.deliveryFlights.size > 0) {
+      logger.log('info', `Waiting for ${this.deliveryFlights.size} active deliveries to complete`);
+      while (this.deliveryFlights.size > 0) {
+        await Promise.allSettled([...this.deliveryFlights]);
+      }
+    }
+    while (this.authenticationUpdateFlights.size > 0) {
+      await Promise.allSettled([...this.authenticationUpdateFlights]);
     }
     
     await this.bounceDrainPromise;
@@ -617,7 +668,7 @@ export class MultiModeDeliverySystem extends EventEmitter {
    * Process ready items from the queue
    * @param items Queue items ready for processing
    */
-  private async processItems(items: IQueueItem[]): Promise<void> {
+  private async processItems(itemIds: string[]): Promise<void> {
     if (!this.running) {
       return;
     }
@@ -630,9 +681,9 @@ export class MultiModeDeliverySystem extends EventEmitter {
     
     let startedDeliveries = 0;
 
-    for (const item of items) {
+    for (const itemId of itemIds) {
       if (this.activeDeliveries.size >= this.options.concurrentDeliveries) break;
-      if (this.activeDeliveries.has(item.id)) continue;
+      if (this.activeDeliveries.has(itemId)) continue;
       if (this.checkRateLimit()) {
         logger.log('debug', 'Rate limit exceeded, throttling deliveries');
         break;
@@ -640,30 +691,43 @@ export class MultiModeDeliverySystem extends EventEmitter {
 
       // Reserve synchronously before the first await so overlapping queue events
       // cannot dispatch the same item while its durable claim is in flight.
-      this.activeDeliveries.add(item.id);
+      this.activeDeliveries.add(itemId);
       this.stats.activeDeliveries = this.activeDeliveries.size;
 
       let claimed = false;
       try {
-        claimed = await this.queue.markProcessing(item.id);
+        claimed = await this.queue.markProcessing(itemId);
       } catch (error) {
-        this.activeDeliveries.delete(item.id);
+        this.activeDeliveries.delete(itemId);
         this.stats.activeDeliveries = this.activeDeliveries.size;
-        logger.log('error', `Failed to claim queue item ${item.id}: ${(error as Error).message}`);
+        logger.log('error', `Failed to claim queue item ${itemId}: ${(error as Error).message}`);
         continue;
       }
 
       if (!claimed) {
-        this.activeDeliveries.delete(item.id);
+        this.activeDeliveries.delete(itemId);
+        this.stats.activeDeliveries = this.activeDeliveries.size;
+        continue;
+      }
+
+      const item = this.queue.getItem(itemId);
+      if (!item) {
+        this.activeDeliveries.delete(itemId);
         this.stats.activeDeliveries = this.activeDeliveries.size;
         continue;
       }
 
       startedDeliveries++;
-      // Deliver asynchronously
-      this.deliverItem(item).catch(err => {
-        logger.log('error', `Unhandled error in delivery: ${err.message}`);
-      });
+      let deliveryFlight: Promise<void>;
+      deliveryFlight = Promise.resolve()
+        .then(() => this.deliverItem(item))
+        .catch((error) => {
+          logger.log('error', `Unhandled error in delivery: ${getErrorMessage(error)}`);
+        })
+        .finally(() => {
+          this.deliveryFlights.delete(deliveryFlight);
+        });
+      this.deliveryFlights.add(deliveryFlight);
     }
 
     if (startedDeliveries > 0) {
@@ -683,10 +747,14 @@ export class MultiModeDeliverySystem extends EventEmitter {
     
     try {
       // Call delivery start hook
-      await callObserverSafely('delivery start hook', this.options.onDeliveryStart, item);
+      await callObserverSafely(
+        'delivery start hook',
+        this.options.onDeliveryStart,
+        this.queue[kQueueCreateItemSnapshot](item),
+      );
       
       // Emit delivery start event
-      await emitSafely(this, 'deliveryStart', item);
+      await this.emitItemEventSafely('deliveryStart', item);
       logger.log('info', `Starting delivery of item ${item.id}, mode: ${item.processingMode}`);
       
       // Choose the appropriate handler based on mode
@@ -694,15 +762,17 @@ export class MultiModeDeliverySystem extends EventEmitter {
       
       switch (item.processingMode) {
         case 'forward':
-          result = await this.options.forwardHandler.deliver(item);
+          result = await this.options.forwardHandler.deliver(this.queue[kQueueCreateItemSnapshot](item));
           break;
           
         case 'mta':
-          result = await this.options.deliveryHandler.deliver(item);
+          result = item.messageFormat === 'raw-rfc822'
+            ? await this.handleMtaDelivery(this.queue[kQueueCreateItemSnapshot](item))
+            : await this.options.deliveryHandler.deliver(this.queue[kQueueCreateItemSnapshot](item));
           break;
           
         case 'process':
-          result = await this.options.processHandler.deliver(item);
+          result = await this.options.processHandler.deliver(this.queue[kQueueCreateItemSnapshot](item));
           break;
           
         default:
@@ -752,11 +822,11 @@ export class MultiModeDeliverySystem extends EventEmitter {
       await callObserverSafely(
         'delivery success hook',
         this.options.onDeliverySuccess,
-        item,
+        this.queue[kQueueCreateItemSnapshot](item),
         result,
       );
 
-      await emitSafely(this, 'deliverySuccess', item, result);
+      await this.emitItemEventSafely('deliverySuccess', item, result);
       logger.log('info', `Item ${item.id} delivered successfully in ${deliveryTime}ms`);
       
       await callObserverSafely('delivery security log', () => SecurityLogger.getInstance().logEvent({
@@ -816,17 +886,17 @@ export class MultiModeDeliverySystem extends EventEmitter {
         await callObserverSafely(
           'delivery failure hook',
           this.options.onDeliveryFailed,
-          item,
+          this.queue[kQueueCreateItemSnapshot](item),
           deliveryError.message,
         );
       }
       
       // Emit delivery failed event
       if (transition.terminal) {
-        await emitSafely(this, 'deliveryFailed', item, deliveryError);
-        await emitSafely(this, 'deliveryTerminalFailure', item, deliveryError, transition);
+        await this.emitItemEventSafely('deliveryFailed', item, deliveryError);
+        await this.emitItemEventSafely('deliveryTerminalFailure', item, deliveryError, transition);
       } else {
-        await emitSafely(this, 'deliveryDeferred', item, deliveryError, transition);
+        await this.emitItemEventSafely('deliveryDeferred', item, deliveryError, transition);
       }
       logger.log(
         transition.terminal ? 'error' : 'warn',
@@ -908,7 +978,7 @@ export class MultiModeDeliverySystem extends EventEmitter {
       const auth = route?.action.forward?.auth as { user: string; pass: string } | undefined;
 
       // Send via Rust SMTP client
-      const result = await this.emailServer.sendOutboundEmail(
+      const result = await this.emailServer[kSendOutboundEmailInternal](
         targetServer,
         targetPort,
         this.createEmailForRecipients(email, recipients),
@@ -1066,6 +1136,18 @@ export class MultiModeDeliverySystem extends EventEmitter {
     logger.log('info', `MTA delivery for item ${item.id}`);
 
     const email = item.processingResult as Email;
+    const rawEnvelope = item.messageFormat === 'raw-rfc822' ? item.smtpEnvelope : undefined;
+    if (item.messageFormat === 'raw-rfc822' && !rawEnvelope) {
+      throw new SmtpDeliveryError({
+        message: `Raw queue item ${item.id} is missing its SMTP envelope metadata`,
+        errorType: 'raw_queue_integrity',
+        retryable: false,
+        phase: 'message_body',
+        transactionLog: [],
+        recipients: [],
+      });
+    }
+    const allRecipients = rawEnvelope?.to || email.getAllRecipients();
 
     // Structural invariant: `deliver` actions are terminal local deliveries
     // and must never reach outbound MX relay (and thus never the self-MX
@@ -1079,7 +1161,7 @@ export class MultiModeDeliverySystem extends EventEmitter {
         retryable: false,
         phase: 'connect',
         transactionLog: [],
-        recipients: email.getAllRecipients(),
+        recipients: allRecipients,
       });
     }
 
@@ -1087,16 +1169,36 @@ export class MultiModeDeliverySystem extends EventEmitter {
       throw new Error('No email server available for MTA delivery');
     }
 
-    // Build DKIM options from route config
-    const effectiveDkimSigning = Boolean(
-      item.route?.action.options?.mtaOptions?.dkimSign || item.route?.action.process?.dkim,
-    );
-    const dkimDomain = effectiveDkimSigning
-      ? (item.route?.action.options?.mtaOptions?.dkimOptions?.domainName || email.getFromDomain() || undefined)
-      : undefined;
-    const dkimSelector = this.resolveDkimSelector(item, dkimDomain);
+    // Raw queue delivery always resolves the current sender-domain selector and
+    // key at final transmit. The unsigned queue bytes never change.
+    let dkimDomain: string | undefined;
+    let dkimSelector: string;
+    if (rawEnvelope) {
+      dkimDomain = rawEnvelope.from.split('@')[1]?.toLowerCase();
+      const domainConfig = dkimDomain
+        ? this.emailServer.getDomainRegistry().getDomainConfig(dkimDomain)
+        : undefined;
+      if (!dkimDomain || !domainConfig?.dkim?.selector) {
+        throw new SmtpDeliveryError({
+          message: `Raw outbound DKIM is unavailable for sender domain ${dkimDomain || 'unknown'}`,
+          errorType: 'dkim',
+          retryable: false,
+          phase: 'message_body',
+          transactionLog: [],
+          recipients: allRecipients,
+        });
+      }
+      dkimSelector = domainConfig.dkim.selector;
+    } else {
+      const effectiveDkimSigning = Boolean(
+        item.route?.action.options?.mtaOptions?.dkimSign || item.route?.action.process?.dkim,
+      );
+      dkimDomain = effectiveDkimSigning
+        ? (item.route?.action.options?.mtaOptions?.dkimOptions?.domainName || email.getFromDomain() || undefined)
+        : undefined;
+      dkimSelector = this.resolveDkimSelector(item, dkimDomain);
+    }
 
-    const allRecipients = email.getAllRecipients();
     if (allRecipients.length === 0) {
       throw new Error('No recipients specified for MTA delivery');
     }
@@ -1193,13 +1295,23 @@ export class MultiModeDeliverySystem extends EventEmitter {
         try {
           logger.log('info', `MTA: trying MX ${mx.exchange}:25 for domain ${domain} (priority ${mx.priority})`);
 
-          // Create a temporary Email scoped to this domain's recipients
-          const domainEmail = this.createEmailForRecipients(email, recipientsForMx);
-
-          const result = await this.emailServer.sendOutboundEmail(mx.exchange, 25, domainEmail, {
-            dkimDomain,
-            dkimSelector,
-          });
+          const result = rawEnvelope
+            ? await this.queue[kQueueWithRawMessage](
+                item.id,
+                (rawMessage) => this.emailServer![kSendRawOutboundEmailInternal](
+                  mx.exchange,
+                  25,
+                  rawMessage,
+                  { from: rawEnvelope.from, to: recipientsForMx },
+                  { dkimDomain: dkimDomain!, dkimSelector },
+                ),
+              )
+            : await this.emailServer[kSendOutboundEmailInternal](
+                mx.exchange,
+                25,
+                this.createEmailForRecipients(email, recipientsForMx),
+                { dkimDomain, dkimSelector },
+              );
 
           const disposition = this.splitRecipientResults(
             result.recipientResults,

package/ts/mail/delivery/functions.raw-email.ts

@@ -0,0 +1,111 @@
+import { createHash, type Hash } from 'node:crypto';
+import { domainToASCII } from 'node:url';
+
+import { Email } from '../core/classes.email.js';
+
+export interface IRawSmtpEnvelope {
+  from: string;
+  to: string[];
+}
+
+export const MAX_RAW_SMTP_RECIPIENTS = 100;
+
+export const assertCanonicalRawRfc822Message = (rawMessageArg: Buffer): void => {
+  if (!Buffer.isBuffer(rawMessageArg) || rawMessageArg.length === 0) {
+    throw new Error('Raw outbound message must be a nonempty Buffer');
+  }
+  for (let index = 0; index < rawMessageArg.length; index++) {
+    const byte = rawMessageArg[index];
+    if (byte === 0x0d && rawMessageArg[index + 1] !== 0x0a) {
+      throw new Error('Raw outbound message must use canonical CRLF line endings');
+    }
+    if (byte === 0x0a && (index === 0 || rawMessageArg[index - 1] !== 0x0d)) {
+      throw new Error('Raw outbound message must use canonical CRLF line endings');
+    }
+  }
+  if (
+    rawMessageArg.length < 2
+    || rawMessageArg[rawMessageArg.length - 2] !== 0x0d
+    || rawMessageArg[rawMessageArg.length - 1] !== 0x0a
+  ) {
+    throw new Error('Raw outbound message must end with CRLF');
+  }
+};
+
+const writeLengthFramed = (hashArg: Hash, labelArg: string, valueArg: Buffer): void => {
+  const label = Buffer.from(labelArg, 'utf8');
+  const labelLength = Buffer.allocUnsafe(4);
+  labelLength.writeUInt32BE(label.length);
+  const valueLength = Buffer.allocUnsafe(8);
+  valueLength.writeBigUInt64BE(BigInt(valueArg.length));
+  hashArg.update(labelLength).update(label).update(valueLength).update(valueArg);
+};
+
+const normalizeRawMailbox = (valueArg: string, fieldArg: string): string => {
+  if (typeof valueArg !== 'string' || /[\r\n<>]/.test(valueArg)) {
+    throw new Error(`Raw SMTP envelope ${fieldArg} must be a plain mailbox address`);
+  }
+  const value = valueArg.trim();
+  const atIndex = value.indexOf('@');
+  if (atIndex < 1 || atIndex !== value.lastIndexOf('@') || atIndex === value.length - 1) {
+    throw new Error(`Raw SMTP envelope ${fieldArg} must be a plain mailbox address`);
+  }
+  const asciiDomain = domainToASCII(value.slice(atIndex + 1));
+  if (!asciiDomain) {
+    throw new Error(`Raw SMTP envelope ${fieldArg} has an invalid domain`);
+  }
+  const normalized = `${value.slice(0, atIndex).toLowerCase()}@${asciiDomain.toLowerCase()}`;
+  try {
+    new Email({ from: normalized, to: normalized, subject: '', text: '' });
+  } catch {
+    throw new Error(`Raw SMTP envelope ${fieldArg} must be a valid mailbox address`);
+  }
+  return normalized;
+};
+
+export const normalizeRawSmtpEnvelope = (envelopeArg: IRawSmtpEnvelope): IRawSmtpEnvelope => {
+  if (!envelopeArg || typeof envelopeArg !== 'object') {
+    throw new Error('Raw SMTP envelope is required');
+  }
+  if (
+    !Array.isArray(envelopeArg.to)
+    || envelopeArg.to.length === 0
+    || envelopeArg.to.length > MAX_RAW_SMTP_RECIPIENTS
+  ) {
+    throw new Error(
+      `Raw SMTP envelope requires 1 to ${MAX_RAW_SMTP_RECIPIENTS} recipients`,
+    );
+  }
+  const from = normalizeRawMailbox(envelopeArg.from, 'sender');
+  const to = envelopeArg.to.map((recipient, index) => (
+    normalizeRawMailbox(recipient, `recipient ${index + 1}`)
+  ));
+  if (new Set(to).size !== to.length) {
+    throw new Error('Raw SMTP envelope recipients must be unique after canonicalization');
+  }
+  return { from, to };
+};
+
+export const cloneRawSmtpEnvelope = (envelopeArg: IRawSmtpEnvelope): IRawSmtpEnvelope => ({
+  from: envelopeArg.from,
+  to: [...envelopeArg.to],
+});
+
+export const createRawQueueFingerprint = (
+  rawMessageArg: Buffer,
+  envelopeArg: IRawSmtpEnvelope,
+): string => {
+  const hash = createHash('sha256');
+  hash.update('smartmta:durable-raw-outbound-queue:fingerprint:v1\0');
+  writeLengthFramed(hash, 'raw-rfc822-bytes', rawMessageArg);
+  writeLengthFramed(hash, 'smtp-envelope-from', Buffer.from(envelopeArg.from, 'utf8'));
+  for (const recipient of envelopeArg.to) {
+    writeLengthFramed(hash, 'smtp-envelope-recipient', Buffer.from(recipient, 'utf8'));
+  }
+  writeLengthFramed(
+    hash,
+    'processing-semantics',
+    Buffer.from('raw-mta;dkim=required-current-sender-domain-key;transform=none', 'utf8'),
+  );
+  return hash.digest('hex');
+};

package/ts/mail/delivery/symbols.delivery.queue.ts

@@ -0,0 +1,6 @@
+export const kQueueCloseAdmission: unique symbol = Symbol('smartmta.queue.closeAdmission');
+export const kQueueCreateItemSnapshot: unique symbol = Symbol('smartmta.queue.createItemSnapshot');
+export const kQueueEnqueueRaw: unique symbol = Symbol('smartmta.queue.enqueueRaw');
+export const kQueueReserveRawAccess: unique symbol = Symbol('smartmta.queue.reserveRawAccess');
+export const kQueueSetReadyItemsHandler: unique symbol = Symbol('smartmta.queue.setReadyItemsHandler');
+export const kQueueWithRawMessage: unique symbol = Symbol('smartmta.queue.withRawMessage');

package/ts/mail/index.ts

@@ -3,6 +3,7 @@ export * from './interfaces.storage.js';
 export * from './routing/index.js';
 export * from './security/index.js';
 export type { ISmtpTransactionAttempt } from './delivery/interfaces.js';
+export type { IRawSmtpEnvelope } from './delivery/functions.raw-email.js';
 
 // Make the core and delivery modules accessible
 import * as Core from './core/index.js';