emailengine-app 2.61.1 → 2.61.3

package/lib/email-client/base-client.js

@@ -1,11 +1,8 @@
 'use strict';
 
-const { parentPort, threadId: workerThreadId } = require('worker_threads');
+const { threadId: workerThreadId } = require('worker_threads');
 const crypto = require('crypto');
 const logger = require('../logger');
-const { webhooks: Webhooks } = require('../webhooks');
-const { getESClient } = require('../document-store');
-const { getThread } = require('../threads');
 const settings = require('../settings');
 const msgpack = require('msgpack5')();
 const { templates } = require('../templates');
@@ -19,6 +16,7 @@ const { getRawEmail } = require('../get-raw-email');
 const { getTemplate } = require('@postalsys/templates');
 const { deepEqual } = require('assert');
 const { arfDetect } = require('../arf-detect');
+const { createTransaction } = require('../redis-operations');
 const simpleParser = require('mailparser').simpleParser;
 const libmime = require('libmime');
 const { bounceDetect } = require('../bounce-detect');
@@ -26,15 +24,20 @@ const ical = require('ical.js');
 const { llmPreProcess } = require('../llm-pre-process');
 const { oauth2Apps } = require('../oauth2-apps');
 const { Account } = require('../account');
-const util = require('util');
-const socks = require('socks');
-const nodemailer = require('nodemailer');
 const { removeBcc } = require('../get-raw-email');
-const { oauth2ProviderData } = require('../oauth2-apps');
+const {
+    SmtpConfigBuilder,
+    NetworkRoutingBuilder,
+    NotificationBuilder,
+    ProviderMessageIdHandler,
+    SmtpErrorBuilder,
+    SentMailCopyDecider
+} = require('./message-builder');
+const { NotificationHandler, postMetrics } = require('./notification-handler');
+const { getMailTransport } = require('./smtp-pool-manager');
 
 // Import various utility functions and constants
 const {
-    getLocalAddress,
     getSignedFormDataSync,
     getServiceSecret,
     convertDataUrisToAttachments,
@@ -44,7 +47,6 @@ const {
     readEnvValue,
     emitChangeEvent,
     filterEmptyObjectValues,
-    resolveCredentials,
     getDateBuckets
 } = require('../tools');
 
@@ -54,17 +56,13 @@ const {
     ACCOUNT_INITIALIZED_NOTIFY,
     REDIS_PREFIX,
     MESSAGE_NEW_NOTIFY,
-    MESSAGE_DELETED_NOTIFY,
-    MESSAGE_UPDATED_NOTIFY,
     EMAIL_BOUNCE_NOTIFY,
-    MAILBOX_DELETED_NOTIFY,
     DEFAULT_DELIVERY_ATTEMPTS,
     MIME_BOUNDARY_PREFIX,
     DEFAULT_DOWNLOAD_CHUNK_SIZE,
     EMAIL_COMPLAINT_NOTIFY,
     MAX_INLINE_ATTACHMENT_SIZE,
     DEFAULT_MAX_IMAP_AUTH_FAILURE_TIME,
-    TLS_DEFAULTS,
     EMAIL_DELIVERY_ERROR_NOTIFY,
     EMAIL_SENT_NOTIFY
 } = require('../consts');
@@ -75,154 +73,9 @@ const DOWNLOAD_CHUNK_SIZE = getByteSize(readEnvValue('EENGINE_CHUNK_SIZE')) || D
 // Configure maximum time to wait before disabling IMAP on authentication failures
 const MAX_IMAP_AUTH_FAILURE_TIME = getDuration(readEnvValue('EENGINE_MAX_IMAP_AUTH_FAILURE_TIME')) || DEFAULT_MAX_IMAP_AUTH_FAILURE_TIME;
 
-/**
- * Sends metrics data to the parent thread for aggregation
- * @param {Object} meta - Metadata to include with the metric
- * @param {Object} logger - Logger instance
- * @param {string} key - Metric key identifier
- * @param {string} method - Metric method (e.g., 'inc', 'dec')
- * @param {...any} args - Additional arguments for the metric
- */
-async function metricsMeta(meta, logger, key, method, ...args) {
-    try {
-        parentPort.postMessage({
-            cmd: 'metrics',
-            key,
-            method,
-            args,
-            meta: meta || {}
-        });
-    } catch (err) {
-        logger.error({ msg: 'Failed to post metrics to parent', err });
-    }
-}
-
 // Map to track pending idempotency operations to prevent duplicate processing
 const pendingIdempotencyOperations = new Map();
 
-// Cache for SMTP connection pools to reuse connections
-const SMTP_POOLS = new Map();
-// Track last usage time for each pool to enable LRU-based cleanup
-const SMTP_POOL_LAST_USED = new Map();
-
-// Maximum idle time for SMTP pool connections (10 minutes of inactivity)
-const SMTP_POOL_MAX_IDLE = 10 * 60 * 1000;
-// Cleanup interval for idle SMTP pools (2 minutes)
-const SMTP_POOL_CLEANUP_INTERVAL = 2 * 60 * 1000;
-
-// Periodic cleanup of idle SMTP pool connections
-let smtpPoolCleanupTimer = setInterval(() => {
-    const now = Date.now();
-    const idleKeys = [];
-
-    for (const [poolKey, lastUsed] of SMTP_POOL_LAST_USED.entries()) {
-        if (now - lastUsed > SMTP_POOL_MAX_IDLE) {
-            idleKeys.push(poolKey);
-        }
-    }
-
-    for (const poolKey of idleKeys) {
-        const transporter = SMTP_POOLS.get(poolKey);
-        if (transporter) {
-            // Check if the transporter has active connections
-            if (transporter._connectionPool && transporter._connectionPool.size > 0) {
-                // Still has active connections, update last used time
-                SMTP_POOL_LAST_USED.set(poolKey, now);
-                logger.trace({ msg: 'SMTP pool still has active connections, skipping cleanup', poolKey });
-                continue;
-            }
-
-            logger.debug({ msg: 'Cleaning up idle SMTP pool connection', poolKey, idleTime: now - SMTP_POOL_LAST_USED.get(poolKey) });
-            try {
-                transporter.close();
-            } catch (err) {
-                logger.error({ msg: 'Failed to close idle SMTP transporter', poolKey, err });
-            }
-        }
-        SMTP_POOLS.delete(poolKey);
-        SMTP_POOL_LAST_USED.delete(poolKey);
-    }
-
-    if (idleKeys.length > 0) {
-        logger.info({ msg: 'SMTP pool cleanup completed', cleaned: idleKeys.length, remaining: SMTP_POOLS.size });
-    }
-}, SMTP_POOL_CLEANUP_INTERVAL);
-
-// Prevent the timer from keeping the process alive
-smtpPoolCleanupTimer.unref();
-
-/**
- * Gets or creates a reusable SMTP transport for the given configuration
- * Connection pooling improves performance by reusing SMTP connections
- * @param {Object} smtpSettings - SMTP configuration settings
- * @returns {Object} Nodemailer transport instance
- */
-function getMailTransport(smtpSettings) {
-    // Extract only the settings that affect connection identity
-    let limitedSettings = {};
-    for (let key of ['name', 'localAddress', 'auth', 'host', 'port', 'secure', 'transactionLog', 'proxy']) {
-        limitedSettings[key] = smtpSettings[key];
-    }
-
-    // Create a unique key for this SMTP configuration
-    let serializedSettings = JSON.stringify(limitedSettings);
-    let poolKey = crypto.createHash('sha256').update(serializedSettings).digest('hex');
-
-    // Return existing transport if available
-    let transporter;
-    if (SMTP_POOLS.has(poolKey)) {
-        transporter = SMTP_POOLS.get(poolKey);
-        // Update last used time for LRU tracking
-        SMTP_POOL_LAST_USED.set(poolKey, Date.now());
-        return transporter;
-    }
-
-    // Configure connection pooling settings
-    smtpSettings.pool = true;
-    smtpSettings.maxConnections = 1;
-    smtpSettings.maxMessages = 100;
-    smtpSettings.socketTimeout = 2 * 60 * 1000; // 2 minute timeout
-
-    // Create new transport with pooling enabled
-    transporter = nodemailer.createTransport(smtpSettings);
-    transporter.set('proxy_socks_module', socks);
-
-    // Handle connection pool cleanup when idle
-    transporter.once('clear', () => {
-        // all emails processed and connection timed out
-        logger.trace({ msg: 'Clearing disconnected SMTP pool', poolKey });
-        SMTP_POOLS.delete(poolKey);
-        SMTP_POOL_LAST_USED.delete(poolKey);
-        try {
-            transporter.close();
-        } catch (closeErr) {
-            logger.error({ msg: 'Failed to close transporter', err: closeErr });
-        }
-        transporter = null;
-    });
-
-    // Handle transport errors by removing from pool
-    transporter.once('error', err => {
-        // not sure what happned, but do not re-use this transporter object anymore
-        logger.error({ msg: 'Transporter failed', err });
-        SMTP_POOLS.delete(poolKey);
-        SMTP_POOL_LAST_USED.delete(poolKey);
-        try {
-            transporter.close();
-        } catch (closeErr) {
-            logger.error({ msg: 'Failed to close transporter', err: closeErr });
-        }
-        transporter = null;
-    });
-
-    // Cache the transport for reuse
-    SMTP_POOLS.set(poolKey, transporter);
-    SMTP_POOL_LAST_USED.set(poolKey, Date.now());
-    logger.trace({ msg: 'Created SMTP pool', poolKey });
-
-    return transporter;
-}
-
 /**
  * Check if an error is a transient error that should be retried
  * @param {Error} err - The error to check
@@ -329,6 +182,14 @@ class BaseClient {
 
         // Track sub-connections (e.g., for IMAP IDLE)
         this.subconnections = [];
+
+        // Initialize notification handler for webhook and event delivery
+        this.notificationHandler = new NotificationHandler({
+            account: this.account,
+            logger: this.logger,
+            flowProducer: this.flowProducer,
+            documentsQueue: this.documentsQueue
+        });
     }
 
     // Stub methods to be implemented by subclasses
@@ -463,18 +324,17 @@ class BaseClient {
      * Sends notification on first successful connection
      */
     async setStateVal() {
-        let [[e1], [e2], [e3, prevVal], [e4, incrVal], [e5, stateVal]] = await this.redis
-            .multi()
-            .hSetExists(this.getAccountKey(), 'state', this.state)
-            .hSetBigger(this.getAccountKey(), 'runIndex', this.runIndex.toString())
-            .hget(this.getAccountKey(), `state:count:${this.state}`)
-            .hIncrbyExists(this.getAccountKey(), `state:count:${this.state}`, 1)
-            .hget(this.getAccountKey(), 'state')
-            .exec();
-
-        if (e1 || e2 || e3 || e4 || e5) {
-            throw e1 || e2 || e3 || e4 || e5;
-        }
+        const txn = createTransaction(this.redis, { logger: this.logger });
+        txn.add('hSetExists', this.getAccountKey(), 'state', this.state);
+        txn.add('hSetBigger', this.getAccountKey(), 'runIndex', this.runIndex.toString());
+        txn.add('hget', this.getAccountKey(), `state:count:${this.state}`);
+        txn.add('hIncrbyExists', this.getAccountKey(), `state:count:${this.state}`, 1);
+        txn.add('hget', this.getAccountKey(), 'state');
+
+        const values = await txn.execOrThrow();
+        const prevVal = values[2];
+        const incrVal = values[3];
+        const stateVal = values[4];
 
         // Detect first successful connection
         if (stateVal === 'connected' && incrVal === 1 && prevVal === '0') {
@@ -532,31 +392,30 @@ class BaseClient {
 
         // Track error occurrences
         if (isFirstOccurrence) {
-            await this.redis
-                .multi()
-                .hSetExists(this.getAccountKey(), 'lastError:errorCount', 1)
-                .hSetExists(this.getAccountKey(), 'lastError:first', new Date().toISOString())
-                .exec();
+            const txn = createTransaction(this.redis, { logger: this.logger });
+            txn.add('hSetExists', this.getAccountKey(), 'lastError:errorCount', 1);
+            txn.add('hSetExists', this.getAccountKey(), 'lastError:first', new Date().toISOString());
+            await txn.exec();
         } else {
             let errorCount;
             let firstError;
 
-            let [[err1, ec], [err2, fe]] = await this.redis
-                .multi()
-                .hIncrbyExists(this.getAccountKey(), `lastError:errorCount`, 1)
-                .hget(this.getAccountKey(), 'lastError:first')
-                .exec();
+            const txn = createTransaction(this.redis, { logger: this.logger });
+            txn.add('hIncrbyExists', this.getAccountKey(), 'lastError:errorCount', 1);
+            txn.add('hget', this.getAccountKey(), 'lastError:first');
+            const { error, values } = await txn.exec();
 
-            if (!err1 && !err2) {
-                errorCount = ec || 0;
+            if (!error) {
+                errorCount = values[0] || 0;
+                const fe = values[1];
                 if (fe) {
-                    fe = new Date(fe);
-                    if (fe.toString() !== 'Invalid Date') {
-                        firstError = fe;
+                    const parsedDate = new Date(fe);
+                    if (parsedDate.toString() !== 'Invalid Date') {
+                        firstError = parsedDate;
                     }
                 }
             } else {
-                this.logger.error({ msg: 'Redis error while checking error state counters', err1, err2 });
+                this.logger.error({ msg: 'Redis error while checking error state counters', error });
             }
 
             // Handle repeated authentication errors by disabling IMAP
@@ -577,19 +436,19 @@ class BaseClient {
                             imapData.disabled = true;
 
                             // Disable IMAP and reset error counters
-                            await this.redis
-                                .multi()
-                                .hSetExists(this.getAccountKey(), 'imap', JSON.stringify(imapData))
-                                .hdel(this.getAccountKey(), 'lastError:errorCount', 'lastError:first')
-                                .hSetExists(
-                                    this.getAccountKey(),
-                                    'lastErrorState',
-                                    JSON.stringify({
-                                        description: 'IMAP was disabled for the account due to exceeding the authentication error threshold',
-                                        response: data.response
-                                    })
-                                )
-                                .exec();
+                            const disableTxn = createTransaction(this.redis, { logger: this.logger });
+                            disableTxn.add('hSetExists', this.getAccountKey(), 'imap', JSON.stringify(imapData));
+                            disableTxn.add('hdel', this.getAccountKey(), 'lastError:errorCount', 'lastError:first');
+                            disableTxn.add(
+                                'hSetExists',
+                                this.getAccountKey(),
+                                'lastErrorState',
+                                JSON.stringify({
+                                    description: 'IMAP was disabled for the account due to exceeding the authentication error threshold',
+                                    response: data.response
+                                })
+                            );
+                            await disableTxn.exec();
 
                             this.logger.info({
                                 msg: 'IMAP was disabled for the account due to exceeding the authentication error threshold',
@@ -614,22 +473,16 @@ class BaseClient {
 
     /**
      * Sends notifications for various email events through webhooks and queues
-     * Handles special processing for certain event types and document store integration
+     * Handles error state tracking for connection/auth errors before delegating
+     * to the notification handler for actual delivery
      * @param {Object} mailbox - Mailbox information
      * @param {string} event - Event type constant
      * @param {Object} data - Event data payload
      * @param {Object} extraOpts - Additional options
      */
     async notify(mailbox, event, data, extraOpts) {
-        extraOpts = extraOpts || {};
-        const { skipWebhook, canSync = true } = extraOpts;
-
-        // Track event metrics
-        metricsMeta({ account: this.account }, this.logger, 'events', 'inc', {
-            event
-        });
-
         // Handle error state tracking for connection/auth errors
+        // This is BaseClient-specific logic that must run before notification
         switch (event) {
             case 'connectError':
             case 'authenticationError': {
@@ -643,133 +496,8 @@ class BaseClient {
             }
         }
 
-        let serviceUrl = (await settings.get('serviceUrl')) || null;
-
-        // Build notification payload
-        let payload = {
-            serviceUrl,
-            account: this.account,
-            date: new Date().toISOString()
-        };
-
-        let path = (mailbox && mailbox.path) || (data && data.path);
-        if (path) {
-            payload.path = path;
-        }
-
-        if (mailbox && mailbox.listingEntry && mailbox.listingEntry.specialUse) {
-            payload.specialUse = mailbox.listingEntry.specialUse;
-        }
-
-        if (event) {
-            payload.event = event;
-        }
-
-        if (data) {
-            payload.data = data;
-        }
-
-        let queueKeep = (await settings.get('queueKeep')) || true;
-
-        // Determine if we need to sync with document store (ElasticSearch)
-        let addDocumentQueueJob =
-            canSync &&
-            this.documentsQueue &&
-            [MESSAGE_NEW_NOTIFY, MESSAGE_DELETED_NOTIFY, MESSAGE_UPDATED_NOTIFY, EMAIL_BOUNCE_NOTIFY, MAILBOX_DELETED_NOTIFY].includes(event) &&
-            (await settings.get('documentStoreEnabled'));
-
-        // Generate thread ID for new messages if needed
-        if (addDocumentQueueJob && payload.data && event === MESSAGE_NEW_NOTIFY && !payload.data.threadId) {
-            // Generate a thread ID for the email. This is also stored in ElasticSearch.
-            const { index, client } = await getESClient(logger);
-            try {
-                if (client) {
-                    let thread = await getThread(client, index, this.account, payload.data, logger);
-                    if (thread) {
-                        payload.data.threadId = thread;
-                        logger.info({
-                            msg: 'Provisioned thread ID for a message',
-                            account: this.account,
-                            message: payload.data.id,
-                            threadId: payload.data.threadId
-                        });
-                    }
-                }
-            } catch (err) {
-                if (logger.notifyError) {
-                    logger.notifyError(err, event => {
-                        event.setUser(this.account);
-                        event.addMetadata('ee', {
-                            index
-                        });
-                    });
-                }
-                logger.error({ msg: 'Failed to resolve thread', account: this.account, message: payload.data.id, err });
-            }
-        }
-
-        // Configure job options for queuing
-        const defaultJobOptions = {
-            removeOnComplete: queueKeep,
-            removeOnFail: queueKeep,
-            attempts: 10,
-            backoff: {
-                type: 'exponential',
-                delay: 5000
-            }
-        };
-
-        // use more attempts for ElasticSearch updates
-        const documentJobOptions = Object.assign(structuredClone(defaultJobOptions), { attempts: 16 });
-
-        // Process notifications with flow (webhook + document store) or separately
-        if (!skipWebhook && addDocumentQueueJob) {
-            // add both jobs as a Flow
-
-            let notifyPayload = await Webhooks.formatPayload(event, payload);
-
-            const queueFlow = [
-                {
-                    name: event,
-                    data: payload,
-                    queueName: 'documents'
-                }
-            ];
-
-            await Webhooks.pushToQueue(event, notifyPayload, {
-                routesOnly: true,
-                queueFlow
-            });
-
-            await this.flowProducer.add(
-                {
-                    name: event,
-                    data: notifyPayload,
-                    queueName: 'notify',
-                    children: queueFlow
-                },
-                {
-                    queuesOptions: {
-                        notify: {
-                            defaultJobOptions
-                        },
-                        documents: {
-                            defaultJobOptions: documentJobOptions
-                        }
-                    }
-                }
-            );
-        } else {
-            // add to queues as normal jobs
-
-            if (!skipWebhook) {
-                await Webhooks.pushToQueue(event, await Webhooks.formatPayload(event, payload));
-            }
-
-            if (addDocumentQueueJob) {
-                await this.documentsQueue.add(event, payload, documentJobOptions);
-            }
-        }
+        // Delegate to notification handler for webhook and document store delivery
+        await this.notificationHandler.notify(mailbox, event, data, extraOpts);
     }
 
     /**
@@ -801,7 +529,7 @@ class BaseClient {
         }
 
         // Check if token needs refresh (with 30 second buffer)
-        if (!accountData.oauth2.accessToken || !accountData.oauth2.expires || accountData.oauth2.expires < new Date(now - 30 * 1000)) {
+        if (!accountData.oauth2.accessToken || !accountData.oauth2.expires || accountData.oauth2.expires < new Date(now + 30 * 1000)) {
             // renew access token
             try {
                 accountData = await accountObject.renewAccessToken({
@@ -3176,210 +2904,66 @@ class BaseClient {
      * @returns {Object} Delivery result with response and messageId
      */
     async submitMessage(data) {
-        let accountData = await this.accountObject.loadAccountData();
+        const accountData = await this.accountObject.loadAccountData();
 
         // Verify SMTP capability
         if (!accountData.smtp && !accountData.oauth2 && !data.gateway) {
-            // can not make connection
-            let err = new Error('SMTP configuration not found');
+            const err = new Error('SMTP configuration not found');
             err.code = 'SMTPUnavailable';
             err.statusCode = 404;
             throw err;
         }
 
-        // Load gateway configuration if specified
-        let gatewayData;
-        let gatewayObject;
-        if (data.gateway) {
-            gatewayObject = new Gateway({ gateway: data.gateway, redis: this.redis, secret: this.secret });
-            try {
-                gatewayData = await gatewayObject.loadGatewayData();
-            } catch (err) {
-                this.logger.info({ msg: 'Failed to load gateway data', messageId: data.messageId, gateway: data.gateway, err });
-            }
-        }
-
-        let smtpConnectionConfig;
-
-        // Configure SMTP connection based on authentication type
-        if (gatewayData) {
-            // Use gateway configuration
-            smtpConnectionConfig = {
-                host: gatewayData.host,
-                port: gatewayData.port,
-                secure: gatewayData.secure
-            };
-            if (gatewayData.user || gatewayData.pass) {
-                smtpConnectionConfig.auth = {
-                    user: gatewayData.user || '',
-                    pass: gatewayData.pass || ''
-                };
-            }
-        } else if (accountData.oauth2 && accountData.oauth2.auth) {
-            // load OAuth2 tokens
-            const { oauth2User, accessToken, oauth2App } = await this.loadOAuth2AccountCredentials(accountData, this, 'smtp');
-            const providerData = oauth2ProviderData(oauth2App.provider, oauth2App.cloud);
-
-            smtpConnectionConfig = Object.assign(
-                {
-                    auth: {
-                        user: oauth2User,
-                        accessToken
-                    },
-                    resyncDelay: 900
-                },
-                providerData.smtp || {}
-            );
-        } else {
-            // deep copy of smtp settings
-            smtpConnectionConfig = JSON.parse(JSON.stringify(accountData.smtp));
-        }
-
-        let { raw, hasBcc, envelope, messageId, queueId, reference, job: jobData } = data;
-
-        let smtpAuth = smtpConnectionConfig.auth;
-        // If authentication server is set then it overrides authentication data
-        if (smtpConnectionConfig.useAuthServer) {
-            try {
-                smtpAuth = await resolveCredentials(this.account, 'smtp');
-            } catch (err) {
-                err.authenticationFailed = true;
-                this.logger.error({
-                    account: this.account,
-                    err
-                });
-                throw err;
-            }
-        }
-
-        // Select local address for outbound connection
-        let { localAddress: address, name, addressSelector: selector } = await getLocalAddress(this.redis, 'smtp', this.account, data.localAddress);
-        this.logger.info({
-            msg: 'Selected local address',
-            account: this.account,
-            proto: 'SMTP',
-            address,
-            name,
-            selector,
-            requestedLocalAddress: data.localAddress
+        // Build SMTP configuration using SmtpConfigBuilder
+        const smtpConfigBuilder = new SmtpConfigBuilder({
+            redis: this.redis,
+            secret: this.secret,
+            logger: this.logger,
+            account: this.account
         });
 
-        // Configure SMTP logger
-        let smtpLogger = {};
-        let smtpSettings = Object.assign(
-            {
-                name,
-                localAddress: address,
-                transactionLog: true,
-                logger: smtpLogger
-            },
-            smtpConnectionConfig
-        );
+        // Load gateway if specified
+        const { gatewayData, gatewayObject } = await smtpConfigBuilder.loadGateway(data.gateway, data.messageId);
 
-        // Set authentication
-        if (smtpAuth) {
-            smtpSettings.auth = {
-                user: smtpAuth.user
-            };
-
-            if (smtpAuth.accessToken) {
-                smtpSettings.auth.type = 'OAuth2';
-                smtpSettings.auth.accessToken = smtpAuth.accessToken;
-            } else {
-                smtpSettings.auth.pass = smtpAuth.pass;
-            }
-        }
+        // Build connection configuration
+        const smtpConnectionConfig = await smtpConfigBuilder.buildConnectionConfig({
+            gatewayData,
+            accountData,
+            loadOAuth2Credentials: this.loadOAuth2AccountCredentials.bind(this),
+            context: this
+        });
 
-        // Configure TLS with defaults
-        if (!smtpSettings.tls) {
-            smtpSettings.tls = {};
-        }
-        for (let key of Object.keys(TLS_DEFAULTS)) {
-            if (!(key in smtpSettings.tls)) {
-                smtpSettings.tls[key] = TLS_DEFAULTS[key];
-            }
-        }
+        // Resolve auth server credentials if configured
+        const smtpAuth = await smtpConfigBuilder.resolveAuthServer(smtpConnectionConfig);
 
-        // Set up logger wrapper
-        for (let level of ['trace', 'debug', 'info', 'warn', 'error', 'fatal']) {
-            smtpLogger[level] = (data, message, ...args) => {
-                if (args && args.length) {
-                    message = util.format(message, ...args);
-                }
-                data.msg = message;
-                data.sub = 'nodemailer';
-                if (typeof this.logger[level] === 'function') {
-                    this.logger[level](data);
-                } else {
-                    this.logger.debug(data);
-                }
-            };
-        }
+        // Build complete SMTP settings
+        const smtpSettings = await smtpConfigBuilder.buildSmtpSettings({
+            smtpConnectionConfig,
+            smtpAuth,
+            accountData,
+            data
+        });
 
-        // set up proxy if needed
-        if (data.proxy) {
-            smtpSettings.proxy = data.proxy;
-        } else if (accountData.proxy) {
-            smtpSettings.proxy = accountData.proxy;
-        } else {
-            let proxyUrl = await settings.get('proxyUrl');
-            let proxyEnabled = await settings.get('proxyEnabled');
-            if (proxyEnabled && proxyUrl && !smtpSettings.proxy) {
-                smtpSettings.proxy = proxyUrl;
-            }
-        }
+        // Build network routing info for notifications
+        const networkRouting = NetworkRoutingBuilder.build(smtpSettings, data);
 
-        // Override EHLO hostname if configured
-        if (accountData.smtpEhloName) {
-            smtpSettings.name = accountData.smtpEhloName;
-        }
+        // Extract message data
+        let { raw, hasBcc, envelope, messageId, queueId, reference, job: jobData } = data;
 
         // Verify job still exists
         const submitJobEntry = await this.submitQueue.getJob(jobData.id);
         if (!submitJobEntry) {
-            // already failed?
-            this.logger.error({
-                msg: 'Submit job was not found',
-                job: jobData.id
-            });
+            this.logger.error({ msg: 'Submit job was not found', job: jobData.id });
             return false;
         }
 
-        // Handle certificate errors if configured
-        const ignoreMailCertErrors = await settings.get('ignoreMailCertErrors');
-        if (ignoreMailCertErrors && smtpSettings?.tls?.rejectUnauthorized !== false) {
-            smtpSettings.tls = smtpSettings.tls || {};
-            smtpSettings.tls.rejectUnauthorized = false;
-        }
-
-        // Prepare network routing info for notifications
-        const networkRouting = smtpSettings.localAddress || smtpSettings.proxy ? {} : null;
-
-        if (networkRouting && smtpSettings.localAddress) {
-            networkRouting.localAddress = smtpSettings.localAddress;
-        }
-
-        if (networkRouting && smtpSettings.proxy) {
-            networkRouting.proxy = smtpSettings.proxy;
-        }
-
-        if (networkRouting && smtpSettings.name) {
-            networkRouting.name = smtpSettings.name;
-        }
-
-        if (networkRouting && data.localAddress && data.localAddress !== networkRouting.localAddress) {
-            networkRouting.requestedLocalAddress = data.localAddress;
-        }
-
-        // Get or create SMTP transport from pool
+        // Get SMTP transport from pool
         const transporter = getMailTransport(smtpSettings);
 
         try {
+            // Update job progress
             try {
-                // try to update job progress
-                await submitJobEntry.updateProgress({
-                    status: 'smtp-starting'
-                });
+                await submitJobEntry.updateProgress({ status: 'smtp-starting' });
             } catch (err) {
                 // ignore
             }
@@ -3388,7 +2972,6 @@ class BaseClient {
             const info = await transporter.sendMail({
                 envelope,
                 messageId,
-                // make sure that Bcc line is removed from the version sent to SMTP
                 raw: !hasBcc ? raw : await removeBcc(raw),
                 dsn: data.dsn || null
             });
@@ -3398,40 +2981,13 @@ class BaseClient {
                 await this.redis.hSetExists(this.getAccountKey(), 'smtpServerEhlo', JSON.stringify(info.ehlo));
             }
 
-            // special rules for MTA servers
-
-            let originalMessageId;
-
-            // Hotmail - extract actual Message-ID from response
-            let hotmailMessageIdMatch = (info.response || '').toString().match(/^250 2.0.0 OK (<[^>]+\.prod\.outlook\.com>)/);
-            if (hotmailMessageIdMatch && hotmailMessageIdMatch[1] !== info.messageId) {
-                // MessageId was overridden
-                originalMessageId = info.messageId;
-                info.messageId = hotmailMessageIdMatch[1];
-            }
-
-            // AWS SES - construct Message-ID from response
-            let awsSesHostMatch = (smtpSettings.host || '').toString().match(/\.([^.]+)\.(amazonaws\.com|awsapps\.com)$/i);
-            let awsSesMessageIdMatch = (info.response || '').toString().match(/^250 Ok ([0-9a-f-]+)$/);
-            if (awsSesHostMatch && awsSesMessageIdMatch) {
-                let region = awsSesHostMatch[1].toLowerCase().trim();
-                let messageId = awsSesMessageIdMatch[1].toLowerCase().trim();
-                if (region === 'us-east-1') {
-                    region = 'email';
-                }
-
-                // MessageId was overridden
-                originalMessageId = info.messageId;
-                info.messageId = '<' + messageId + (!/@/.test(messageId) ? '@' + region + '.amazonses.com' : '') + '>';
-            }
-
-            // done
+            // Handle provider-specific message ID extraction
+            const originalMessageId = ProviderMessageIdHandler.processResponse(info, smtpSettings.host);
 
+            // Update job progress
             try {
-                // try to update job progress
                 await submitJobEntry.updateProgress({
                     status: 'smtp-completed',
-
                     response: info.response,
                     messageId: info.messageId,
                     originalMessageId
@@ -3441,269 +2997,223 @@ class BaseClient {
             }
 
             // Send success notification
-            await this.notify(false, EMAIL_SENT_NOTIFY, {
-                messageId: info.messageId,
-                originalMessageId,
-                response: info.response,
-                queueId,
-                envelope,
-                networkRouting
-            });
+            await this.notify(
+                false,
+                EMAIL_SENT_NOTIFY,
+                NotificationBuilder.buildSuccessPayload({ info, originalMessageId, queueId, envelope, networkRouting })
+            );
 
-            // clean up possible cached SMTP error
+            // Clear cached SMTP error
             try {
-                await this.redis.hset(
-                    this.getAccountKey(),
-                    'smtpStatus',
-                    JSON.stringify({
-                        created: Date.now(),
-                        status: 'ok',
-                        response: info.response
-                    })
-                );
+                await this.redis.hset(this.getAccountKey(), 'smtpStatus', JSON.stringify({ created: Date.now(), status: 'ok', response: info.response }));
             } catch (err) {
-                // ignore?
-            }
-
-            // The default is to copy message to Sent Mail folder
-            let shouldCopy = !Object.prototype.hasOwnProperty.call(accountData, 'copy');
-
-            // Account specific setting
-            if (typeof accountData.copy === 'boolean') {
-                shouldCopy = accountData.copy;
-            }
-
-            // Suppress uploads for Gmail and Outlook
-            // Unfortunately, previous default schema for all added accounts was copy=true, so can't prefer account specific setting here
-
-            // Emails for delegated accounts will be uploaded as the sender is different.
-            // SMTP is disabled for shared mailboxes, so we need to send using the main account.
-            let skipIfOutlook = this.isOutlook && (!accountData.oauth2 || !accountData.oauth2.auth || !accountData.oauth2.auth.delegatedUser);
-
-            if ((this.isGmail || skipIfOutlook) && !gatewayData) {
-                shouldCopy = false;
-            }
-
-            // Message specific setting, overrides all other settings
-            if (typeof data.copy === 'boolean') {
-                shouldCopy = data.copy;
+                // ignore
             }
 
-            // Check if IMAP is available
-            if ((!accountData.imap && !accountData.oauth2) || (accountData.imap && accountData.imap.disabled)) {
-                // IMAP is disabled for this account
-                shouldCopy = false;
-            }
+            // Handle sent mail copy to folder
+            const shouldCopy = SentMailCopyDecider.shouldCopy({
+                accountData,
+                data,
+                isGmail: this.isGmail,
+                isOutlook: this.isOutlook,
+                gatewayData
+            });
 
-            let connectionOptions = { allowSecondary: true };
+            const connectionOptions = { allowSecondary: true };
 
             if (shouldCopy) {
-                // NB! IMAP only
-                // Upload the message to Sent Mail folder
-
-                try {
-                    this.checkIMAPConnection(connectionOptions);
-
-                    // Find or use specified sent folder
-                    let sentMailbox =
-                        data.sentMailPath && typeof data.sentMailPath === 'string'
-                            ? {
-                                  path: data.sentMailPath
-                              }
-                            : await this.getSpecialUseMailbox('\\Sent');
-
-                    if (sentMailbox) {
-                        if (raw.buffer) {
-                            // convert from a Uint8Array to a Buffer
-                            raw = Buffer.from(raw);
-                        }
-
-                        const connectionClient = await this.getImapConnection(connectionOptions, 'submitMessage');
-
-                        // Upload message with Seen flag
-                        await connectionClient.append(sentMailbox.path, raw, ['\\Seen']);
-
-                        // Return to IDLE if using primary connection
-                        if (connectionClient === this.imapClient && this.imapClient.mailbox && !this.imapClient.idling) {
-                            // force back to IDLE
-                            this.imapClient.idle().catch(err => {
-                                this.logger.error({ msg: 'IDLE error', err });
-                            });
-                        }
-                    }
-                } catch (err) {
-                    this.logger.error({ msg: 'Failed to upload Sent mail', queueId, messageId, err });
-                }
+                await this.uploadToSentFolder(raw, data, queueId, messageId, connectionOptions);
             }
 
-            // Add \Answered flag to referenced message if needed
+            // Update reference flags if needed
             if (reference && reference.update) {
-                try {
-                    this.checkIMAPConnection(connectionOptions);
-                    await this.updateMessage(
-                        reference.message,
-                        {
-                            flags: {
-                                add: ['\\Answered'].concat(reference.action === 'forward' ? '$Forwarded' : [])
-                            }
-                        },
-                        connectionOptions
-                    );
-                } catch (err) {
-                    this.logger.error({ msg: 'Failed to update reference flags', queueId, messageId, reference, err });
-                }
+                await this.updateReferenceFlags(reference, queueId, messageId, connectionOptions);
             }
 
             // Update feedback key if provided
             if (data.feedbackKey) {
-                await this.redis
-                    .multi()
-                    .hset(data.feedbackKey, 'success', 'true')
-                    .expire(1 * 60 * 60);
+                await this.updateFeedbackKey(data.feedbackKey, true);
             }
 
             // Update gateway usage stats
             if (gatewayData) {
-                try {
-                    await gatewayObject.update({
-                        lastError: null,
-                        lastUse: new Date(),
-                        deliveries: { inc: 1 }
-                    });
-                } catch (err) {
-                    this.logger.error({ msg: 'Failed to update gateway', queueId, messageId, reference, gateway: gatewayData.gateway, err });
-                }
+                await this.updateGatewayStats(gatewayObject, queueId, messageId, reference);
             }
 
-            return {
-                response: info.response,
-                messageId: info.messageId
-            };
+            return { response: info.response, messageId: info.messageId };
         } catch (err) {
-            // Handle permanent failures
-            if (err.responseCode >= 500 && jobData.opts?.attempts <= jobData.attemptsMade) {
-                jobData.nextAttempt = false;
-            }
-
-            // Track SMTP errors for diagnostics
-            let smtpStatus = false;
-            switch (err.code) {
-                case 'ESOCKET':
-                    if (err.cert && err.reason) {
-                        smtpStatus = {
-                            description: `Certificate check for ${smtpSettings.host}:${smtpSettings.port} failed. ${err.reason}`
-                        };
-                    }
-                    break;
-                case 'EMESSAGE':
-                case 'ESTREAM':
-                case 'EENVELOPE':
-                    // Ignore. Too generic or message related
-                    break;
-                case 'ETIMEDOUT':
-                    // firewall?
-                    smtpStatus = {
-                        description: `Request timed out. Possibly a firewall issue or a wrong hostname/port (${smtpSettings.host}:${smtpSettings.port}).`
-                    };
-                    break;
-                case 'ETLS':
-                    smtpStatus = {
-                        description: `EmailEngine failed to set up TLS session with ${smtpSettings.host}:${smtpSettings.port}`
-                    };
-                    break;
-                case 'EDNS':
-                    smtpStatus = {
-                        description: `EmailEngine failed to resolve DNS record for ${smtpSettings.host}`
-                    };
-                    break;
-                case 'ECONNECTION':
-                    smtpStatus = {
-                        description: `EmailEngine failed to establish TCP connection against ${smtpSettings.host}`
-                    };
-                    break;
-                case 'EPROTOCOL':
-                    smtpStatus = {
-                        description: `Unexpected response from ${smtpSettings.host}`
-                    };
-                    break;
-                case 'EAUTH':
-                    smtpStatus = {
-                        description: `Authentication failed`
-                    };
-                    break;
-            }
+            await this.handleSubmitError(err, {
+                smtpSettings,
+                networkRouting,
+                gatewayData,
+                gatewayObject,
+                data,
+                jobData,
+                queueId,
+                envelope
+            });
+            throw err;
+        }
+    }
 
-            if (smtpStatus) {
-                let lastError = Object.assign(
-                    {
-                        created: Date.now(),
-                        status: 'error',
-                        response: err.response,
-                        responseCode: err.responseCode,
-                        code: err.code,
-                        command: err.command,
-                        networkRouting
-                    },
-                    smtpStatus
-                );
+    /**
+     * Uploads sent message to the Sent folder via IMAP
+     * @param {Buffer} raw - Raw message content
+     * @param {Object} data - Message data
+     * @param {string} queueId - Queue ID for logging
+     * @param {string} messageId - Message ID for logging
+     * @param {Object} connectionOptions - IMAP connection options
+     */
+    async uploadToSentFolder(raw, data, queueId, messageId, connectionOptions) {
+        try {
+            this.checkIMAPConnection(connectionOptions);
 
-                // store SMTP error for the account
-                try {
-                    await this.redis.hset(this.getAccountKey(), 'smtpStatus', JSON.stringify(lastError));
-                } catch (err) {
-                    // ignore?
+            const sentMailbox =
+                data.sentMailPath && typeof data.sentMailPath === 'string' ? { path: data.sentMailPath } : await this.getSpecialUseMailbox('\\Sent');
+
+            if (sentMailbox) {
+                if (raw.buffer) {
+                    raw = Buffer.from(raw);
                 }
 
-                // Update gateway error status
-                if (gatewayData) {
-                    try {
-                        await gatewayObject.update({
-                            lastError,
-                            lastUse: new Date()
-                        });
-                    } catch (err) {
-                        // ignore?
-                    }
+                const connectionClient = await this.getImapConnection(connectionOptions, 'submitMessage');
+                await connectionClient.append(sentMailbox.path, raw, ['\\Seen']);
+
+                if (connectionClient === this.imapClient && this.imapClient.mailbox && !this.imapClient.idling) {
+                    this.imapClient.idle().catch(err => {
+                        this.logger.error({ msg: 'IDLE error', err });
+                    });
                 }
             }
+        } catch (err) {
+            this.logger.error({ msg: 'Failed to upload Sent mail', queueId, messageId, err });
+        }
+    }
 
-            // Update feedback key with failure status
-            if (data.feedbackKey && !jobData.nextAttempt) {
-                await this.redis
-                    .multi()
-                    .hset(data.feedbackKey, 'success', 'false')
-                    .hset(data.feedbackKey, 'error', ((smtpStatus && smtpStatus.description) || '').toString() || 'Failed to send email')
-                    .expire(data.feedbackKey, 1 * 60 * 60)
-                    .exec();
-            }
+    /**
+     * Updates flags on the referenced message (for replies/forwards)
+     * @param {Object} reference - Reference message info
+     * @param {string} queueId - Queue ID for logging
+     * @param {string} messageId - Message ID for logging
+     * @param {Object} connectionOptions - IMAP connection options
+     */
+    async updateReferenceFlags(reference, queueId, messageId, connectionOptions) {
+        try {
+            this.checkIMAPConnection(connectionOptions);
+            await this.updateMessage(
+                reference.message,
+                {
+                    flags: {
+                        add: ['\\Answered'].concat(reference.action === 'forward' ? '$Forwarded' : [])
+                    }
+                },
+                connectionOptions
+            );
+        } catch (err) {
+            this.logger.error({ msg: 'Failed to update reference flags', queueId, messageId, reference, err });
+        }
+    }
+
+    /**
+     * Updates feedback key with success/failure status
+     * @param {string} feedbackKey - Redis key for feedback
+     * @param {boolean} success - Whether the operation succeeded
+     * @param {string} [errorMessage] - Error message if failed
+     */
+    async updateFeedbackKey(feedbackKey, success, errorMessage) {
+        const feedbackTxn = createTransaction(this.redis, { logger: this.logger });
+        feedbackTxn.add('hset', feedbackKey, 'success', success ? 'true' : 'false');
+        if (!success && errorMessage) {
+            feedbackTxn.add('hset', feedbackKey, 'error', errorMessage);
+        }
+        feedbackTxn.add('expire', feedbackKey, 1 * 60 * 60);
+        await feedbackTxn.exec();
+    }
 
-            // Send delivery error notification
-            await this.notify(false, EMAIL_DELIVERY_ERROR_NOTIFY, {
+    /**
+     * Updates gateway usage statistics after successful send
+     * @param {Object} gatewayObject - Gateway object
+     * @param {string} queueId - Queue ID for logging
+     * @param {string} messageId - Message ID for logging
+     * @param {Object} reference - Reference message info
+     */
+    async updateGatewayStats(gatewayObject, queueId, messageId, reference) {
+        try {
+            await gatewayObject.update({
+                lastError: null,
+                lastUse: new Date(),
+                deliveries: { inc: 1 }
+            });
+        } catch (err) {
+            this.logger.error({
+                msg: 'Failed to update gateway',
                 queueId,
-                envelope,
+                messageId,
+                reference,
+                gateway: gatewayObject.gateway,
+                err
+            });
+        }
+    }
 
-                messageId: data.messageId,
+    /**
+     * Handles errors during message submission
+     * @param {Error} err - The error that occurred
+     * @param {Object} context - Error handling context
+     */
+    async handleSubmitError(err, context) {
+        const { smtpSettings, networkRouting, gatewayData, gatewayObject, data, jobData, queueId, envelope } = context;
 
-                error: err.message,
-                errorCode: err.code,
+        // Handle permanent failures
+        if (err.responseCode >= 500 && jobData.opts?.attempts <= jobData.attemptsMade) {
+            jobData.nextAttempt = false;
+        }
 
-                smtpResponse: err.response,
-                smtpResponseCode: err.responseCode,
-                smtpCommand: err.command,
+        // Build SMTP status from error
+        const smtpStatus = SmtpErrorBuilder.buildStatus(err, smtpSettings, networkRouting);
 
-                networkRouting,
+        if (smtpStatus) {
+            // Store SMTP error for the account
+            try {
+                await this.redis.hset(this.getAccountKey(), 'smtpStatus', JSON.stringify(smtpStatus));
+            } catch (storeErr) {
+                // ignore
+            }
 
-                job: jobData
-            });
+            // Update gateway error status
+            if (gatewayData) {
+                try {
+                    await gatewayObject.update({ lastError: smtpStatus, lastUse: new Date() });
+                } catch (updateErr) {
+                    // ignore
+                }
+            }
+        }
 
-            // Enhance error with additional context
-            err.code = err.code || 'SubmitFail';
-            err.statusCode = Number(err.responseCode) || null;
+        // Update feedback key with failure status
+        if (data.feedbackKey && !jobData.nextAttempt) {
+            const errorMessage = (smtpStatus && smtpStatus.description) || 'Failed to send email';
+            await this.updateFeedbackKey(data.feedbackKey, false, errorMessage);
+        }
 
-            err.info = { networkRouting };
+        // Send delivery error notification
+        await this.notify(
+            false,
+            EMAIL_DELIVERY_ERROR_NOTIFY,
+            NotificationBuilder.buildErrorPayload({
+                error: err,
+                queueId,
+                envelope,
+                messageId: data.messageId,
+                networkRouting,
+                jobData
+            })
+        );
 
-            throw err;
-        }
+        // Enhance error with additional context
+        err.code = err.code || 'SubmitFail';
+        err.statusCode = Number(err.responseCode) || null;
+        err.info = { networkRouting };
     }
 
     // stub - to be implemented by subclasses
@@ -3712,4 +3222,5 @@ class BaseClient {
     }
 }
 
-module.exports = { BaseClient, metricsMeta };
+// Re-export postMetrics from notification-handler for backward compatibility
+module.exports = { BaseClient, metricsMeta: postMetrics };