emailengine-app 2.70.0 → 2.72.0

package/workers/export.js

@@ -20,7 +20,7 @@ const {
 const { getDuration, readEnvValue, threadStats, maybeReloadHttpProxyAgent } = require('../lib/tools');
 const { webhooks: Webhooks } = require('../lib/webhooks');
 const settings = require('../lib/settings');
-const { Export, isTransientError, isSkippableError, isFolderMissingError } = require('../lib/export');
+const { Export, isTransientError, isSkippableError, isFolderMissingError, isRetryableError } = require('../lib/export');
 
 const { initSentry } = require('../lib/sentry');
 initSentry('export');
@@ -192,10 +192,9 @@ async function indexMessages(job, exportData) {
         }
 
         const folderPath = foldersToProcess[i];
-        let retries = FOLDER_INDEX_MAX_RETRIES;
-        let lastError = null;
+        let attempt = 0;
 
-        while (retries > 0) {
+        while (true) {
             try {
                 const remaining = maxMessages ? maxMessages - totalIndexed : 0;
                 const queued = await indexFolder(accountObject, account, exportId, folderPath, startDate, endDate, indexingStartTime, remaining);
@@ -211,40 +210,43 @@ async function indexMessages(job, exportData) {
                     totalIndexed
                 });
 
-                lastError = null;
                 break;
             } catch (err) {
-                lastError = err;
-                retries--;
-                if (retries > 0) {
-                    const attemptNumber = FOLDER_INDEX_MAX_RETRIES - retries;
-                    const delay = FOLDER_INDEX_RETRY_DELAY_MS * Math.pow(2, attemptNumber - 1);
-                    logger.warn({
-                        msg: 'Folder indexing failed, retrying',
+                attempt++;
+                // A folder that was deleted mid-export is handled inside indexFolder (treated as empty),
+                // so any error reaching here is a real failure. Only transient errors are worth retrying;
+                // a permanent error (deleted account, auth failure, unexpected server response) is not
+                // specific to this folder and would repeat for the rest, so fail the whole export. Silently
+                // skipping folders and marking an incomplete export "completed" hides data loss from the caller.
+                if (!isTransientError(err) || attempt >= FOLDER_INDEX_MAX_RETRIES) {
+                    logger.error({
+                        msg: 'Failed to index folder, failing export',
                         account,
                         exportId,
                         folder: folderPath,
-                        retriesLeft: retries,
-                        delayMs: delay,
+                        attempts: attempt,
                         err
                     });
-                    await new Promise(resolve => setTimeout(resolve, delay));
+                    err.message = `Failed to index folder "${folderPath}": ${err.message}`;
+                    throw err;
                 }
-            }
-        }
 
-        if (lastError) {
-            logger.warn({
-                msg: 'Failed to index folder after retries',
-                account,
-                exportId,
-                folder: folderPath,
-                maxRetries: FOLDER_INDEX_MAX_RETRIES,
-                err: lastError
-            });
+                const delay = FOLDER_INDEX_RETRY_DELAY_MS * Math.pow(2, attempt - 1);
+                logger.warn({
+                    msg: 'Folder indexing failed, retrying',
+                    account,
+                    exportId,
+                    folder: folderPath,
+                    attempt,
+                    maxRetries: FOLDER_INDEX_MAX_RETRIES,
+                    delayMs: delay,
+                    err
+                });
+                await new Promise(resolve => setTimeout(resolve, delay));
+            }
         }
 
-        // Count the folder as scanned whether or not it succeeded so progress reaches foldersTotal.
+        // The folder indexed successfully (failures throw above); count it toward progress.
         await Export.update(account, exportId, { foldersScanned: i + 1 });
 
         if (maxMessages && totalIndexed >= maxMessages) {
@@ -420,14 +422,14 @@ async function exportMessages(job, exportData) {
 
     let lastAccountCheck = Date.now();
     // Refreshes the job lock and export key expiry on a fixed cadence; called from the main loop and
-    // from inside the rate-limit backoff so a batch stuck retrying does not let the lock lapse.
+    // from inside the retry backoff so a batch stuck retrying does not let the lock lapse.
     const maybeExtendLease = createLeaseExtender(job, account, exportId);
 
     const accountData = await accountObject.loadAccountData(account);
     const isApiAccount = await accountObject.isApiClient(accountData);
     const MESSAGE_FETCH_BATCH_SIZE = 10; // Batch size for parallel message fetching
-    const MAX_RATE_LIMIT_RETRIES = 5; // Max retries for rate-limited messages
-    const RATE_LIMIT_BASE_DELAY = 5000; // Base delay for rate limit backoff (5 seconds)
+    const MAX_BATCH_RETRIES = 5; // Max retries for rate-limited or transient per-message errors within a batch
+    const BATCH_RETRY_BASE_DELAY = 5000; // Base delay for batch retry backoff (5 seconds)
 
     async function processMessage(message, entry) {
         message.path = entry.folder;
@@ -531,7 +533,7 @@ async function exportMessages(job, exportData) {
                     }
 
                     let fetchBatch = entriesToFetch.slice(i, i + MESSAGE_FETCH_BATCH_SIZE);
-                    let rateLimitRetry = 0;
+                    let batchRetry = 0;
 
                     while (fetchBatch.length > 0) {
                         const messageIds = fetchBatch.map(e => e.messageId);
@@ -542,14 +544,17 @@ async function exportMessages(job, exportData) {
                             resultMap.set(result.messageId, result);
                         }
 
-                        const rateLimitedEntries = [];
+                        const retryEntries = [];
 
                         for (const entry of fetchBatch) {
                             const result = resultMap.get(entry.messageId);
 
                             if (result && result.error) {
                                 const err = result.error;
-                                const isRateLimited = err.statusCode === 429 || err.code === 'rateLimitExceeded' || err.code === 'userRateLimitExceeded';
+                                // A single transient blip (rate limit, dropped batch response, network/5xx)
+                                // must not fail an entire multi-message export; only give up once the retry
+                                // budget is exhausted. See isRetryableError for the full classification.
+                                const isRetryable = isRetryableError(err);
 
                                 if (isSkippableError(err)) {
                                     logger.warn({
@@ -561,8 +566,8 @@ async function exportMessages(job, exportData) {
                                         reason: err.message || err.code
                                     });
                                     await Export.incrementSkipped(account, exportId);
-                                } else if (isRateLimited && rateLimitRetry < MAX_RATE_LIMIT_RETRIES) {
-                                    rateLimitedEntries.push(entry);
+                                } else if (isRetryable && batchRetry < MAX_BATCH_RETRIES) {
+                                    retryEntries.push(entry);
                                 } else {
                                     const error = new Error(err.message);
                                     error.code = err.code;
@@ -592,21 +597,21 @@ async function exportMessages(job, exportData) {
                             break;
                         }
 
-                        if (rateLimitedEntries.length > 0) {
-                            rateLimitRetry++;
-                            const delay = RATE_LIMIT_BASE_DELAY * Math.pow(2, rateLimitRetry - 1) + Math.random() * 1000;
+                        if (retryEntries.length > 0) {
+                            batchRetry++;
+                            const delay = BATCH_RETRY_BASE_DELAY * Math.pow(2, batchRetry - 1) + Math.random() * 1000;
                             logger.warn({
-                                msg: 'Rate limited during export, retrying batch',
+                                msg: 'Retrying failed messages during export batch',
                                 account,
                                 exportId,
-                                rateLimitedCount: rateLimitedEntries.length,
-                                attempt: rateLimitRetry,
-                                maxAttempts: MAX_RATE_LIMIT_RETRIES,
+                                retryCount: retryEntries.length,
+                                attempt: batchRetry,
+                                maxAttempts: MAX_BATCH_RETRIES,
                                 delayMs: Math.round(delay)
                             });
                             await new Promise(resolve => setTimeout(resolve, delay));
                             await maybeExtendLease();
-                            fetchBatch = rateLimitedEntries;
+                            fetchBatch = retryEntries;
                         } else {
                             break;
                         }
@@ -807,6 +812,11 @@ const exportWorker = new Worker(
         lockDuration: 10 * 60 * 1000,
         stalledInterval: 2 * 60 * 1000,
         maxStalledCount: 5,
+        // Do not start consuming jobs at construction. Startup recovery (markInterruptedAsFailed +
+        // cleanup) must finish first, otherwise the worker can pick up a queued export and start
+        // processing it while recovery concurrently marks that same export failed and deletes its
+        // file. run() is called from the startup IIFE once recovery completes.
+        autorun: false,
         ...queueConf
     }
 );
@@ -857,6 +867,11 @@ function onCommand(command) {
         logger.error({ msg: 'Failed to clean up export files', err });
     }
 
+    // Now that interrupted exports have been reconciled and orphaned files cleaned, start consuming
+    // jobs. Mirrors BullMQ's own autorun: a fatal run() failure is surfaced as an 'error' event,
+    // which (no listener) crashes the worker thread so the main process can restart it.
+    exportWorker.run().catch(error => exportWorker.emit('error', error));
+
     setInterval(() => {
         try {
             parentPort.postMessage({ cmd: 'heartbeat' });

package/workers/smtp.js

@@ -7,7 +7,7 @@ const config = require('@zone-eu/wild-config');
 const logger = require('../lib/logger');
 
 const { getDuration, emitChangeEvent, readEnvValue, threadStats, loadTlsConfig, getByteSize } = require('../lib/tools');
-const { matchIp } = require('../lib/utils/network');
+const { createSmtpAuthHandler } = require('../lib/smtp-auth');
 
 const { initSentry } = require('../lib/sentry');
 initSentry('smtp');
@@ -20,7 +20,6 @@ const getSecret = require('../lib/get-secret');
 const { Splitter, Joiner } = require('@zone-eu/mailsplit');
 const { HeadersRewriter } = require('../lib/headers-rewriter');
 const settings = require('../lib/settings');
-const tokens = require('../lib/tokens');
 
 const { encrypt, decrypt } = require('../lib/encrypt');
 const { Certs } = require('@postalsys/certs');
@@ -109,89 +108,10 @@ for (let level of ['trace', 'debug', 'info', 'warn', 'error', 'fatal']) {
     };
 }
 
-async function onAuth(auth, session) {
-    if (!session.eeAuthEnabled) {
-        throw new Error('Authentication not enabled');
-    }
-
-    let account = auth.username;
-
-    let smtpPassword = await settings.get('smtpServerPassword');
-    let authPass = false;
-
-    if (!smtpPassword || auth.password !== smtpPassword) {
-        if (/^[0-9a-f]{64}$/i.test(auth.password)) {
-            // fallback to tokens
-            let tokenData;
-            try {
-                tokenData = await tokens.get(auth.password, false, { log: true, remoteAddress: session.remoteAddress });
-            } catch (err) {
-                // ignore?
-            }
-
-            if (tokenData) {
-                if (tokenData.account && tokenData.account !== auth.username) {
-                    throw new Error('Access denied, invalid username');
-                }
-
-                if (tokenData.scopes && !tokenData.scopes.includes('smtp') && !tokenData.scopes.includes('*')) {
-                    logger.error({
-                        msg: 'Trying to use invalid scope for a token',
-                        tokenAccount: tokenData.account,
-                        tokenId: tokenData.id,
-                        account,
-                        requestedScope: 'smtp',
-                        scopes: tokenData.scopes
-                    });
-
-                    throw new Error('Access denied, invalid scope');
-                }
-
-                if (tokenData.restrictions && tokenData.restrictions.addresses && !matchIp(session.remoteAddress, tokenData.restrictions.addresses)) {
-                    logger.error({
-                        msg: 'Trying to use invalid IP for a token',
-                        tokenAccount: tokenData.account,
-                        tokenId: tokenData.id,
-                        account,
-                        remoteAddress: session.remoteAddress,
-                        addressAllowlist: tokenData.restrictions.addresses
-                    });
-
-                    throw new Error('Access denied, traffic not accepted from this IP');
-                }
-
-                authPass = true;
-            }
-        }
-
-        if (!authPass) {
-            throw new Error('Failed to authenticate user');
-        }
-    }
-
-    let accountObject = new Account({ account, redis, call, secret: await getSecret() });
-    let accountData;
-    try {
-        accountData = await accountObject.loadAccountData();
-    } catch (err) {
-        let respErr = new Error('Failed to authenticate user');
-
-        if (!err.output || err.output.statusCode !== 404) {
-            // only log non-obvious errors
-            logger.error({ msg: 'Failed to load account data', account: auth.username, err });
-            respErr.statusCode = 454;
-        }
-
-        throw respErr;
-    }
-
-    if (!accountData) {
-        throw new Error('Failed to authenticate user');
-    }
-
-    ACCOUNT_CACHE.set(session, accountObject);
-    return { user: accountData.account };
-}
+// Authentication logic lives in lib/smtp-auth.js so it can be unit tested
+// without booting this worker. The shared ACCOUNT_CACHE and call() are injected
+// so onAuth caches the Account for later processing steps.
+const onAuth = createSmtpAuthHandler({ accountCache: ACCOUNT_CACHE, call });
 
 function processMessage(stream, session, meta) {
     meta = meta || {};

package/workers/submit.js

@@ -42,15 +42,7 @@ const SUBMIT_QC = (readEnvValue('EENGINE_SUBMIT_QC') && Number(readEnvValue('EEN
 
 const SUBMIT_DELAY = getDuration(readEnvValue('EENGINE_SUBMIT_DELAY') || config.submitDelay) || null;
 
-const NON_RETRYABLE_CODES = new Set([
-    'EAUTH', // authentication failed
-    'ENOAUTH', // no credentials provided
-    'EOAUTH2', // OAuth2 token failure
-    'ETLS', // TLS handshake failed
-    'EENVELOPE', // invalid sender/recipients
-    'EMESSAGE', // message content error
-    'EPROTOCOL' // SMTP protocol mismatch
-]);
+const { shouldDiscardJob } = require('../lib/delivery-error');
 
 let callQueue = new Map();
 let mids = 0;
@@ -288,9 +280,7 @@ const submitWorker = new Worker(
                 // ignore
             }
 
-            const isPermanentSmtp = err.statusCode >= 500 && err.statusCode !== 503;
-            const isPermanentCode = NON_RETRYABLE_CODES.has(err.code);
-            if ((isPermanentSmtp || isPermanentCode) && job.attemptsMade < job.opts.attempts) {
+            if (shouldDiscardJob(err, job)) {
                 try {
                     // do not retry after 5xx error (except 503 which is transient)
                     await job.discard();