emailengine-app 2.69.0 → 2.71.0

package/workers/documents.js

@@ -14,28 +14,8 @@ const GB_COLLECT_DELAY = 6 * 3600 * 1000; // 6h
 const GB_FAILURE_DELAY = 3 * 1000;
 const GB_EMPTY_DELAY = 10 * 1000;
 
-const Bugsnag = require('@bugsnag/js');
-if (readEnvValue('BUGSNAG_API_KEY')) {
-    Bugsnag.start({
-        apiKey: readEnvValue('BUGSNAG_API_KEY'),
-        appVersion: packageData.version,
-        logger: {
-            debug(...args) {
-                logger.debug({ msg: args.shift(), worker: 'documents', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            info(...args) {
-                logger.debug({ msg: args.shift(), worker: 'documents', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            warn(...args) {
-                logger.warn({ msg: args.shift(), worker: 'documents', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            error(...args) {
-                logger.error({ msg: args.shift(), worker: 'documents', source: 'bugsnag', args: args.length ? args : undefined });
-            }
-        }
-    });
-    logger.notifyError = Bugsnag.notify.bind(Bugsnag);
-}
+const { initSentry } = require('../lib/sentry');
+initSentry('documents');
 
 const { redis, queueConf } = require('../lib/db');
 const { Worker } = require('bullmq');

package/workers/export.js

@@ -20,30 +20,10 @@ const {
 const { getDuration, readEnvValue, threadStats, maybeReloadHttpProxyAgent } = require('../lib/tools');
 const { webhooks: Webhooks } = require('../lib/webhooks');
 const settings = require('../lib/settings');
-const { Export } = require('../lib/export');
-
-const Bugsnag = require('@bugsnag/js');
-if (readEnvValue('BUGSNAG_API_KEY')) {
-    Bugsnag.start({
-        apiKey: readEnvValue('BUGSNAG_API_KEY'),
-        appVersion: packageData.version,
-        logger: {
-            debug(...args) {
-                logger.debug({ msg: args.shift(), worker: 'export', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            info(...args) {
-                logger.debug({ msg: args.shift(), worker: 'export', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            warn(...args) {
-                logger.warn({ msg: args.shift(), worker: 'export', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            error(...args) {
-                logger.error({ msg: args.shift(), worker: 'export', source: 'bugsnag', args: args.length ? args : undefined });
-            }
-        }
-    });
-    logger.notifyError = Bugsnag.notify.bind(Bugsnag);
-}
+const { Export, isTransientError, isSkippableError, isFolderMissingError, isRetryableError } = require('../lib/export');
+
+const { initSentry } = require('../lib/sentry');
+initSentry('export');
 
 const { redis, queueConf } = require('../lib/db');
 const { Worker } = require('bullmq');
@@ -74,23 +54,6 @@ const IMAP_MESSAGE_RETRY_BASE_DELAY = 2000;
 const ACCOUNT_CHECK_INTERVAL = 60 * 1000;
 const LOCK_EXTENSION_INTERVAL = 5 * 60 * 1000;
 
-function isTransientError(err) {
-    if (['ETIMEDOUT', 'ECONNRESET', 'ENOTFOUND', 'EAI_AGAIN', 'ECONNREFUSED', 'EPIPE', 'EHOSTUNREACH'].includes(err.code)) {
-        return true;
-    }
-    if (err.statusCode >= 500 && err.statusCode < 600) {
-        return true;
-    }
-    if (err.code === 'Timeout' || err.message?.includes('timeout')) {
-        return true;
-    }
-    return false;
-}
-
-function isSkippableError(err) {
-    return err.code === 'MessageNotFound' || err.statusCode === 404 || err.message?.includes('Failed to generate message ID');
-}
-
 let callQueue = new Map();
 let mids = 0;
 
@@ -229,10 +192,9 @@ async function indexMessages(job, exportData) {
         }
 
         const folderPath = foldersToProcess[i];
-        let retries = FOLDER_INDEX_MAX_RETRIES;
-        let lastError = null;
+        let attempt = 0;
 
-        while (retries > 0) {
+        while (true) {
             try {
                 const remaining = maxMessages ? maxMessages - totalIndexed : 0;
                 const queued = await indexFolder(accountObject, account, exportId, folderPath, startDate, endDate, indexingStartTime, remaining);
@@ -248,40 +210,43 @@ async function indexMessages(job, exportData) {
                     totalIndexed
                 });
 
-                lastError = null;
                 break;
             } catch (err) {
-                lastError = err;
-                retries--;
-                if (retries > 0) {
-                    const attemptNumber = FOLDER_INDEX_MAX_RETRIES - retries;
-                    const delay = FOLDER_INDEX_RETRY_DELAY_MS * Math.pow(2, attemptNumber - 1);
-                    logger.warn({
-                        msg: 'Folder indexing failed, retrying',
+                attempt++;
+                // A folder that was deleted mid-export is handled inside indexFolder (treated as empty),
+                // so any error reaching here is a real failure. Only transient errors are worth retrying;
+                // a permanent error (deleted account, auth failure, unexpected server response) is not
+                // specific to this folder and would repeat for the rest, so fail the whole export. Silently
+                // skipping folders and marking an incomplete export "completed" hides data loss from the caller.
+                if (!isTransientError(err) || attempt >= FOLDER_INDEX_MAX_RETRIES) {
+                    logger.error({
+                        msg: 'Failed to index folder, failing export',
                         account,
                         exportId,
                         folder: folderPath,
-                        retriesLeft: retries,
-                        delayMs: delay,
+                        attempts: attempt,
                         err
                     });
-                    await new Promise(resolve => setTimeout(resolve, delay));
+                    err.message = `Failed to index folder "${folderPath}": ${err.message}`;
+                    throw err;
                 }
-            }
-        }
 
-        if (lastError) {
-            logger.warn({
-                msg: 'Failed to index folder after retries',
-                account,
-                exportId,
-                folder: folderPath,
-                maxRetries: FOLDER_INDEX_MAX_RETRIES,
-                err: lastError
-            });
+                const delay = FOLDER_INDEX_RETRY_DELAY_MS * Math.pow(2, attempt - 1);
+                logger.warn({
+                    msg: 'Folder indexing failed, retrying',
+                    account,
+                    exportId,
+                    folder: folderPath,
+                    attempt,
+                    maxRetries: FOLDER_INDEX_MAX_RETRIES,
+                    delayMs: delay,
+                    err
+                });
+                await new Promise(resolve => setTimeout(resolve, delay));
+            }
         }
 
-        // Count the folder as scanned whether or not it succeeded so progress reaches foldersTotal.
+        // The folder indexed successfully (failures throw above); count it toward progress.
         await Export.update(account, exportId, { foldersScanned: i + 1 });
 
         if (maxMessages && totalIndexed >= maxMessages) {
@@ -341,7 +306,18 @@ async function indexFolder(accountObject, account, exportId, folderPath, startDa
             cursor
         };
 
-        const result = await accountObject.listMessages(listOptions);
+        let result;
+        try {
+            result = await accountObject.listMessages(listOptions);
+        } catch (err) {
+            // Other errors (e.g. a deleted account) propagate
+            if (isFolderMissingError(err)) {
+                // The folder disappeared mid-export - treat it as empty instead of failing the export
+                logger.warn({ msg: 'Export folder was not found, skipping', account, exportId, folder: folderPath, err });
+                return queued;
+            }
+            throw err;
+        }
 
         for (const msg of result.messages || []) {
             if (maxMessages && queued >= maxMessages) {
@@ -446,14 +422,14 @@ async function exportMessages(job, exportData) {
 
     let lastAccountCheck = Date.now();
     // Refreshes the job lock and export key expiry on a fixed cadence; called from the main loop and
-    // from inside the rate-limit backoff so a batch stuck retrying does not let the lock lapse.
+    // from inside the retry backoff so a batch stuck retrying does not let the lock lapse.
     const maybeExtendLease = createLeaseExtender(job, account, exportId);
 
     const accountData = await accountObject.loadAccountData(account);
     const isApiAccount = await accountObject.isApiClient(accountData);
     const MESSAGE_FETCH_BATCH_SIZE = 10; // Batch size for parallel message fetching
-    const MAX_RATE_LIMIT_RETRIES = 5; // Max retries for rate-limited messages
-    const RATE_LIMIT_BASE_DELAY = 5000; // Base delay for rate limit backoff (5 seconds)
+    const MAX_BATCH_RETRIES = 5; // Max retries for rate-limited or transient per-message errors within a batch
+    const BATCH_RETRY_BASE_DELAY = 5000; // Base delay for batch retry backoff (5 seconds)
 
     async function processMessage(message, entry) {
         message.path = entry.folder;
@@ -557,7 +533,7 @@ async function exportMessages(job, exportData) {
                     }
 
                     let fetchBatch = entriesToFetch.slice(i, i + MESSAGE_FETCH_BATCH_SIZE);
-                    let rateLimitRetry = 0;
+                    let batchRetry = 0;
 
                     while (fetchBatch.length > 0) {
                         const messageIds = fetchBatch.map(e => e.messageId);
@@ -568,14 +544,17 @@ async function exportMessages(job, exportData) {
                             resultMap.set(result.messageId, result);
                         }
 
-                        const rateLimitedEntries = [];
+                        const retryEntries = [];
 
                         for (const entry of fetchBatch) {
                             const result = resultMap.get(entry.messageId);
 
                             if (result && result.error) {
                                 const err = result.error;
-                                const isRateLimited = err.statusCode === 429 || err.code === 'rateLimitExceeded' || err.code === 'userRateLimitExceeded';
+                                // A single transient blip (rate limit, dropped batch response, network/5xx)
+                                // must not fail an entire multi-message export; only give up once the retry
+                                // budget is exhausted. See isRetryableError for the full classification.
+                                const isRetryable = isRetryableError(err);
 
                                 if (isSkippableError(err)) {
                                     logger.warn({
@@ -587,8 +566,8 @@ async function exportMessages(job, exportData) {
                                         reason: err.message || err.code
                                     });
                                     await Export.incrementSkipped(account, exportId);
-                                } else if (isRateLimited && rateLimitRetry < MAX_RATE_LIMIT_RETRIES) {
-                                    rateLimitedEntries.push(entry);
+                                } else if (isRetryable && batchRetry < MAX_BATCH_RETRIES) {
+                                    retryEntries.push(entry);
                                 } else {
                                     const error = new Error(err.message);
                                     error.code = err.code;
@@ -618,21 +597,21 @@ async function exportMessages(job, exportData) {
                             break;
                         }
 
-                        if (rateLimitedEntries.length > 0) {
-                            rateLimitRetry++;
-                            const delay = RATE_LIMIT_BASE_DELAY * Math.pow(2, rateLimitRetry - 1) + Math.random() * 1000;
+                        if (retryEntries.length > 0) {
+                            batchRetry++;
+                            const delay = BATCH_RETRY_BASE_DELAY * Math.pow(2, batchRetry - 1) + Math.random() * 1000;
                             logger.warn({
-                                msg: 'Rate limited during export, retrying batch',
+                                msg: 'Retrying failed messages during export batch',
                                 account,
                                 exportId,
-                                rateLimitedCount: rateLimitedEntries.length,
-                                attempt: rateLimitRetry,
-                                maxAttempts: MAX_RATE_LIMIT_RETRIES,
+                                retryCount: retryEntries.length,
+                                attempt: batchRetry,
+                                maxAttempts: MAX_BATCH_RETRIES,
                                 delayMs: Math.round(delay)
                             });
                             await new Promise(resolve => setTimeout(resolve, delay));
                             await maybeExtendLease();
-                            fetchBatch = rateLimitedEntries;
+                            fetchBatch = retryEntries;
                         } else {
                             break;
                         }
@@ -833,6 +812,11 @@ const exportWorker = new Worker(
         lockDuration: 10 * 60 * 1000,
         stalledInterval: 2 * 60 * 1000,
         maxStalledCount: 5,
+        // Do not start consuming jobs at construction. Startup recovery (markInterruptedAsFailed +
+        // cleanup) must finish first, otherwise the worker can pick up a queued export and start
+        // processing it while recovery concurrently marks that same export failed and deletes its
+        // file. run() is called from the startup IIFE once recovery completes.
+        autorun: false,
         ...queueConf
     }
 );
@@ -883,6 +867,11 @@ function onCommand(command) {
         logger.error({ msg: 'Failed to clean up export files', err });
     }
 
+    // Now that interrupted exports have been reconciled and orphaned files cleaned, start consuming
+    // jobs. Mirrors BullMQ's own autorun: a fatal run() failure is surfaced as an 'error' event,
+    // which (no listener) crashes the worker thread so the main process can restart it.
+    exportWorker.run().catch(error => exportWorker.emit('error', error));
+
     setInterval(() => {
         try {
             parentPort.postMessage({ cmd: 'heartbeat' });
@@ -960,11 +949,3 @@ parentPort.on('message', message => {
 });
 
 logger.info({ msg: 'Started export worker thread', version: packageData.version });
-
-module.exports = {
-    isTransientError,
-    isSkippableError,
-    IMAP_MESSAGE_MAX_RETRIES,
-    IMAP_MESSAGE_RETRY_BASE_DELAY,
-    ACCOUNT_CHECK_INTERVAL
-};

package/workers/imap-proxy.js

@@ -5,32 +5,12 @@ const { parentPort } = require('worker_threads');
 const packageData = require('../package.json');
 const logger = require('../lib/logger');
 
-const { readEnvValue, threadStats } = require('../lib/tools');
+const { threadStats } = require('../lib/tools');
 
 const { run } = require('../lib/imapproxy/imap-server');
 
-const Bugsnag = require('@bugsnag/js');
-if (readEnvValue('BUGSNAG_API_KEY')) {
-    Bugsnag.start({
-        apiKey: readEnvValue('BUGSNAG_API_KEY'),
-        appVersion: packageData.version,
-        logger: {
-            debug(...args) {
-                logger.debug({ msg: args.shift(), worker: 'imapProxy', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            info(...args) {
-                logger.debug({ msg: args.shift(), worker: 'imapProxy', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            warn(...args) {
-                logger.warn({ msg: args.shift(), worker: 'imapProxy', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            error(...args) {
-                logger.error({ msg: args.shift(), worker: 'imapProxy', source: 'bugsnag', args: args.length ? args : undefined });
-            }
-        }
-    });
-    logger.notifyError = Bugsnag.notify.bind(Bugsnag);
-}
+const { initSentry } = require('../lib/sentry');
+initSentry('imapProxy');
 
 async function onCommand(command) {
     switch (command.cmd) {

package/workers/imap.js

@@ -9,28 +9,8 @@ const { REDIS_PREFIX } = require('../lib/consts');
 
 const { getDuration, getBoolean, emitChangeEvent, readEnvValue, hasEnvValue, threadStats, maybeReloadHttpProxyAgent } = require('../lib/tools');
 
-const Bugsnag = require('@bugsnag/js');
-if (readEnvValue('BUGSNAG_API_KEY')) {
-    Bugsnag.start({
-        apiKey: readEnvValue('BUGSNAG_API_KEY'),
-        appVersion: packageData.version,
-        logger: {
-            debug(...args) {
-                logger.debug({ msg: args.shift(), worker: 'imap', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            info(...args) {
-                logger.debug({ msg: args.shift(), worker: 'imap', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            warn(...args) {
-                logger.warn({ msg: args.shift(), worker: 'imap', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            error(...args) {
-                logger.error({ msg: args.shift(), worker: 'imap', source: 'bugsnag', args: args.length ? args : undefined });
-            }
-        }
-    });
-    logger.notifyError = Bugsnag.notify.bind(Bugsnag);
-}
+const { initSentry } = require('../lib/sentry');
+initSentry('imap');
 
 const { IMAPClient } = require('../lib/email-client/imap-client');
 const { GmailClient } = require('../lib/email-client/gmail-client');

package/workers/smtp.js

@@ -9,28 +9,8 @@ const logger = require('../lib/logger');
 const { getDuration, emitChangeEvent, readEnvValue, threadStats, loadTlsConfig, getByteSize } = require('../lib/tools');
 const { matchIp } = require('../lib/utils/network');
 
-const Bugsnag = require('@bugsnag/js');
-if (readEnvValue('BUGSNAG_API_KEY')) {
-    Bugsnag.start({
-        apiKey: readEnvValue('BUGSNAG_API_KEY'),
-        appVersion: packageData.version,
-        logger: {
-            debug(...args) {
-                logger.debug({ msg: args.shift(), worker: 'smtp', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            info(...args) {
-                logger.debug({ msg: args.shift(), worker: 'smtp', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            warn(...args) {
-                logger.warn({ msg: args.shift(), worker: 'smtp', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            error(...args) {
-                logger.error({ msg: args.shift(), worker: 'smtp', source: 'bugsnag', args: args.length ? args : undefined });
-            }
-        }
-    });
-    logger.notifyError = Bugsnag.notify.bind(Bugsnag);
-}
+const { initSentry } = require('../lib/sentry');
+initSentry('smtp');
 
 const { SMTPServer } = require('smtp-server');
 const util = require('util');

package/workers/submit.js

@@ -11,28 +11,8 @@ const { getDuration, readEnvValue, threadStats, maybeReloadHttpProxyAgent } = re
 const { webhooks: Webhooks } = require('../lib/webhooks');
 const settings = require('../lib/settings');
 
-const Bugsnag = require('@bugsnag/js');
-if (readEnvValue('BUGSNAG_API_KEY')) {
-    Bugsnag.start({
-        apiKey: readEnvValue('BUGSNAG_API_KEY'),
-        appVersion: packageData.version,
-        logger: {
-            debug(...args) {
-                logger.debug({ msg: args.shift(), worker: 'submit', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            info(...args) {
-                logger.debug({ msg: args.shift(), worker: 'submit', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            warn(...args) {
-                logger.warn({ msg: args.shift(), worker: 'submit', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            error(...args) {
-                logger.error({ msg: args.shift(), worker: 'submit', source: 'bugsnag', args: args.length ? args : undefined });
-            }
-        }
-    });
-    logger.notifyError = Bugsnag.notify.bind(Bugsnag);
-}
+const { initSentry } = require('../lib/sentry');
+initSentry('submit');
 
 const util = require('util');
 const { redis, queueConf, submitQueue } = require('../lib/db');
@@ -222,13 +202,15 @@ const submitWorker = new Worker(
             }
 
             let backoffDelay = Number(job.opts.backoff && job.opts.backoff.delay) || 0;
-            let nextAttempt = job.attemptsMade < job.opts.attempts ? Math.round(job.processedOn + Math.pow(2, job.attemptsMade) * backoffDelay) : false;
+            // job.attemptsMade is not yet incremented for the ongoing attempt, so the
+            // next retry (if this attempt fails) is delayed by 2^attemptsMade * base
+            let nextAttempt = job.attemptsMade + 1 < job.opts.attempts ? Math.round(job.processedOn + Math.pow(2, job.attemptsMade) * backoffDelay) : false;
 
             queueEntry.job = {
                 id: job.id,
                 attemptsMade: job.attemptsMade,
                 attempts: job.opts.attempts,
-                nextAttempt: new Date(nextAttempt).toISOString()
+                nextAttempt: nextAttempt ? new Date(nextAttempt).toISOString() : false
             };
 
             let res = await accountObject.submitMessage(queueEntry);

package/workers/webhooks.js

@@ -13,28 +13,8 @@ const { GooglePubSub } = require('../lib/oauth/pubsub/google');
 const { readEnvValue, threadStats, getDuration, httpAgent, getServiceSecret, maybeReloadHttpProxyAgent } = require('../lib/tools');
 const { sendWebhookRequest } = require('../lib/webhook-request');
 
-const Bugsnag = require('@bugsnag/js');
-if (readEnvValue('BUGSNAG_API_KEY')) {
-    Bugsnag.start({
-        apiKey: readEnvValue('BUGSNAG_API_KEY'),
-        appVersion: packageData.version,
-        logger: {
-            debug(...args) {
-                logger.debug({ msg: args.shift(), worker: 'webhooks', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            info(...args) {
-                logger.debug({ msg: args.shift(), worker: 'webhooks', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            warn(...args) {
-                logger.warn({ msg: args.shift(), worker: 'webhooks', source: 'bugsnag', args: args.length ? args : undefined });
-            },
-            error(...args) {
-                logger.error({ msg: args.shift(), worker: 'webhooks', source: 'bugsnag', args: args.length ? args : undefined });
-            }
-        }
-    });
-    logger.notifyError = Bugsnag.notify.bind(Bugsnag);
-}
+const { initSentry } = require('../lib/sentry');
+initSentry('webhooks');
 
 const { redis, queueConf } = require('../lib/db');
 const { Worker } = require('bullmq');