emailengine-app 2.70.0 → 2.71.0

package/workers/export.js

@@ -20,7 +20,7 @@ const {
 const { getDuration, readEnvValue, threadStats, maybeReloadHttpProxyAgent } = require('../lib/tools');
 const { webhooks: Webhooks } = require('../lib/webhooks');
 const settings = require('../lib/settings');
-const { Export, isTransientError, isSkippableError, isFolderMissingError } = require('../lib/export');
+const { Export, isTransientError, isSkippableError, isFolderMissingError, isRetryableError } = require('../lib/export');
 
 const { initSentry } = require('../lib/sentry');
 initSentry('export');
@@ -192,10 +192,9 @@ async function indexMessages(job, exportData) {
         }
 
         const folderPath = foldersToProcess[i];
-        let retries = FOLDER_INDEX_MAX_RETRIES;
-        let lastError = null;
+        let attempt = 0;
 
-        while (retries > 0) {
+        while (true) {
             try {
                 const remaining = maxMessages ? maxMessages - totalIndexed : 0;
                 const queued = await indexFolder(accountObject, account, exportId, folderPath, startDate, endDate, indexingStartTime, remaining);
@@ -211,40 +210,43 @@ async function indexMessages(job, exportData) {
                     totalIndexed
                 });
 
-                lastError = null;
                 break;
             } catch (err) {
-                lastError = err;
-                retries--;
-                if (retries > 0) {
-                    const attemptNumber = FOLDER_INDEX_MAX_RETRIES - retries;
-                    const delay = FOLDER_INDEX_RETRY_DELAY_MS * Math.pow(2, attemptNumber - 1);
-                    logger.warn({
-                        msg: 'Folder indexing failed, retrying',
+                attempt++;
+                // A folder that was deleted mid-export is handled inside indexFolder (treated as empty),
+                // so any error reaching here is a real failure. Only transient errors are worth retrying;
+                // a permanent error (deleted account, auth failure, unexpected server response) is not
+                // specific to this folder and would repeat for the rest, so fail the whole export. Silently
+                // skipping folders and marking an incomplete export "completed" hides data loss from the caller.
+                if (!isTransientError(err) || attempt >= FOLDER_INDEX_MAX_RETRIES) {
+                    logger.error({
+                        msg: 'Failed to index folder, failing export',
                         account,
                         exportId,
                         folder: folderPath,
-                        retriesLeft: retries,
-                        delayMs: delay,
+                        attempts: attempt,
                         err
                     });
-                    await new Promise(resolve => setTimeout(resolve, delay));
+                    err.message = `Failed to index folder "${folderPath}": ${err.message}`;
+                    throw err;
                 }
-            }
-        }
 
-        if (lastError) {
-            logger.warn({
-                msg: 'Failed to index folder after retries',
-                account,
-                exportId,
-                folder: folderPath,
-                maxRetries: FOLDER_INDEX_MAX_RETRIES,
-                err: lastError
-            });
+                const delay = FOLDER_INDEX_RETRY_DELAY_MS * Math.pow(2, attempt - 1);
+                logger.warn({
+                    msg: 'Folder indexing failed, retrying',
+                    account,
+                    exportId,
+                    folder: folderPath,
+                    attempt,
+                    maxRetries: FOLDER_INDEX_MAX_RETRIES,
+                    delayMs: delay,
+                    err
+                });
+                await new Promise(resolve => setTimeout(resolve, delay));
+            }
         }
 
-        // Count the folder as scanned whether or not it succeeded so progress reaches foldersTotal.
+        // The folder indexed successfully (failures throw above); count it toward progress.
         await Export.update(account, exportId, { foldersScanned: i + 1 });
 
         if (maxMessages && totalIndexed >= maxMessages) {
@@ -420,14 +422,14 @@ async function exportMessages(job, exportData) {
 
     let lastAccountCheck = Date.now();
     // Refreshes the job lock and export key expiry on a fixed cadence; called from the main loop and
-    // from inside the rate-limit backoff so a batch stuck retrying does not let the lock lapse.
+    // from inside the retry backoff so a batch stuck retrying does not let the lock lapse.
     const maybeExtendLease = createLeaseExtender(job, account, exportId);
 
     const accountData = await accountObject.loadAccountData(account);
     const isApiAccount = await accountObject.isApiClient(accountData);
     const MESSAGE_FETCH_BATCH_SIZE = 10; // Batch size for parallel message fetching
-    const MAX_RATE_LIMIT_RETRIES = 5; // Max retries for rate-limited messages
-    const RATE_LIMIT_BASE_DELAY = 5000; // Base delay for rate limit backoff (5 seconds)
+    const MAX_BATCH_RETRIES = 5; // Max retries for rate-limited or transient per-message errors within a batch
+    const BATCH_RETRY_BASE_DELAY = 5000; // Base delay for batch retry backoff (5 seconds)
 
     async function processMessage(message, entry) {
         message.path = entry.folder;
@@ -531,7 +533,7 @@ async function exportMessages(job, exportData) {
                     }
 
                     let fetchBatch = entriesToFetch.slice(i, i + MESSAGE_FETCH_BATCH_SIZE);
-                    let rateLimitRetry = 0;
+                    let batchRetry = 0;
 
                     while (fetchBatch.length > 0) {
                         const messageIds = fetchBatch.map(e => e.messageId);
@@ -542,14 +544,17 @@ async function exportMessages(job, exportData) {
                             resultMap.set(result.messageId, result);
                         }
 
-                        const rateLimitedEntries = [];
+                        const retryEntries = [];
 
                         for (const entry of fetchBatch) {
                             const result = resultMap.get(entry.messageId);
 
                             if (result && result.error) {
                                 const err = result.error;
-                                const isRateLimited = err.statusCode === 429 || err.code === 'rateLimitExceeded' || err.code === 'userRateLimitExceeded';
+                                // A single transient blip (rate limit, dropped batch response, network/5xx)
+                                // must not fail an entire multi-message export; only give up once the retry
+                                // budget is exhausted. See isRetryableError for the full classification.
+                                const isRetryable = isRetryableError(err);
 
                                 if (isSkippableError(err)) {
                                     logger.warn({
@@ -561,8 +566,8 @@ async function exportMessages(job, exportData) {
                                         reason: err.message || err.code
                                     });
                                     await Export.incrementSkipped(account, exportId);
-                                } else if (isRateLimited && rateLimitRetry < MAX_RATE_LIMIT_RETRIES) {
-                                    rateLimitedEntries.push(entry);
+                                } else if (isRetryable && batchRetry < MAX_BATCH_RETRIES) {
+                                    retryEntries.push(entry);
                                 } else {
                                     const error = new Error(err.message);
                                     error.code = err.code;
@@ -592,21 +597,21 @@ async function exportMessages(job, exportData) {
                             break;
                         }
 
-                        if (rateLimitedEntries.length > 0) {
-                            rateLimitRetry++;
-                            const delay = RATE_LIMIT_BASE_DELAY * Math.pow(2, rateLimitRetry - 1) + Math.random() * 1000;
+                        if (retryEntries.length > 0) {
+                            batchRetry++;
+                            const delay = BATCH_RETRY_BASE_DELAY * Math.pow(2, batchRetry - 1) + Math.random() * 1000;
                             logger.warn({
-                                msg: 'Rate limited during export, retrying batch',
+                                msg: 'Retrying failed messages during export batch',
                                 account,
                                 exportId,
-                                rateLimitedCount: rateLimitedEntries.length,
-                                attempt: rateLimitRetry,
-                                maxAttempts: MAX_RATE_LIMIT_RETRIES,
+                                retryCount: retryEntries.length,
+                                attempt: batchRetry,
+                                maxAttempts: MAX_BATCH_RETRIES,
                                 delayMs: Math.round(delay)
                             });
                             await new Promise(resolve => setTimeout(resolve, delay));
                             await maybeExtendLease();
-                            fetchBatch = rateLimitedEntries;
+                            fetchBatch = retryEntries;
                         } else {
                             break;
                         }
@@ -807,6 +812,11 @@ const exportWorker = new Worker(
         lockDuration: 10 * 60 * 1000,
         stalledInterval: 2 * 60 * 1000,
         maxStalledCount: 5,
+        // Do not start consuming jobs at construction. Startup recovery (markInterruptedAsFailed +
+        // cleanup) must finish first, otherwise the worker can pick up a queued export and start
+        // processing it while recovery concurrently marks that same export failed and deletes its
+        // file. run() is called from the startup IIFE once recovery completes.
+        autorun: false,
         ...queueConf
     }
 );
@@ -857,6 +867,11 @@ function onCommand(command) {
         logger.error({ msg: 'Failed to clean up export files', err });
     }
 
+    // Now that interrupted exports have been reconciled and orphaned files cleaned, start consuming
+    // jobs. Mirrors BullMQ's own autorun: a fatal run() failure is surfaced as an 'error' event,
+    // which (no listener) crashes the worker thread so the main process can restart it.
+    exportWorker.run().catch(error => exportWorker.emit('error', error));
+
     setInterval(() => {
         try {
             parentPort.postMessage({ cmd: 'heartbeat' });