emailengine-app 2.61.4 → 2.62.0

package/lib/email-client/outlook-client.js

@@ -30,7 +30,8 @@ const {
     OUTLOOK_RETRY_MAX_DELAY,
     MESSAGE_UPDATED_NOTIFY,
     MESSAGE_DELETED_NOTIFY,
-    MESSAGE_MISSING_NOTIFY
+    MESSAGE_MISSING_NOTIFY,
+    DEFAULT_OUTLOOK_EXPORT_BATCH_SIZE
 } = require('../consts');
 
 // Maximum number of operations in a single batch request to Microsoft Graph API
@@ -340,51 +341,66 @@ class OutlookClient extends BaseClient {
         }
 
         // Format mailbox data for API response
-        let mailboxes = mailboxListing
-            .map(entry => {
-                let folderData = {
-                    id: entry.id,
-                    path: entry.pathName,
-                    delimiter: '/',
-                    parentPath: entry.parentPath,
-                    name: entry.displayName,
-                    listed: true,
-                    subscribed: true
+        let mailboxes = mailboxListing.map(entry => {
+            let folderData = {
+                id: entry.id,
+                path: entry.pathName,
+                delimiter: '/',
+                parentPath: entry.parentPath,
+                name: entry.displayName,
+                listed: true,
+                subscribed: true
+            };
+
+            if (entry.specialUse) {
+                folderData.specialUse = entry.specialUse;
+                folderData.specialUseSource = 'extension';
+            }
+
+            // Include message counts if requested
+            if (options?.statusQuery?.messages) {
+                folderData.status = {
+                    messages: entry.totalItemCount,
+                    unseen: entry.unreadItemCount
                 };
+            }
 
-                if (entry.specialUse) {
-                    folderData.specialUse = entry.specialUse;
-                    folderData.specialUseSource = 'extension';
-                }
+            return folderData;
+        });
 
-                // Include message counts if requested
-                if (options?.statusQuery?.messages) {
-                    folderData.status = {
-                        messages: entry.totalItemCount,
-                        unseen: entry.unreadItemCount
-                    };
-                }
+        // Add virtual "All Mail" folder for Outlook API
+        // This allows exporting all messages without scanning individual folders
+        mailboxes.unshift({
+            id: 'virtual_all',
+            path: '\\All',
+            delimiter: '/',
+            parentPath: '',
+            name: 'All Mail',
+            listed: true,
+            subscribed: false,
+            noSelect: true,
+            specialUse: '\\All',
+            specialUseSource: 'extension'
+        });
 
-                return folderData;
-            })
-            .sort((a, b) => {
-                // INBOX always comes first
-                if (a.path === 'INBOX' || a.specialUse === '\\Inbox') {
-                    return -1;
-                } else if (b.path === 'INBOX' || b.specialUse === '\\Inbox') {
-                    return 1;
-                }
+        mailboxes.sort((a, b) => {
+            // INBOX always comes first
+            if (a.path === 'INBOX' || a.specialUse === '\\Inbox') {
+                return -1;
+            } else if (b.path === 'INBOX' || b.specialUse === '\\Inbox') {
+                return 1;
+            }
 
-                // Special use folders come before regular folders
-                if (a.specialUse && !b.specialUse) {
-                    return -1;
-                } else if (!a.specialUse && b.specialUse) {
-                    return 1;
-                }
+            // Special use folders come before regular folders
+            if (a.specialUse && !b.specialUse) {
+                return -1;
+            } else if (!a.specialUse && b.specialUse) {
+                return 1;
+            }
 
-                // Alphabetical sorting for the rest
-                return a.path.toLowerCase().localeCompare(b.path.toLowerCase());
-            });
+            // Alphabetical sorting for the rest
+            return a.path.toLowerCase().localeCompare(b.path.toLowerCase());
+        });
 
         return mailboxes;
     }
@@ -395,6 +411,8 @@ class OutlookClient extends BaseClient {
      */
     async listMessages(query, options) {
         options = options || {};
+        // Support metadataOnly passed via query (from export worker)
+        options.metadataOnly = options.metadataOnly || query.metadataOnly;
 
         await this.prepare();
 
@@ -447,6 +465,53 @@ class OutlookClient extends BaseClient {
             }
         }
 
+        // Build select fields based on detail level
+        let selectFields;
+        let expandFields;
+
+        if (options.minimalFields) {
+            selectFields = ['id', 'conversationId', 'receivedDateTime'];
+        } else if (options.metadataOnly) {
+            selectFields = [
+                'id',
+                'conversationId',
+                'receivedDateTime',
+                'isRead',
+                'isDraft',
+                'flag',
+                'body',
+                'subject',
+                'from',
+                'replyTo',
+                'sender',
+                'internetMessageId'
+            ];
+        } else {
+            selectFields = [
+                'id',
+                'conversationId',
+                'receivedDateTime',
+                'isRead',
+                'isDraft',
+                'flag',
+                'body',
+                'subject',
+                'from',
+                'replyTo',
+                'sender',
+                'toRecipients',
+                'ccRecipients',
+                'bccRecipients',
+                'internetMessageId',
+                'bodyPreview'
+            ];
+            expandFields = 'attachments($select=id,name,contentType,size,isInline,microsoft.graph.fileAttachment/contentId)';
+        }
+
+        if (!folder) {
+            selectFields.push('parentFolderId');
+        }
+
         // Build Graph API query parameters
         let requestQuery = {
             $count: true,
@@ -454,30 +519,8 @@ class OutlookClient extends BaseClient {
             $skip: page * pageSize,
             $skiptoken,
             $orderBy: 'receivedDateTime desc',
-            $select: (options.metadataOnly
-                ? ['id', 'conversationId', 'receivedDateTime', 'isRead', 'isDraft', 'flag', 'body', 'subject', 'from', 'replyTo', 'sender', 'internetMessageId']
-                : [
-                      'id',
-                      'conversationId',
-                      'receivedDateTime',
-                      'isRead',
-                      'isDraft',
-                      'flag',
-                      'body',
-                      'subject',
-                      'from',
-                      'replyTo',
-                      'sender',
-                      'toRecipients',
-                      'ccRecipients',
-                      'bccRecipients',
-                      'internetMessageId',
-                      'bodyPreview'
-                  ]
-            )
-                .concat(!folder ? 'parentFolderId' : [])
-                .join(','),
-            $expand: options.metadataOnly ? undefined : 'attachments($select=id,name,contentType,size,isInline,microsoft.graph.fileAttachment/contentId)'
+            $select: selectFields.join(','),
+            $expand: expandFields
         };
 
         let useOutlookSearch = false;
@@ -523,7 +566,11 @@ class OutlookClient extends BaseClient {
 
             messages =
                 listing?.value?.map(messageData =>
-                    this.formatMessage(messageData, { path: quickResolveFolder(messageData.parentFolderId)?.pathName, showPath: !folder })
+                    this.formatMessage(messageData, {
+                        path: quickResolveFolder(messageData.parentFolderId)?.pathName,
+                        showPath: !folder,
+                        minimalFields: options.minimalFields
+                    })
                 ) || [];
         } catch (err) {
             this.logger.error({
@@ -1571,6 +1618,229 @@ class OutlookClient extends BaseClient {
         return formattedMessage;
     }
 
+    /**
+     * Fetches multiple messages using MS Graph batch API for efficient export operations
+     * Uses /$batch endpoint to send up to 20 message requests in a single HTTP call
+     * @param {string[]} emailIds - Array of message IDs
+     * @param {Object} options - Fetch options
+     * @returns {Object[]} Array of results with messageId, data, and error fields
+     */
+    async getMessages(emailIds, options) {
+        options = options || {};
+        await this.prepare();
+
+        // Pre-fetch folder listing once for all messages to avoid repeated lookups
+        const mailboxListing = await this.getMailboxListing();
+        const folderMap = new Map(mailboxListing.map(f => [f.id, f]));
+
+        const results = [];
+        let idGen = 0;
+
+        // Build select fields for batch request
+        const selectFields = [
+            'id',
+            'conversationId',
+            'receivedDateTime',
+            'isRead',
+            'isDraft',
+            'flag',
+            'body',
+            'subject',
+            'from',
+            'replyTo',
+            'sender',
+            'toRecipients',
+            'ccRecipients',
+            'bccRecipients',
+            'internetMessageId',
+            'bodyPreview',
+            'internetMessageHeaders',
+            'parentFolderId',
+            'categories'
+        ].join(',');
+
+        const expandFields = 'attachments($select=id,name,contentType,size,isInline,microsoft.graph.fileAttachment/contentId)';
+
+        // Build Prefer header for body content type
+        // Note: IdType="ImmutableId" is added by requestWithRetry automatically
+        let preferHeader = '';
+        if (options.textType === 'plain') {
+            preferHeader = 'outlook.body-content-type="text"';
+        } else if (options.textType === 'html') {
+            preferHeader = 'outlook.body-content-type="html"';
+        }
+
+        // Get configured batch size, capped at MS Graph API limit
+        const settingsBatchSize = await settings.get('outlookExportBatchSize');
+        const batchSize = Math.min(settingsBatchSize || DEFAULT_OUTLOOK_EXPORT_BATCH_SIZE, MAX_BATCH_SIZE);
+
+        for (let i = 0; i < emailIds.length; i += batchSize) {
+            const batch = emailIds.slice(i, i + batchSize);
+            const messageMap = new Map();
+
+            // Build batch request with up to MAX_BATCH_SIZE individual requests
+            const requests = batch.map(emailId => {
+                const reqId = `msg_${++idGen}`;
+                messageMap.set(reqId, emailId);
+
+                const request = {
+                    id: reqId,
+                    method: 'GET',
+                    url: `/${this.oauth2UserPath}/messages/${emailId}?$select=${selectFields}&$expand=${expandFields}`
+                };
+
+                // Add Prefer header for body content type if specified
+                if (preferHeader) {
+                    request.headers = { Prefer: preferHeader };
+                }
+
+                return request;
+            });
+
+            // Submit batch request - single HTTP call for up to 20 messages
+            // Retry on transient errors (429 rate limit, 5xx server errors)
+            const BATCH_MAX_RETRIES = 3;
+            const BATCH_RETRY_BASE_DELAY = 5000;
+
+            let responseData;
+            let batchSuccess = false;
+
+            for (let attempt = 0; attempt <= BATCH_MAX_RETRIES; attempt++) {
+                try {
+                    responseData = await this.request('/$batch', 'post', { requests });
+                    batchSuccess = true;
+                    break;
+                } catch (err) {
+                    const isRetryable = err.oauthRequest?.status === 429 || (err.oauthRequest?.status >= 500 && err.oauthRequest?.status < 600);
+
+                    if (!isRetryable || attempt === BATCH_MAX_RETRIES) {
+                        // Non-retryable error or max retries reached - return errors for all messages in this batch
+                        for (const emailId of batch) {
+                            results.push({
+                                messageId: emailId,
+                                data: null,
+                                error: { message: err.message, code: err.code, statusCode: err.oauthRequest?.status }
+                            });
+                        }
+                        break;
+                    }
+
+                    // Wait before retrying with exponential backoff + jitter
+                    const delay = BATCH_RETRY_BASE_DELAY * Math.pow(2, attempt) + Math.random() * 1000;
+                    this.logger.warn({
+                        msg: 'Batch request failed, retrying',
+                        account: this.account,
+                        attempt: attempt + 1,
+                        maxAttempts: BATCH_MAX_RETRIES + 1,
+                        statusCode: err.oauthRequest?.status,
+                        delayMs: Math.round(delay)
+                    });
+                    await new Promise(resolve => setTimeout(resolve, delay));
+                }
+            }
+
+            if (!batchSuccess) {
+                continue;
+            }
+
+            // Process batch responses - match by request ID and track which were received
+            const respondedIds = new Set();
+            for (const response of responseData?.responses || []) {
+                const emailId = messageMap.get(response.id);
+                if (!emailId) {
+                    continue;
+                }
+
+                respondedIds.add(response.id);
+
+                if (response.status >= 200 && response.status < 300 && response.body) {
+                    try {
+                        const messageData = response.body;
+
+                        // Resolve folder from pre-fetched map (O(1) lookup)
+                        let path, specialUse;
+                        if (messageData.parentFolderId) {
+                            const folder = folderMap.get(messageData.parentFolderId);
+                            if (folder) {
+                                path = folder.pathName;
+                                specialUse = folder.specialUse;
+                            }
+                        }
+
+                        // Process email headers
+                        if (messageData.internetMessageHeaders) {
+                            let headers = {};
+                            for (let header of messageData.internetMessageHeaders || []) {
+                                let { name, value } = header;
+                                name = (name || '').toString().trim().toLowerCase();
+                                value = (value || '').toString().trim();
+                                if (!(name in headers)) {
+                                    headers[name] = [];
+                                }
+                                if (!Array.isArray(headers[name])) {
+                                    continue;
+                                }
+                                headers[name].push(value);
+                                if (name === 'in-reply-to') {
+                                    messageData.inReplyTo = value;
+                                }
+                            }
+                            messageData.headers = headers;
+                        }
+
+                        const formattedMessage = this.formatMessage(messageData, {
+                            extended: true,
+                            path,
+                            textType: options.textType,
+                            showPath: options.showPath
+                        });
+
+                        if (specialUse) {
+                            formattedMessage.messageSpecialUse = specialUse;
+                        }
+
+                        results.push({
+                            messageId: emailId,
+                            data: formattedMessage,
+                            error: null
+                        });
+                    } catch (err) {
+                        results.push({
+                            messageId: emailId,
+                            data: null,
+                            error: { message: err.message, code: err.code }
+                        });
+                    }
+                } else {
+                    // Individual message error within the batch
+                    const graphError = response.body?.error;
+                    results.push({
+                        messageId: emailId,
+                        data: null,
+                        error: {
+                            message: graphError?.message || 'Request failed',
+                            code: graphError?.code,
+                            statusCode: response.status
+                        }
+                    });
+                }
+            }
+
+            // Reconcile: detect messages that got no response from the batch API
+            for (const [reqId, emailId] of messageMap) {
+                if (!respondedIds.has(reqId)) {
+                    results.push({
+                        messageId: emailId,
+                        data: null,
+                        error: { message: 'No response received from batch API', code: 'EMISSING_RESPONSE' }
+                    });
+                }
+            }
+        }
+
+        return results;
+    }
+
     /**
      * Get message text content only
      * More efficient than getMessage when only text is needed
@@ -1765,7 +2035,7 @@ class OutlookClient extends BaseClient {
                 case 'attachments': {
                     messageUploadObj.attachments = [];
                     let attachmentCounter = 0;
-                    for (let attachment of emailObject.attachments) {
+                    for (let attachment of emailObject.attachments || []) {
                         let attachmentEntry = {
                             '@odata.type': '#microsoft.graph.fileAttachment'
                         };
@@ -2088,7 +2358,8 @@ class OutlookClient extends BaseClient {
             await this.redis
                 .multi()
                 .hset(data.feedbackKey, 'success', 'true')
-                .expire(1 * 60 * 60);
+                .expire(data.feedbackKey, 1 * 60 * 60)
+                .exec();
         }
 
         return {
@@ -2963,7 +3234,7 @@ class OutlookClient extends BaseClient {
             try {
                 outlookSubscription = JSON.parse(outlookSubscription);
             } catch (err) {
-                // ignore, I guess?
+                outlookSubscription = {};
             }
         }
 
@@ -3092,13 +3363,35 @@ class OutlookClient extends BaseClient {
      * Handles all message properties and attachments
      */
     formatMessage(messageData, options) {
-        let { extended, path, textType, showPath } = options || {};
+        let { extended, path, textType, showPath, minimalFields } = options || {};
 
         let date = messageData.receivedDateTime ? new Date(messageData.receivedDateTime) : undefined;
         if (date?.toString() === 'Invalid Date') {
             date = undefined;
         }
 
+        // For minimalFields mode, return only essential fields for export indexing
+        if (minimalFields) {
+            // Derive a numeric uid from message ID for score tiebreaker
+            // Graph API doesn't have IMAP-style UIDs, so we hash the ID
+            let uid = 0;
+            if (messageData.id) {
+                for (let i = 0; i < messageData.id.length; i++) {
+                    uid = ((uid << 5) - uid + messageData.id.charCodeAt(i)) | 0;
+                }
+                uid = Math.abs(uid);
+            }
+
+            return {
+                id: messageData.id,
+                emailId: messageData.id,
+                threadId: messageData.conversationId || undefined,
+                date: date ? date.toISOString() : undefined,
+                uid,
+                size: 0 // Graph API doesn't provide size in minimal listing
+            };
+        }
+
         // Map Graph API flags to IMAP flags
         const flags = [];
         if (messageData.isRead) {

package/lib/email-client/smtp-pool-manager.js

@@ -54,7 +54,7 @@ function setupTransporterHandlers(transporter, poolKey) {
     // Handle transport errors by removing from pool
     transporter.once('error', err => {
         // Not sure what happened, but do not re-use this transporter object anymore
-        logger.error({ msg: 'Transporter failed', err });
+        logger.warn({ msg: 'Transporter failed', err });
         SMTP_POOLS.delete(poolKey);
         SMTP_POOL_LAST_USED.delete(poolKey);
         try {