emailengine-app 2.61.5 → 2.62.1

package/lib/email-client/gmail-client.js

@@ -11,6 +11,7 @@ const he = require('he');
 const { BaseClient, metricsMeta } = require('./base-client');
 const { mimeHtml } = require('@postalsys/email-text-tools');
 const { emitChangeEvent } = require('../tools');
+const crypto = require('crypto');
 const { Gateway } = require('../gateway');
 
 const {
@@ -20,11 +21,16 @@ const {
     EMAIL_SENT_NOTIFY,
     REDIS_PREFIX,
     AUTH_ERROR_NOTIFY,
-    AUTH_SUCCESS_NOTIFY
+    AUTH_SUCCESS_NOTIFY,
+    DEFAULT_GMAIL_EXPORT_BATCH_SIZE
 } = require('../consts');
 
+const settings = require('../settings');
+
 const { GMAIL_API_BASE, LIST_BATCH_SIZE, request: gmailApiRequest } = require('./gmail/gmail-api');
 
+const MAX_GMAIL_BATCH_SIZE = 50;
+
 // Labels to exclude from folder listings
 const SKIP_LABELS = ['UNREAD', 'STARRED', 'IMPORTANT', 'CHAT', 'CATEGORY_PERSONAL'];
 
@@ -602,23 +608,39 @@ class GmailClient extends BaseClient {
 
                 return folderData;
             })
-            .filter(value => value)
-            .sort((a, b) => {
-                // Sort: INBOX first, then special folders, then alphabetical
-                if (a.path === 'INBOX') {
-                    return -1;
-                } else if (b.path === 'INBOX') {
-                    return 1;
-                }
+            .filter(value => value);
+
+        // Add virtual "All Mail" folder for Gmail API
+        // This allows exporting all messages without scanning individual labels
+        mailboxes.unshift({
+            id: 'virtual_all',
+            path: '\\All',
+            delimiter: '/',
+            parentPath: '',
+            name: 'All Mail',
+            listed: true,
+            subscribed: false,
+            noSelect: true,
+            specialUse: '\\All',
+            specialUseSource: 'extension'
+        });
 
-                if (a.specialUse && !b.specialUse) {
-                    return -1;
-                } else if (!a.specialUse && b.specialUse) {
-                    return 1;
-                }
+        mailboxes.sort((a, b) => {
+            // Sort: INBOX first, then special folders, then alphabetical
+            if (a.path === 'INBOX') {
+                return -1;
+            } else if (b.path === 'INBOX') {
+                return 1;
+            }
 
-                return a.path.toLowerCase().localeCompare(b.path.toLowerCase());
-            });
+            if (a.specialUse && !b.specialUse) {
+                return -1;
+            } else if (!a.specialUse && b.specialUse) {
+                return 1;
+            }
+
+            return a.path.toLowerCase().localeCompare(b.path.toLowerCase());
+        });
 
         return mailboxes;
     }
@@ -658,6 +680,7 @@ class GmailClient extends BaseClient {
                 return false;
             }
             requestQuery.labelIds = [label.id];
+            path = query.path;
         }
 
         let messageList = [];
@@ -756,7 +779,7 @@ class GmailClient extends BaseClient {
         let nextPageCursor = pageCursor.nextPageCursor(listingResult.nextPageToken);
         let prevPageCursor = pageCursor.prevPageCursor();
 
-        if (options.metadataOnly) {
+        if (options.metadataOnly || query.metadataOnly) {
             // Return just IDs without fetching full content
             return {
                 total: messageCount,
@@ -769,6 +792,8 @@ class GmailClient extends BaseClient {
         }
 
         // Fetch message content for matching messages in batches
+        // Use format=minimal for minimalFields option (faster, returns only id, threadId, labelIds, internalDate, sizeEstimate)
+        const messageFormat = options.minimalFields ? 'minimal' : undefined;
 
         let promises = [];
 
@@ -782,14 +807,16 @@ class GmailClient extends BaseClient {
                     throw entry.reason;
                 }
                 if (entry.value) {
-                    messageList.push(this.formatMessage(entry.value, { path }));
+                    messageList.push(this.formatMessage(entry.value, { path, minimalFields: options.minimalFields }));
                 }
             }
             promises = [];
         };
 
         for (let { id: message } of listingResult.messages || []) {
-            promises.push(this.request(`${GMAIL_API_BASE}/gmail/v1/users/me/messages/${message}`));
+            let requestUrl = `${GMAIL_API_BASE}/gmail/v1/users/me/messages/${message}`;
+            let requestParams = messageFormat ? { format: messageFormat } : undefined;
+            promises.push(this.request(requestUrl, 'get', requestParams));
             if (promises.length > LIST_BATCH_SIZE) {
                 await resolvePromises();
             }
@@ -1027,13 +1054,11 @@ class GmailClient extends BaseClient {
                 messages = messages.concat(messageListResult?.messages);
                 if (messages.length >= maxMessages) {
                     messages = messages.slice(0, maxMessages);
-                    notDone = false;
                     break;
                 }
             }
 
             if (!messageListResult.nextPageCursor) {
-                notDone = false;
                 break;
             }
             cursor = messageListResult.nextPageCursor;
@@ -1269,6 +1294,9 @@ class GmailClient extends BaseClient {
 
         let formattedMessage = this.formatMessage(messageData, { extended: true, textType: options.textType });
 
+        // Resolve label IDs to human-readable names
+        await this.resolveLabels(formattedMessage);
+
         // Mark as seen if requested
         if (options.markAsSeen && (!formattedMessage.flags || !formattedMessage.flags.includes('\\Seen'))) {
             //
@@ -1325,6 +1353,64 @@ class GmailClient extends BaseClient {
         return formattedMessage;
     }
 
+    /**
+     * Fetches multiple messages in parallel for batch export operations
+     * @param {string[]} emailIds - Array of message IDs
+     * @param {Object} options - Fetch options
+     * @returns {Object[]} Array of results with messageId, data, and error fields
+     */
+    async getMessages(emailIds, options) {
+        options = options || {};
+        await this.prepare();
+
+        // Pre-fetch labels to resolve label IDs to names
+        const labelMap = new Map();
+        try {
+            const labelsResult = await this.getLabels();
+            for (const label of labelsResult || []) {
+                labelMap.set(label.id, label.name);
+            }
+        } catch (err) {
+            this.logger.warn({ msg: 'Failed to fetch labels for export, using raw label IDs', account: this.account, err });
+        }
+
+        const results = [];
+        const settingsBatchSize = await settings.get('gmailExportBatchSize');
+        const batchSize = Math.min(settingsBatchSize || DEFAULT_GMAIL_EXPORT_BATCH_SIZE, MAX_GMAIL_BATCH_SIZE);
+
+        for (let i = 0; i < emailIds.length; i += batchSize) {
+            const batch = emailIds.slice(i, i + batchSize);
+
+            const batchResults = await Promise.all(
+                batch.map(async emailId => {
+                    try {
+                        const requestQuery = { format: 'full' };
+                        const messageData = await this.request(`${GMAIL_API_BASE}/gmail/v1/users/me/messages/${emailId}`, 'get', requestQuery);
+                        const formattedMessage = this.formatMessage(messageData, { extended: true, textType: options.textType });
+
+                        await this.resolveLabels(formattedMessage, labelMap);
+
+                        return {
+                            messageId: emailId,
+                            data: formattedMessage,
+                            error: null
+                        };
+                    } catch (err) {
+                        return {
+                            messageId: emailId,
+                            data: null,
+                            error: { message: err.message, code: err.code, statusCode: err.statusCode }
+                        };
+                    }
+                })
+            );
+
+            results.push(...batchResults);
+        }
+
+        return results;
+    }
+
     /**
      * Fetches text content for a message
      * @param {string} textId - Encoded text identifier
@@ -1521,19 +1607,45 @@ class GmailClient extends BaseClient {
             return false;
         }
 
-        // Prepare for Gmail API send
-        let contentType = 'message/rfc822';
-        let payload = raw;
-        let targetEndpoint = `/upload/gmail/v1/users/me/messages/send`;
+        // Gmail JSON endpoint: 5MB body limit (~3.5MB raw before base64url overhead)
+        // Gmail upload endpoint: 35MB raw RFC822
+        let contentType;
+        let payload;
+        let targetEndpoint;
+        const JSON_SEND_LIMIT = 3.5 * 1024 * 1024;
 
-        // Use different endpoint for thread replies
-        if (data?.reference?.threadId) {
-            targetEndpoint = `/gmail/v1/users/me/messages/send`;
+        if (raw.length <= JSON_SEND_LIMIT) {
+            // JSON endpoint with base64url encoding (retry-safe, no ArrayBuffer issues)
             contentType = 'application/json';
-            payload = {
-                raw: raw.toString('base64'),
-                threadId: data?.reference?.threadId
-            };
+            payload = { raw: raw.toString('base64url') };
+            targetEndpoint = `/gmail/v1/users/me/messages/send`;
+            if (data?.reference?.threadId) {
+                payload.threadId = data.reference.threadId;
+            }
+        } else if (data?.reference?.threadId) {
+            // Large threaded reply: multipart upload preserves explicit threadId
+            // via JSON metadata alongside the raw RFC822 message body
+            const boundary = `ee_${crypto.randomBytes(16).toString('hex')}`;
+            const metadata = JSON.stringify({ threadId: data.reference.threadId });
+            const preamble = Buffer.from(
+                `--${boundary}\r\n` +
+                    `Content-Type: application/json; charset=UTF-8\r\n` +
+                    `\r\n` +
+                    `${metadata}\r\n` +
+                    `--${boundary}\r\n` +
+                    `Content-Type: message/rfc822\r\n` +
+                    `\r\n`
+            );
+            const epilogue = Buffer.from(`\r\n--${boundary}--`);
+
+            contentType = `multipart/related; boundary=${boundary}`;
+            payload = Buffer.concat([preamble, raw, epilogue]);
+            targetEndpoint = `/upload/gmail/v1/users/me/messages/send?uploadType=multipart`;
+        } else {
+            // Large non-threaded message: simple upload with raw RFC822 Buffer
+            contentType = 'message/rfc822';
+            payload = raw;
+            targetEndpoint = `/upload/gmail/v1/users/me/messages/send`;
         }
 
         // Send via Gmail API
@@ -1591,7 +1703,8 @@ class GmailClient extends BaseClient {
             await this.redis
                 .multi()
                 .hset(data.feedbackKey, 'success', 'true')
-                .expire(1 * 60 * 60);
+                .expire(data.feedbackKey, 1 * 60 * 60)
+                .exec();
         }
 
         return {
@@ -2042,6 +2155,35 @@ class GmailClient extends BaseClient {
         }
     }
 
+    /**
+     * Resolves label IDs to human-readable names on a formatted message.
+     * Mutates formattedMessage.labels in place. Labels starting with '\\' are
+     * treated as special-use labels and left as-is.
+     *
+     * @param {Object} formattedMessage - Message with a .labels array
+     * @param {Map} [labelMap] - Optional pre-built id-to-name map (avoids an extra getLabels call)
+     */
+    async resolveLabels(formattedMessage, labelMap) {
+        if (!Array.isArray(formattedMessage?.labels)) {
+            return;
+        }
+
+        if (!labelMap) {
+            const labelsResult = await this.getLabels();
+            labelMap = new Map();
+            for (const label of labelsResult || []) {
+                labelMap.set(label.id, label.name);
+            }
+        }
+
+        formattedMessage.labels = formattedMessage.labels.map(label => {
+            if (label.startsWith('\\')) {
+                return label;
+            }
+            return labelMap.get(label) || label;
+        });
+    }
+
     /**
      * Resolves a label by path or ID
      * @param {string} path - Label path or ID
@@ -2292,7 +2434,7 @@ class GmailClient extends BaseClient {
      * @returns {Object} Formatted message
      */
     formatMessage(messageData, options) {
-        let { extended, path, textType } = options || {};
+        let { extended, path, textType, minimalFields } = options || {};
 
         let date = messageData.internalDate && !isNaN(messageData.internalDate) ? new Date(Number(messageData.internalDate)) : undefined;
         if (date?.toString() === 'Invalid Date') {
@@ -2301,6 +2443,34 @@ class GmailClient extends BaseClient {
 
         let { flags, labels, category } = this.formatFlagsAndLabels(messageData);
 
+        // For minimalFields mode (format=minimal), payload is not available
+        // Return only basic fields: id, threadId, labelIds, internalDate, sizeEstimate
+        if (minimalFields) {
+            const result = {
+                id: messageData.id,
+                emailId: messageData.id || undefined,
+                threadId: messageData.threadId || undefined,
+                date: date ? date.toISOString() : undefined,
+                flags,
+                labels,
+                category,
+                unseen: !flags.includes('\\Seen') ? true : undefined,
+                flagged: flags.includes('\\Flagged') ? true : undefined,
+                draft: flags.includes('\\Draft') ? true : undefined,
+                size: messageData.sizeEstimate
+            };
+
+            // Set special-use based on labels
+            for (let specialUseTag of ['\\Junk', '\\Sent', '\\Trash', '\\Inbox', '\\Drafts']) {
+                if (result.labels && result.labels.includes(specialUseTag)) {
+                    result.messageSpecialUse = specialUseTag;
+                    break;
+                }
+            }
+
+            return result;
+        }
+
         let envelope = this.getEnvelope(messageData);
 
         // Extract all headers
@@ -2424,7 +2594,7 @@ class GmailClient extends BaseClient {
      * @returns {string} Formatted term
      */
     formatSearchTerm(term, quot = '"') {
-        if (typeof term === 'object' && term && Object.prototype.toString.apply(new Date()) === '[object Date]') {
+        if (typeof term === 'object' && term && Object.prototype.toString.apply(term) === '[object Date]') {
             term = term.toISOString().substring(0, 10);
         }
 

package/lib/email-client/imap/mailbox.js

@@ -99,6 +99,18 @@ class Mailbox {
 
         let mailboxInfo = connectionClient.mailbox;
 
+        // Log diagnostic info if mailbox object is missing or incomplete
+        if (!mailboxInfo) {
+            this.logger.warn({
+                msg: 'IMAP mailbox object is not available',
+                path: this.path,
+                connectionState: connectionClient.state,
+                usable: connectionClient.usable,
+                idling: connectionClient.idling
+            });
+            throw new Error('IMAP mailbox state is not available');
+        }
+
         let status = {
             path: this.path
         };
@@ -112,6 +124,21 @@ class Mailbox {
         // Total message count
         status.messages = mailboxInfo.exists ? mailboxInfo.exists : 0;
 
+        // Log diagnostic info if uidValidity is invalid
+        if (!status.uidValidity) {
+            this.logger.warn({
+                msg: 'Invalid uidValidity received from IMAP server',
+                path: this.path,
+                rawUidValidity: mailboxInfo.uidValidity,
+                rawUidValidityType: typeof mailboxInfo.uidValidity,
+                uidNext: status.uidNext,
+                messages: status.messages,
+                mailboxPath: mailboxInfo.path,
+                connectionState: connectionClient.state,
+                usable: connectionClient.usable
+            });
+        }
+
         return status;
     }
 
@@ -122,6 +149,20 @@ class Mailbox {
     async getStoredStatus() {
         let data = await this.connection.redis.hgetall(this.getMailboxKey());
         data = data || {};
+
+        // Log diagnostic info if stored uidValidity is invalid or missing
+        if (!validUidValidity(data.uidValidity)) {
+            this.logger.warn({
+                msg: 'Invalid or missing uidValidity in stored status',
+                path: this.path,
+                redisKey: this.getMailboxKey(),
+                rawUidValidity: data.uidValidity,
+                rawUidValidityType: typeof data.uidValidity,
+                hasData: Object.keys(data).length > 0,
+                storedKeys: Object.keys(data)
+            });
+        }
+
         return {
             path: data.path || this.path,
             uidValidity: validUidValidity(data.uidValidity) ? BigInt(data.uidValidity) : false,
@@ -135,6 +176,25 @@ class Mailbox {
         };
     }
 
+    /**
+     * Packs a UID into external reference format with validation and logging
+     * @param {number} uid - Message UID to pack
+     * @param {string} notificationType - Type of notification for logging context
+     * @returns {string|null} Packed UID or null if packing failed
+     */
+    async packUidWithLogging(uid, notificationType) {
+        let packedUid = await this.connection.packUid(this, uid);
+        if (!packedUid) {
+            this.logger.warn({
+                msg: `Skipping ${notificationType} notification - packUid returned invalid result`,
+                uid,
+                path: this.path
+            });
+            return null;
+        }
+        return packedUid;
+    }
+
     /**
      * Updates known mailbox state in Redis
      * @param {Object} data - Status data to store
@@ -149,10 +209,32 @@ class Mailbox {
             .map(key => {
                 switch (key) {
                     case 'path':
-                    case 'uidValidity':
                     case 'highestModseq':
                     case 'messages':
+                        return [key, data[key].toString()];
+
+                    case 'uidValidity':
+                        // Skip storing invalid uidValidity to prevent sync state corruption
+                        if (!validUidValidity(data[key])) {
+                            this.logger.warn({
+                                msg: 'Skipping invalid uidValidity value in updateStoredStatus',
+                                path: this.path,
+                                uidValidityValue: data[key]
+                            });
+                            return null;
+                        }
+                        return [key, data[key].toString()];
+
                     case 'uidNext':
+                        // Skip storing invalid uidNext to prevent sync state corruption
+                        if (!data[key]) {
+                            this.logger.warn({
+                                msg: 'Skipping invalid uidNext value in updateStoredStatus',
+                                path: this.path,
+                                uidNextValue: data[key]
+                            });
+                            return null;
+                        }
                         return [key, data[key].toString()];
 
                     case 'lastFullSync':
@@ -644,7 +726,10 @@ class Mailbox {
         */
 
         // Generate packed UID for external reference
-        let packedUid = await this.connection.packUid(this, messageData.uid);
+        let packedUid = await this.packUidWithLogging(messageData.uid, 'delete');
+        if (!packedUid) {
+            return;
+        }
         await this.connection.notify(this, MESSAGE_DELETED_NOTIFY, {
             id: packedUid,
             uid: messageData.uid
@@ -707,7 +792,10 @@ class Mailbox {
 
         if (!messageInfo) {
             // Message not found after retries - send missing notification
-            let packedUid = await this.connection.packUid(this, messageData.uid);
+            let packedUid = await this.packUidWithLogging(messageData.uid, 'missing');
+            if (!packedUid) {
+                return;
+            }
             await this.connection.notify(this, MESSAGE_MISSING_NOTIFY, {
                 id: packedUid,
                 uid: messageData.uid,
@@ -766,7 +854,7 @@ class Mailbox {
         if (this.mightBeAComplaint(messageInfo)) {
             try {
                 // Download relevant attachments for ARF parsing
-                for (let attachment of messageInfo.attachments) {
+                for (let attachment of messageInfo.attachments || []) {
                     if (!['message/feedback-report', 'message/rfc822-headers', 'message/rfc822'].includes(attachment.contentType)) {
                         continue;
                     }
@@ -1074,7 +1162,7 @@ class Mailbox {
         // Fetch inline attachments referenced in HTML
         if (messageInfo.attachments?.length && messageInfo.text?.html) {
             // fetch inline attachments
-            for (let attachment of messageInfo.attachments) {
+            for (let attachment of messageInfo.attachments || []) {
                 if (attachment.encodedSize && attachment.encodedSize > MAX_INLINE_ATTACHMENT_SIZE) {
                     // skip large attachments
                     continue;
@@ -1366,7 +1454,7 @@ class Mailbox {
                 let partList = [];
 
                 // Collect CID-referenced attachments
-                for (let attachment of messageInfo.attachments) {
+                for (let attachment of messageInfo.attachments || []) {
                     let contentId = attachment.contentId && attachment.contentId.replace(/^<|>$/g, '');
                     if (contentId && messageInfo.text.html.indexOf(contentId) >= 0) {
                         attachmentList.set(contentId, {
@@ -1786,7 +1874,10 @@ class Mailbox {
      * @param {Object} changes - What changed
      */
     async processChanges(messageData, changes) {
-        let packedUid = await this.connection.packUid(this, messageData.uid);
+        let packedUid = await this.packUidWithLogging(messageData.uid, 'update');
+        if (!packedUid) {
+            return;
+        }
         await this.connection.notify(this, MESSAGE_UPDATED_NOTIFY, {
             id: packedUid,
             uid: messageData.uid,
@@ -2303,7 +2394,7 @@ class Mailbox {
                 let partList = [];
 
                 // Find images referenced by CID
-                for (let attachment of messageInfo.attachments) {
+                for (let attachment of messageInfo.attachments || []) {
                     let contentId = attachment.contentId && attachment.contentId.replace(/^<|>$/g, '');
                     if (contentId && messageInfo.text.html.indexOf(contentId) >= 0) {
                         attachmentList.set(contentId, { attachment, content: null });

package/lib/email-client/imap/subconnection.js

@@ -112,7 +112,7 @@ class Subconnection extends EventEmitter {
             this.disabledReason = false;
         } catch (err) {
             // ended in an unconncted state
-            this.logger.error({ msg: 'Failed to set up subconnection', err });
+            this.logger.warn({ msg: 'Failed to set up subconnection', err });
         }
     }
 
@@ -124,11 +124,11 @@ class Subconnection extends EventEmitter {
             try {
                 prevImapClient.removeAllListeners();
                 prevImapClient.once('error', err => {
-                    this.logger.error({ msg: 'IMAP connection error', previous: true, account: this.account, err });
+                    this.logger.warn({ msg: 'IMAP connection error', previous: true, account: this.account, err });
                 });
                 prevImapClient.close();
             } catch (err) {
-                this.logger.error({ msg: 'IMAP close error', err });
+                this.logger.warn({ msg: 'IMAP close error', err });
             } finally {
                 if (prevImapClient === this.imapClient) {
                     this.imapClient = null;
@@ -153,13 +153,13 @@ class Subconnection extends EventEmitter {
         imapClient.subConnection = true;
 
         imapClient.on('error', err => {
-            imapClient?.log.error({ msg: 'IMAP connection error', account: this.account, err });
+            imapClient?.log.warn({ msg: 'IMAP connection error', account: this.account, err });
             if (imapClient !== this.imapClient || this._connecting) {
                 return;
             }
             imapClient.close();
             this.reconnect().catch(err => {
-                this.logger.error({ msg: 'IMAP reconnection error', account: this.account, err });
+                this.logger.warn({ msg: 'IMAP reconnection error', account: this.account, err });
             });
         });