emailengine-app 2.61.4 → 2.62.0

package/lib/email-client/imap/mailbox.js

@@ -99,6 +99,18 @@ class Mailbox {
 
         let mailboxInfo = connectionClient.mailbox;
 
+        // Log diagnostic info if mailbox object is missing or incomplete
+        if (!mailboxInfo) {
+            this.logger.warn({
+                msg: 'IMAP mailbox object is not available',
+                path: this.path,
+                connectionState: connectionClient.state,
+                usable: connectionClient.usable,
+                idling: connectionClient.idling
+            });
+            throw new Error('IMAP mailbox state is not available');
+        }
+
         let status = {
             path: this.path
         };
@@ -112,6 +124,21 @@ class Mailbox {
         // Total message count
         status.messages = mailboxInfo.exists ? mailboxInfo.exists : 0;
 
+        // Log diagnostic info if uidValidity is invalid
+        if (!status.uidValidity) {
+            this.logger.warn({
+                msg: 'Invalid uidValidity received from IMAP server',
+                path: this.path,
+                rawUidValidity: mailboxInfo.uidValidity,
+                rawUidValidityType: typeof mailboxInfo.uidValidity,
+                uidNext: status.uidNext,
+                messages: status.messages,
+                mailboxPath: mailboxInfo.path,
+                connectionState: connectionClient.state,
+                usable: connectionClient.usable
+            });
+        }
+
         return status;
     }
 
@@ -122,6 +149,20 @@ class Mailbox {
     async getStoredStatus() {
         let data = await this.connection.redis.hgetall(this.getMailboxKey());
         data = data || {};
+
+        // Log diagnostic info if stored uidValidity is invalid or missing
+        if (!validUidValidity(data.uidValidity)) {
+            this.logger.warn({
+                msg: 'Invalid or missing uidValidity in stored status',
+                path: this.path,
+                redisKey: this.getMailboxKey(),
+                rawUidValidity: data.uidValidity,
+                rawUidValidityType: typeof data.uidValidity,
+                hasData: Object.keys(data).length > 0,
+                storedKeys: Object.keys(data)
+            });
+        }
+
         return {
             path: data.path || this.path,
             uidValidity: validUidValidity(data.uidValidity) ? BigInt(data.uidValidity) : false,
@@ -135,6 +176,25 @@ class Mailbox {
         };
     }
 
+    /**
+     * Packs a UID into external reference format with validation and logging
+     * @param {number} uid - Message UID to pack
+     * @param {string} notificationType - Type of notification for logging context
+     * @returns {string|null} Packed UID or null if packing failed
+     */
+    async packUidWithLogging(uid, notificationType) {
+        let packedUid = await this.connection.packUid(this, uid);
+        if (!packedUid) {
+            this.logger.warn({
+                msg: `Skipping ${notificationType} notification - packUid returned invalid result`,
+                uid,
+                path: this.path
+            });
+            return null;
+        }
+        return packedUid;
+    }
+
     /**
      * Updates known mailbox state in Redis
      * @param {Object} data - Status data to store
@@ -149,10 +209,32 @@ class Mailbox {
             .map(key => {
                 switch (key) {
                     case 'path':
-                    case 'uidValidity':
                     case 'highestModseq':
                     case 'messages':
+                        return [key, data[key].toString()];
+
+                    case 'uidValidity':
+                        // Skip storing invalid uidValidity to prevent sync state corruption
+                        if (!validUidValidity(data[key])) {
+                            this.logger.warn({
+                                msg: 'Skipping invalid uidValidity value in updateStoredStatus',
+                                path: this.path,
+                                uidValidityValue: data[key]
+                            });
+                            return null;
+                        }
+                        return [key, data[key].toString()];
+
                     case 'uidNext':
+                        // Skip storing invalid uidNext to prevent sync state corruption
+                        if (!data[key]) {
+                            this.logger.warn({
+                                msg: 'Skipping invalid uidNext value in updateStoredStatus',
+                                path: this.path,
+                                uidNextValue: data[key]
+                            });
+                            return null;
+                        }
                         return [key, data[key].toString()];
 
                     case 'lastFullSync':
@@ -644,7 +726,10 @@ class Mailbox {
         */
 
         // Generate packed UID for external reference
-        let packedUid = await this.connection.packUid(this, messageData.uid);
+        let packedUid = await this.packUidWithLogging(messageData.uid, 'delete');
+        if (!packedUid) {
+            return;
+        }
         await this.connection.notify(this, MESSAGE_DELETED_NOTIFY, {
             id: packedUid,
             uid: messageData.uid
@@ -707,7 +792,10 @@ class Mailbox {
 
         if (!messageInfo) {
             // Message not found after retries - send missing notification
-            let packedUid = await this.connection.packUid(this, messageData.uid);
+            let packedUid = await this.packUidWithLogging(messageData.uid, 'missing');
+            if (!packedUid) {
+                return;
+            }
             await this.connection.notify(this, MESSAGE_MISSING_NOTIFY, {
                 id: packedUid,
                 uid: messageData.uid,
@@ -766,7 +854,7 @@ class Mailbox {
         if (this.mightBeAComplaint(messageInfo)) {
             try {
                 // Download relevant attachments for ARF parsing
-                for (let attachment of messageInfo.attachments) {
+                for (let attachment of messageInfo.attachments || []) {
                     if (!['message/feedback-report', 'message/rfc822-headers', 'message/rfc822'].includes(attachment.contentType)) {
                         continue;
                     }
@@ -1074,7 +1162,7 @@ class Mailbox {
         // Fetch inline attachments referenced in HTML
         if (messageInfo.attachments?.length && messageInfo.text?.html) {
             // fetch inline attachments
-            for (let attachment of messageInfo.attachments) {
+            for (let attachment of messageInfo.attachments || []) {
                 if (attachment.encodedSize && attachment.encodedSize > MAX_INLINE_ATTACHMENT_SIZE) {
                     // skip large attachments
                     continue;
@@ -1366,7 +1454,7 @@ class Mailbox {
                 let partList = [];
 
                 // Collect CID-referenced attachments
-                for (let attachment of messageInfo.attachments) {
+                for (let attachment of messageInfo.attachments || []) {
                     let contentId = attachment.contentId && attachment.contentId.replace(/^<|>$/g, '');
                     if (contentId && messageInfo.text.html.indexOf(contentId) >= 0) {
                         attachmentList.set(contentId, {
@@ -1786,7 +1874,10 @@ class Mailbox {
      * @param {Object} changes - What changed
      */
     async processChanges(messageData, changes) {
-        let packedUid = await this.connection.packUid(this, messageData.uid);
+        let packedUid = await this.packUidWithLogging(messageData.uid, 'update');
+        if (!packedUid) {
+            return;
+        }
         await this.connection.notify(this, MESSAGE_UPDATED_NOTIFY, {
             id: packedUid,
             uid: messageData.uid,
@@ -2303,7 +2394,7 @@ class Mailbox {
                 let partList = [];
 
                 // Find images referenced by CID
-                for (let attachment of messageInfo.attachments) {
+                for (let attachment of messageInfo.attachments || []) {
                     let contentId = attachment.contentId && attachment.contentId.replace(/^<|>$/g, '');
                     if (contentId && messageInfo.text.html.indexOf(contentId) >= 0) {
                         attachmentList.set(contentId, { attachment, content: null });

package/lib/email-client/imap/subconnection.js

@@ -112,7 +112,7 @@ class Subconnection extends EventEmitter {
             this.disabledReason = false;
         } catch (err) {
             // ended in an unconncted state
-            this.logger.error({ msg: 'Failed to set up subconnection', err });
+            this.logger.warn({ msg: 'Failed to set up subconnection', err });
         }
     }
 
@@ -124,11 +124,11 @@ class Subconnection extends EventEmitter {
             try {
                 prevImapClient.removeAllListeners();
                 prevImapClient.once('error', err => {
-                    this.logger.error({ msg: 'IMAP connection error', previous: true, account: this.account, err });
+                    this.logger.warn({ msg: 'IMAP connection error', previous: true, account: this.account, err });
                 });
                 prevImapClient.close();
             } catch (err) {
-                this.logger.error({ msg: 'IMAP close error', err });
+                this.logger.warn({ msg: 'IMAP close error', err });
             } finally {
                 if (prevImapClient === this.imapClient) {
                     this.imapClient = null;
@@ -153,13 +153,13 @@ class Subconnection extends EventEmitter {
         imapClient.subConnection = true;
 
         imapClient.on('error', err => {
-            imapClient?.log.error({ msg: 'IMAP connection error', account: this.account, err });
+            imapClient?.log.warn({ msg: 'IMAP connection error', account: this.account, err });
             if (imapClient !== this.imapClient || this._connecting) {
                 return;
             }
             imapClient.close();
             this.reconnect().catch(err => {
-                this.logger.error({ msg: 'IMAP reconnection error', account: this.account, err });
+                this.logger.warn({ msg: 'IMAP reconnection error', account: this.account, err });
             });
         });
 

package/lib/email-client/imap-client.js

@@ -288,7 +288,7 @@ class IMAPClient extends BaseClient {
 
             // Set up error handling for the command connection
             const onErr = err => {
-                commandClient?.log.error({ msg: 'IMAP connection error', cid: commandCid, channel: 'command', account: this.account, err });
+                commandClient?.log.warn({ msg: 'IMAP connection error', cid: commandCid, channel: 'command', account: this.account, err });
                 commandClient.close();
                 this.commandClient = null;
             };
@@ -301,7 +301,7 @@ class IMAPClient extends BaseClient {
                 if (this.connections.delete(commandClient)) {
                     await this.redis.hSetExists(this.getAccountKey(), 'connections', this.connections.size.toString());
                 }
-                commandClient.log.error({ msg: 'Failed to connect command client', cid: commandCid, channel: 'command', account: this.account, err });
+                commandClient.log.warn({ msg: 'Failed to connect command client', cid: commandCid, channel: 'command', account: this.account, err });
                 throw err;
             }
 
@@ -371,8 +371,47 @@ class IMAPClient extends BaseClient {
         }
 
         // Get stored mailbox status including UID validity
-        const storedStatus = await mailbox.getStoredStatus();
+        let storedStatus = await mailbox.getStoredStatus();
+
+        // If stored status missing uidValidity, try live IMAP mailbox status
+        if (!validUidValidity(storedStatus.uidValidity)) {
+            try {
+                const liveStatus = mailbox.getMailboxStatus();
+                if (validUidValidity(liveStatus.uidValidity)) {
+                    this.logger.debug({
+                        msg: 'packUid using live mailbox status (stored uidValidity missing)',
+                        path: mailbox.path,
+                        liveUidValidity: liveStatus.uidValidity.toString()
+                    });
+                    storedStatus = {
+                        ...storedStatus,
+                        uidValidity: liveStatus.uidValidity,
+                        path: liveStatus.path || mailbox.path
+                    };
+                }
+            } catch (err) {
+                // Live status not available (mailbox not selected)
+                this.logger.debug({
+                    msg: 'packUid live status fallback unavailable',
+                    path: mailbox.path,
+                    error: err.message
+                });
+            }
+        }
+
+        // Final validation
         if (!validUidValidity(storedStatus.uidValidity) || !storedStatus.path) {
+            this.logger.warn({
+                msg: 'packUid failed due to invalid stored status',
+                uid,
+                mailboxPath: mailbox.path,
+                storedUidValidity: storedStatus.uidValidity,
+                storedUidValidityType: typeof storedStatus.uidValidity,
+                storedPath: storedStatus.path,
+                storedUidNext: storedStatus.uidNext,
+                storedMessages: storedStatus.messages,
+                syncing: mailbox.syncing || false
+            });
             return false;
         }
 
@@ -498,7 +537,9 @@ class IMAPClient extends BaseClient {
 
         const connectionClient = await this.getImapConnection(connectionOptions, 'getCurrentListing');
         if (!connectionClient) {
-            this.imapClient.close();
+            if (this.imapClient) {
+                this.imapClient.close();
+            }
             let error = new Error('Failed to get connection');
             error.code = 'ConnectionError';
             throw error;
@@ -522,7 +563,9 @@ class IMAPClient extends BaseClient {
         let listing = await connectionClient.list(options);
         if (!listing.length) {
             // server bug, the list can never be empty
-            this.imapClient.close();
+            if (this.imapClient) {
+                this.imapClient.close();
+            }
             let error = new Error('Server bug: empty mailbox listing');
             error.code = 'ServerBug';
             throw error;
@@ -879,7 +922,7 @@ class IMAPClient extends BaseClient {
             }
         } catch (err) {
             // ended in an unconncted state
-            this.logger.error({ msg: 'Failed to set up connection, will retry', err });
+            this.logger.warn({ msg: 'Failed to set up connection, will retry', err });
 
             // Calculate delay with capped exponential backoff
             const retryDelay = Math.min(this.reconnectMaxDelay, 1000 * Math.pow(1.5, Math.min(this.reconnectRetries, 10)));
@@ -898,7 +941,7 @@ class IMAPClient extends BaseClient {
                         this.reconnectRetries = 0;
                     })
                     .catch(err => {
-                        this.logger.error({
+                        this.logger.warn({
                             msg: 'Connection retry failed',
                             err,
                             attempt: this.reconnectRetries
@@ -935,6 +978,10 @@ class IMAPClient extends BaseClient {
 
         // Sync existing folders that weren't just synced
         for (let mailbox of this.mailboxes.values()) {
+            if (!this.imapClient || !this.imapClient.usable) {
+                this.logger.debug({ msg: 'IMAP client disconnected during mailbox sync' });
+                return;
+            }
             if (!synced || !synced.has(mailbox)) {
                 await mailbox.sync();
             }
@@ -949,12 +996,19 @@ class IMAPClient extends BaseClient {
         this.state = 'connected';
         await this.setStateVal();
 
+        // Capture local reference to avoid race conditions during async operations
+        const imapClient = this.imapClient;
+        if (!imapClient || !imapClient.usable) {
+            this.logger.debug({ msg: 'IMAP client disconnected or not usable after state update' });
+            return;
+        }
+
         // Store IMAP server capabilities for reference
-        const capabilities = (this.imapClient.rawCapabilities || []).map(entry => entry && entry.value).filter(entry => entry);
+        const capabilities = (imapClient.rawCapabilities || []).map(entry => entry && entry.value).filter(entry => entry);
         const authCapabilities = [];
         let lastUsedAuthCapability = null;
-        if (this.imapClient.authCapabilities) {
-            for (let [authCapa, usedAuth] of this.imapClient.authCapabilities) {
+        if (imapClient.authCapabilities) {
+            for (let [authCapa, usedAuth] of imapClient.authCapabilities) {
                 authCapabilities.push(authCapa);
                 if (usedAuth) {
                     lastUsedAuthCapability = authCapa;
@@ -962,7 +1016,7 @@ class IMAPClient extends BaseClient {
             }
         }
 
-        const serverInfo = Object.assign({}, this.imapClient.serverInfo || {}, {
+        const serverInfo = Object.assign({}, imapClient.serverInfo || {}, {
             capabilities,
             authCapabilities,
             lastUsedAuthCapability
@@ -980,6 +1034,12 @@ class IMAPClient extends BaseClient {
             return;
         }
 
+        // Check if client disconnected before mailbox selection
+        if (!this.imapClient || !this.imapClient.usable) {
+            this.logger.debug({ msg: 'IMAP client disconnected before mailbox selection' });
+            return;
+        }
+
         this.logger.debug({ msg: 'Syncing completed, selecting main path', path: mainPath });
         // start waiting for changes
         await this.select(mainPath);
@@ -1195,7 +1255,7 @@ class IMAPClient extends BaseClient {
                 prevImapClient.removeAllListeners();
 
                 const prevImapErrorHandler = err => {
-                    this.logger.error({ msg: 'IMAP connection error', type: 'imapClient', previous: true, account: this.account, err });
+                    this.logger.warn({ msg: 'IMAP connection error', type: 'imapClient', previous: true, account: this.account, err });
                 };
 
                 prevImapClient.once('error', prevImapErrorHandler);
@@ -1207,7 +1267,7 @@ class IMAPClient extends BaseClient {
                     this.commandClient.close();
                 }
             } catch (err) {
-                this.logger.error({ msg: 'IMAP close error', err });
+                this.logger.warn({ msg: 'IMAP close error', err });
             } finally {
                 if (prevImapClient === this.imapClient) {
                     this.imapClient = null;
@@ -1267,7 +1327,7 @@ class IMAPClient extends BaseClient {
 
             // Handle connection errors
             imapClient.on('error', err => {
-                imapClient?.log.error({ msg: 'IMAP connection error', type: 'imapClient', account: this.account, err });
+                imapClient?.log.warn({ msg: 'IMAP connection error', type: 'imapClient', account: this.account, err });
                 if (imapClient !== this.imapClient || this._connecting) {
                     return;
                 }
@@ -1291,7 +1351,7 @@ class IMAPClient extends BaseClient {
                                 this.errorReconnectDelay = 2000;
                             })
                             .catch(err => {
-                                this.logger.error({
+                                this.logger.warn({
                                     msg: 'IMAP reconnection error',
                                     account: this.account,
                                     err,
@@ -2303,7 +2363,7 @@ class IMAPClient extends BaseClient {
             let uploadResponse = await connectionClient.append(data.path, raw, data.flags, data.internalDate);
 
             // Return to IDLE if using primary connection
-            if (connectionClient === this.imapClient && this.imapClient.mailbox && !this.imapClient.idling) {
+            if (this.imapClient && connectionClient === this.imapClient && this.imapClient.mailbox && !this.imapClient.idling) {
                 // force back to IDLE
                 this.imapClient.idle().catch(err => {
                     this.logger.error({ msg: 'IDLE error', err });

package/lib/email-client/message-builder.js

@@ -467,7 +467,9 @@ const SMTP_ERROR_DESCRIPTIONS = {
     EDNS: settings => `EmailEngine failed to resolve DNS record for ${settings.host}`,
     ECONNECTION: settings => `EmailEngine failed to establish TCP connection against ${settings.host}`,
     EPROTOCOL: settings => `Unexpected response from ${settings.host}`,
-    EAUTH: () => 'Authentication failed'
+    EAUTH: () => 'Authentication failed',
+    ENOAUTH: () => 'Authentication credentials were not provided',
+    EOAUTH2: () => 'OAuth2 token generation or refresh failed'
 };
 
 /**

package/lib/email-client/notification-handler.js

@@ -19,12 +19,13 @@ const DOCUMENT_SYNC_EVENTS = [MESSAGE_NEW_NOTIFY, MESSAGE_DELETED_NOTIFY, MESSAG
  * Default job options for notification queue
  */
 const DEFAULT_JOB_OPTIONS = {
-    removeOnComplete: true,
-    removeOnFail: true,
+    removeOnComplete: { age: 24 * 3600, count: 1000 },
+    removeOnFail: { age: 24 * 3600, count: 1000 },
     attempts: 10,
     backoff: {
         type: 'exponential',
-        delay: 5000
+        delay: 5000,
+        jitter: 0.2 // 20% randomization to prevent thundering herd
     }
 };
 
@@ -236,18 +237,20 @@ class NotificationHandler {
 
     /**
      * Builds job options based on queue keep setting
-     * @param {boolean} queueKeep - Whether to keep completed/failed jobs
+     * @param {boolean|number} queueKeep - Whether/how many jobs to keep
      * @returns {Object} Job options for notify and documents queues
      */
     buildJobOptions(queueKeep) {
+        const retention = typeof queueKeep === 'number' ? { age: 24 * 3600, count: queueKeep } : queueKeep;
+
         const notifyOptions = Object.assign({}, DEFAULT_JOB_OPTIONS, {
-            removeOnComplete: queueKeep,
-            removeOnFail: queueKeep
+            removeOnComplete: retention,
+            removeOnFail: retention
         });
 
         const documentOptions = Object.assign({}, DOCUMENT_JOB_OPTIONS, {
-            removeOnComplete: queueKeep,
-            removeOnFail: queueKeep
+            removeOnComplete: retention,
+            removeOnFail: retention
         });
 
         return {
@@ -289,7 +292,7 @@ class NotificationHandler {
         }
 
         // Get queue retention setting
-        const queueKeep = (await settings.get('queueKeep')) || true;
+        const queueKeep = (await settings.get('queueKeep')) ?? true;
 
         // Process notification based on required destinations
         if (!skipWebhook && addDocumentQueueJob) {