emailengine-app 2.65.0 → 2.67.0

package/lib/email-client/imap/sync-operations.js

@@ -3,7 +3,7 @@
 const crypto = require('crypto');
 const { compareExisting, calculateFetchBackoff, readEnvValue } = require('../../tools');
 const config = require('@zone-eu/wild-config');
-const { DEFAULT_FETCH_BATCH_SIZE } = require('../../consts');
+const { DEFAULT_FETCH_BATCH_SIZE, FETCH_RETRY_MAX_TIME, FETCH_RETRY_MIN_ATTEMPTS } = require('../../consts');
 
 // Configurable batch size for fetching messages (default: 250)
 const FETCH_BATCH_SIZE = Number(readEnvValue('EENGINE_FETCH_BATCH_SIZE') || config.service.fetchBatchSize) || DEFAULT_FETCH_BATCH_SIZE;
@@ -11,6 +11,15 @@ const FETCH_BATCH_SIZE = Number(readEnvValue('EENGINE_FETCH_BATCH_SIZE') || conf
 // Do not check for flag updates using full sync more often than this value (30 minutes)
 const FULL_SYNC_DELAY = 30 * 60 * 1000;
 
+function checkFetchRetryTimeout(retryCount, startTime) {
+    if (retryCount >= FETCH_RETRY_MIN_ATTEMPTS && Date.now() - startTime > FETCH_RETRY_MAX_TIME) {
+        const err = new Error('FETCH retry time limit exceeded');
+        err.code = 'FetchRetryTimeout';
+        err.statusCode = 504;
+        throw err;
+    }
+}
+
 /**
  * Calculates the next range of sequence numbers to fetch based on the last fetched range
  * @param {Number} totalMessages - Total number of messages in the mailbox
@@ -234,6 +243,7 @@ class SyncOperations {
                 // Fetch messages with retry logic
                 let fetchCompleted = false;
                 let fetchRetryCount = 0;
+                let fetchStartTime = Date.now();
 
                 while (!fetchCompleted) {
                     try {
@@ -312,9 +322,11 @@ class SyncOperations {
 
                         // Retry with exponential backoff
                         if (!imapClient.usable) {
-                            // nothing to do here, connection closed
-                            this.logger.error({ msg: `FETCH failed, connection already closed, not retrying`, err });
-                            return;
+                            // connection closed, throw so callers know sync was interrupted
+                            const connErr = new Error('IMAP connection closed during sync');
+                            connErr.code = 'IMAPConnectionClosing';
+                            connErr.statusCode = 503;
+                            throw connErr;
                         }
 
                         const fetchRetryDelay = calculateFetchBackoff(++fetchRetryCount);
@@ -322,10 +334,14 @@ class SyncOperations {
                         await new Promise(r => setTimeout(r, fetchRetryDelay));
 
                         if (!imapClient.usable) {
-                            // nothing to do here, connection closed
-                            this.logger.error({ msg: `FETCH failed, connection already closed, not retrying`, err });
-                            return;
+                            // connection closed, throw so callers know sync was interrupted
+                            const connErr = new Error('IMAP connection closed during sync');
+                            connErr.code = 'IMAPConnectionClosing';
+                            connErr.statusCode = 503;
+                            throw connErr;
                         }
+
+                        checkFetchRetryTimeout(fetchRetryCount, fetchStartTime);
                     }
                 }
             }
@@ -379,6 +395,7 @@ class SyncOperations {
                 // only fetch messages if there are some
                 let fetchCompleted = false;
                 let fetchRetryCount = 0;
+                let fetchStartTime = Date.now();
                 while (!fetchCompleted) {
                     // Get fresh imapClient reference inside retry loop
                     let imapClient = this.connection.imapClient;
@@ -461,9 +478,11 @@ class SyncOperations {
 
                         // Retry with exponential backoff
                         if (!imapClient.usable) {
-                            // nothing to do here, connection closed
-                            this.logger.error({ msg: `FETCH failed, connection already closed, not retrying` });
-                            return;
+                            // connection closed, throw so callers know sync was interrupted
+                            const connErr = new Error('IMAP connection closed during sync');
+                            connErr.code = 'IMAPConnectionClosing';
+                            connErr.statusCode = 503;
+                            throw connErr;
                         }
 
                         const fetchRetryDelay = calculateFetchBackoff(++fetchRetryCount);
@@ -471,10 +490,14 @@ class SyncOperations {
                         await new Promise(r => setTimeout(r, fetchRetryDelay));
 
                         if (!imapClient.usable) {
-                            // nothing to do here, connection closed
-                            this.logger.error({ msg: `FETCH failed, connection already closed, not retrying` });
-                            return;
+                            // connection closed, throw so callers know sync was interrupted
+                            const connErr = new Error('IMAP connection closed during sync');
+                            connErr.code = 'IMAPConnectionClosing';
+                            connErr.statusCode = 503;
+                            throw connErr;
                         }
+
+                        checkFetchRetryTimeout(fetchRetryCount, fetchStartTime);
                     }
                 }
             }
@@ -545,6 +568,8 @@ class SyncOperations {
                 let range = false;
                 let lastHighestUid = 0;
                 let batchNumber = 0;
+                let fetchStartTime = Date.now();
+                let totalFetchRetryCount = 0;
                 // process messages in batches
                 while ((range = getFetchRange(mailboxStatus.messages, range))) {
                     batchNumber++;
@@ -703,9 +728,11 @@ class SyncOperations {
                             });
 
                             if (!imapClient.usable) {
-                                // nothing to do here, connection closed
-                                this.logger.error({ msg: `FETCH failed, connection already closed, not retrying`, loopId });
-                                return;
+                                // connection closed, throw so callers know sync was interrupted
+                                const connErr = new Error('IMAP connection closed during sync');
+                                connErr.code = 'IMAPConnectionClosing';
+                                connErr.statusCode = 503;
+                                throw connErr;
                             }
 
                             try {
@@ -728,14 +755,17 @@ class SyncOperations {
                             }
 
                             // Retry with exponential backoff
+                            totalFetchRetryCount++;
                             const fetchRetryDelay = calculateFetchBackoff(++fetchRetryCount);
                             this.logger.error({ msg: `FETCH failed, retrying in ${Math.round(fetchRetryDelay / 1000)}s`, loopId, batchNumber });
                             await new Promise(r => setTimeout(r, fetchRetryDelay));
 
                             if (!imapClient.usable) {
-                                // nothing to do here, connection closed
-                                this.logger.error({ msg: `FETCH failed, connection already closed, not retrying`, loopId });
-                                return;
+                                // connection closed, throw so callers know sync was interrupted
+                                const connErr = new Error('IMAP connection closed during sync');
+                                connErr.code = 'IMAPConnectionClosing';
+                                connErr.statusCode = 503;
+                                throw connErr;
                             }
 
                             // Verify we're still on the correct mailbox after the delay
@@ -764,6 +794,8 @@ class SyncOperations {
                                 newMessages: mailboxStatus.messages,
                                 range
                             });
+
+                            checkFetchRetryTimeout(totalFetchRetryCount, fetchStartTime);
                         }
                     }
                 }

package/lib/email-client/imap-client.js

@@ -299,7 +299,9 @@ class IMAPClient extends BaseClient {
 
             // Set up error handling for the command connection
             const onErr = err => {
-                commandClient?.log.warn({ msg: 'IMAP connection error', cid: commandCid, channel: 'command', account: this.account, err });
+                // Idle timeout (ETIMEOUT) is expected - the command client has no keepalive
+                const logLevel = err.code === 'ETIMEOUT' ? 'debug' : 'warn';
+                commandClient?.log[logLevel]({ msg: 'IMAP connection error', cid: commandCid, channel: 'command', account: this.account, err });
                 commandClient.close();
                 this.commandClient = null;
             };
@@ -605,9 +607,15 @@ class IMAPClient extends BaseClient {
             })
             .filter(entry => entry);
 
+        // Build lookup map from stored listing for O(1) access
+        const storedListingMap = new Map();
+        for (const entry of storedListing) {
+            storedListingMap.set(normalizePath(entry.path), entry);
+        }
+
         // compare listings to detect changes
         for (let mailbox of listing) {
-            let existingMailbox = storedListing.find(entry => normalizePath(entry.path) === normalizePath(mailbox.path));
+            let existingMailbox = storedListingMap.get(normalizePath(mailbox.path));
             if (!existingMailbox) {
                 // found new!
                 mailbox.isNew = true;
@@ -622,8 +630,9 @@ class IMAPClient extends BaseClient {
         }
 
         // Check for deleted mailboxes
+        const listingPathSet = new Set(listing.map(mailbox => normalizePath(mailbox.path)));
         for (let entry of storedListing) {
-            if (!listing.some(mailbox => normalizePath(entry.path) === normalizePath(mailbox.path))) {
+            if (!listingPathSet.has(normalizePath(entry.path))) {
                 // found deleted!
                 await this.clearMailboxEntry(entry);
                 hasChanges = true;
@@ -1207,9 +1216,14 @@ class IMAPClient extends BaseClient {
         }
 
         // Provider-specific workarounds
-        if (/(\.rambler\.ru|\.163\.com)$/i.test(imapConfig.host)) {
-            // Special case for Rambler and 163. Break IDLE at least once a minute
+        if (/\.rambler\.ru$/i.test(imapConfig.host)) {
+            // Rambler supports IDLE but needs it restarted frequently
+            imapConfig.maxIdleTime = 55 * 1000;
+        } else if (/\.163\.com$/i.test(imapConfig.host)) {
+            // 163.com (Coremail) does not support IDLE and ignores NOOP for idle timer reset.
+            // Use STATUS as the keepalive command - it counts as real activity for the server.
             imapConfig.maxIdleTime = 55 * 1000;
+            imapConfig.missingIdleCommand = 'STATUS';
         } else if (/\.yahoo\.com$/i.test(imapConfig.host)) {
             // Special case for Yahoo. Break IDLE at least once every three minutes
             imapConfig.maxIdleTime = 3 * 60 * 1000;
@@ -2461,6 +2475,9 @@ class IMAPClient extends BaseClient {
         const mailboxes = [];
 
         const listing = await this.getCurrentListing(false, { allowSecondary: true });
+        // Capture reference so we can close it after the await-heavy loop below
+        // without risking closure of a different command client obtained by a concurrent caller
+        const commandClientForListing = this.commandClient;
 
         // Process each configured subconnection
         for (const path of accountData.subconnections || []) {
@@ -2588,6 +2605,12 @@ class IMAPClient extends BaseClient {
             await subconnection.init();
         }
 
+        // Close the command client if it is still the same instance we captured above
+        if (commandClientForListing && this.commandClient === commandClientForListing) {
+            this.commandClient.close();
+            this.commandClient = null;
+        }
+
         return this.subconnections.length;
     }
 

package/lib/email-client/outlook-client.js

@@ -2875,7 +2875,7 @@ class OutlookClient extends BaseClient {
             return this.oAuth2Client;
         }
         let accountData = await (this.delegatedAccountObject || this.accountObject).loadAccountData();
-        this.oAuth2Client = await oauth2Apps.getClient(accountData.oauth2.provider, {
+        this.oAuth2Client = await oauth2Apps.getClient(accountData.oauth2?.provider || accountData._app?.id, {
             logger: this.logger,
             logRaw: this.options.logRaw
         });
@@ -3857,8 +3857,11 @@ class OutlookClient extends BaseClient {
     async processHistory() {
         let event;
         let newMessageOptions;
+        let processedAny = false;
 
         while ((event = await this.accountObject.pullQueueEvent()) !== null) {
+            processedAny = true;
+
             switch (event.type) {
                 case 'updated':
                     await this.processUpdatedMessage(event.message);
@@ -3900,6 +3903,157 @@ class OutlookClient extends BaseClient {
                     break;
             }
         }
+
+        if (processedAny) {
+            await this._updateLastNotificationTime();
+        }
+    }
+
+    async _updateLastNotificationTime() {
+        try {
+            await this.redis.hset(this.getAccountKey(), 'outlookLastNotification', Date.now().toString());
+        } catch (err) {
+            this.logger.error({ msg: 'Failed to update last notification time', err });
+        }
+    }
+
+    /**
+     * Recover messages missed due to failed webhook delivery.
+     * Triggered by MS Graph 'missed' lifecycle event.
+     * Queries recent messages and processes any not already seen via normal webhooks.
+     * @returns {boolean} Whether recovery ran (false if debounced)
+     */
+    async syncMissedMessages() {
+        // Debounce: skip if recovery already ran recently
+        const cooldownKey = `${REDIS_PREFIX}iam:${this.account}:missedRecovery`;
+        const acquired = await this.redis.set(cooldownKey, '1', 'EX', 300, 'NX');
+        if (!acquired) {
+            this.logger.debug({ msg: 'Missed notification recovery skipped (cooldown active)', account: this.account });
+            return false;
+        }
+
+        // Determine lookback window
+        let sinceTime;
+        const lastNotification = await this.redis.hget(this.getAccountKey(), 'outlookLastNotification');
+        if (lastNotification) {
+            // 2-minute safety margin before last known good notification
+            sinceTime = new Date(Number(lastNotification) - 2 * 60 * 1000);
+        } else {
+            sinceTime = new Date(Date.now() - 30 * 60 * 1000);
+        }
+
+        this.logger.info({
+            msg: 'Recovering missed notifications',
+            account: this.account,
+            since: sinceTime.toISOString()
+        });
+
+        let messages = [];
+        let reqUrl = `/${this.oauth2UserPath}/messages`;
+        let reqPayload = {
+            $filter: `receivedDateTime gt ${sinceTime.toISOString()}`,
+            $select: 'id,parentFolderId',
+            $top: 250,
+            $orderby: 'receivedDateTime desc'
+        };
+
+        try {
+            while (reqUrl) {
+                let pageResult = await this.request(reqUrl, 'get', reqPayload);
+
+                if (pageResult?.value?.length) {
+                    messages.push(...pageResult.value);
+                    if (messages.length >= 1000) {
+                        messages.length = 1000;
+                        break;
+                    }
+                }
+
+                if (pageResult['@odata.nextLink']) {
+                    reqUrl = pageResult['@odata.nextLink'];
+                    reqPayload = null; // nextLink includes all query params
+                } else {
+                    reqUrl = null;
+                }
+            }
+        } catch (err) {
+            this.logger.error({ msg: 'Failed to query recent messages for missed notification recovery', account: this.account, err });
+            await this.redis.del(cooldownKey);
+            return false;
+        }
+
+        if (!messages.length) {
+            this.logger.debug({ msg: 'No recent messages found during missed notification recovery', account: this.account });
+            return true;
+        }
+
+        this.logger.info({
+            msg: 'Found recent messages for missed notification recovery',
+            account: this.account,
+            count: messages.length
+        });
+
+        let newMessageOptions = await this.getMessageFetchOptions();
+        newMessageOptions.showPath = true;
+
+        // Pre-resolve unique folder IDs once to avoid per-message Redis lookups
+        let folderMap = new Map();
+        let cachedListing = await this.getCachedMailboxListing();
+        if (cachedListing) {
+            for (let msg of messages) {
+                if (msg.parentFolderId && !folderMap.has(msg.parentFolderId)) {
+                    let folder = cachedListing.find(entry => entry.id === msg.parentFolderId);
+                    if (folder) {
+                        folderMap.set(msg.parentFolderId, folder);
+                    }
+                }
+            }
+        }
+
+        let recoveredCount = 0;
+
+        for (const msg of messages) {
+            try {
+                // Pre-filter: check rollingBucketLock BEFORE the expensive getMessage() call
+                let preFilterChecked = false;
+                let folder = folderMap.get(msg.parentFolderId);
+                if (folder) {
+                    let recentlySeen = await this.rollingBucketLock(`${msg.id}:created`, folder.pathName);
+                    if (recentlySeen) {
+                        continue;
+                    }
+                    preFilterChecked = true;
+                }
+
+                let messageData = await this.prepareNewMessage(msg.id, newMessageOptions);
+                if (!messageData) {
+                    continue;
+                }
+
+                if (!preFilterChecked) {
+                    let recentlySeen = await this.rollingBucketLock(`${messageData.id}:created`, messageData.path);
+                    if (recentlySeen) {
+                        continue;
+                    }
+                }
+
+                await this.processNew(messageData, newMessageOptions);
+                recoveredCount++;
+            } catch (err) {
+                this.logger.error({ msg: 'Failed to process message during missed notification recovery', emailId: msg.id, account: this.account, err });
+            }
+        }
+
+        this.logger.info({
+            msg: 'Missed notification recovery completed',
+            account: this.account,
+            checked: messages.length,
+            recovered: recoveredCount
+        });
+
+        await this._updateLastNotificationTime();
+
+        return true;
     }
 
     /**

package/lib/oauth/gmail.js

@@ -20,6 +20,8 @@ const GMAIL_API_SCOPES = {
     labels: 'https://www.googleapis.com/auth/gmail.labels'
 };
 
+const OPENID_SCOPES = ['openid', 'email', 'profile'];
+
 const EXPOSE_PARTIAL_SECRET_KEY_REGEX = /Unauthorized/i;
 
 const checkForFlags = (err, isPrincipal) => {
@@ -171,7 +173,7 @@ class GmailOauth {
         // Service accounts use JWT-based authentication and don't need OpenID scopes
         // These are non-privileged scopes that provide email and profile info for 3-legged OAuth
         if (!opts.serviceClient) {
-            const openidScopes = ['openid', 'email', 'profile'];
+            const openidScopes = OPENID_SCOPES;
 
             // Add OpenID scopes if not already present
             for (let scope of openidScopes) {
@@ -203,6 +205,7 @@ class GmailOauth {
         this.setFlag = opts.setFlag;
 
         this.tokenUrl = `https://oauth2.googleapis.com/token`;
+        this.revokeUrl = `https://oauth2.googleapis.com/revoke`;
     }
 
     generateAuthUrl(opts) {
@@ -675,8 +678,56 @@ class GmailOauth {
         const signature = crypto.createSign('RSA-SHA256').update(encodedPayload).sign(this.serviceKey);
         return [encodedPayload, Buffer.from(signature).toString('base64url')].join('.');
     }
+
+    async revokeToken(token) {
+        const fetchOpts = {
+            method: 'post',
+            headers: {
+                'Content-Type': 'application/x-www-form-urlencoded',
+                'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`
+            },
+            body: new URLSearchParams({ token }).toString()
+        };
+
+        try {
+            let res = await fetchCmd(this.revokeUrl, Object.assign(fetchOpts, { dispatcher: httpAgent.retry }));
+
+            let responseText;
+            try {
+                responseText = await res.text();
+            } catch (err) {
+                // ignore
+            }
+
+            if (this.logger) {
+                this.logger.info({
+                    msg: 'OAuth2 token revocation request',
+                    action: 'oauth2Fetch',
+                    fn: 'revokeToken',
+                    url: this.revokeUrl,
+                    success: !!res.ok,
+                    status: res.status
+                });
+            }
+
+            if (!res.ok) {
+                if (this.logger) {
+                    this.logger.warn({
+                        msg: 'OAuth2 token revocation failed',
+                        status: res.status,
+                        response: responseText
+                    });
+                }
+            }
+        } catch (err) {
+            if (this.logger) {
+                this.logger.error({ msg: 'OAuth2 token revocation error', err });
+            }
+        }
+    }
 }
 
 module.exports.GmailOauth = GmailOauth;
 module.exports.GMAIL_SCOPES = GMAIL_SCOPES;
 module.exports.GMAIL_API_SCOPES = GMAIL_API_SCOPES;
+module.exports.OPENID_SCOPES = OPENID_SCOPES;