emailengine-app 2.66.0 → 2.67.0

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## [2.67.0](https://github.com/postalsys/emailengine/compare/v2.66.0...v2.67.0) (2026-03-31)
+
+
+### Features
+
+* handle MS Graph 'missed' lifecycle event for Outlook notification recovery ([d55f48c](https://github.com/postalsys/emailengine/commit/d55f48c9a183f9e0108801563b1085c176dbeb06))
+* show human-readable missing scopes on OAuth scope error page ([4ee2365](https://github.com/postalsys/emailengine/commit/4ee2365449e3cbece6f4a83d540ae63c18464535))
+
+
+### Bug Fixes
+
+* add IMAP fetch retry timeout and harden passkey registration ([3a1b61b](https://github.com/postalsys/emailengine/commit/3a1b61b00e916c308cae5e7493a2207164d9d45f))
+* close command client proactively after subconnection setup ([02d3687](https://github.com/postalsys/emailengine/commit/02d36874d8533beabe5ad0fba978851b3791cbb5))
+* correct translation errors and remove EmailEngine branding from OAuth scope error ([9284ddd](https://github.com/postalsys/emailengine/commit/9284ddd1e28e40fd24abeea7296636b21c4b3878))
+* use STATUS instead of NOOP as keepalive for 163.com IMAP ([3b3516f](https://github.com/postalsys/emailengine/commit/3b3516feb14e64bab5ac586378237f2160cdf010))
+
 ## [2.66.0](https://github.com/postalsys/emailengine/compare/v2.65.0...v2.66.0) (2026-03-29)
 
 

package/data/google-crawlers.json

@@ -1,5 +1,5 @@
 {
-    "creationTime": "2026-03-27T15:45:58.000000",
+    "creationTime": "2026-03-30T14:45:52.000000",
     "prefixes": [
         {
             "ipv6Prefix": "2001:4860:4801:2008::/64"

package/lib/consts.js

@@ -193,6 +193,8 @@ module.exports = {
     FETCH_RETRY_INTERVAL: 1000,
     FETCH_RETRY_EXPONENTIAL: 2,
     FETCH_RETRY_MAX: 60 * 1000,
+    FETCH_RETRY_MAX_TIME: 24 * 60 * 60 * 1000, // 1 day maximum retry duration
+    FETCH_RETRY_MIN_ATTEMPTS: 20, // minimum attempts before time limit applies
     DEFAULT_FETCH_BATCH_SIZE: 1000,
 
     // MS Graph webhook subscription settings

package/lib/email-client/imap/sync-operations.js

@@ -3,7 +3,7 @@
 const crypto = require('crypto');
 const { compareExisting, calculateFetchBackoff, readEnvValue } = require('../../tools');
 const config = require('@zone-eu/wild-config');
-const { DEFAULT_FETCH_BATCH_SIZE } = require('../../consts');
+const { DEFAULT_FETCH_BATCH_SIZE, FETCH_RETRY_MAX_TIME, FETCH_RETRY_MIN_ATTEMPTS } = require('../../consts');
 
 // Configurable batch size for fetching messages (default: 250)
 const FETCH_BATCH_SIZE = Number(readEnvValue('EENGINE_FETCH_BATCH_SIZE') || config.service.fetchBatchSize) || DEFAULT_FETCH_BATCH_SIZE;
@@ -11,6 +11,15 @@ const FETCH_BATCH_SIZE = Number(readEnvValue('EENGINE_FETCH_BATCH_SIZE') || conf
 // Do not check for flag updates using full sync more often than this value (30 minutes)
 const FULL_SYNC_DELAY = 30 * 60 * 1000;
 
+function checkFetchRetryTimeout(retryCount, startTime) {
+    if (retryCount >= FETCH_RETRY_MIN_ATTEMPTS && Date.now() - startTime > FETCH_RETRY_MAX_TIME) {
+        const err = new Error('FETCH retry time limit exceeded');
+        err.code = 'FetchRetryTimeout';
+        err.statusCode = 504;
+        throw err;
+    }
+}
+
 /**
  * Calculates the next range of sequence numbers to fetch based on the last fetched range
  * @param {Number} totalMessages - Total number of messages in the mailbox
@@ -234,6 +243,7 @@ class SyncOperations {
                 // Fetch messages with retry logic
                 let fetchCompleted = false;
                 let fetchRetryCount = 0;
+                let fetchStartTime = Date.now();
 
                 while (!fetchCompleted) {
                     try {
@@ -330,6 +340,8 @@ class SyncOperations {
                             connErr.statusCode = 503;
                             throw connErr;
                         }
+
+                        checkFetchRetryTimeout(fetchRetryCount, fetchStartTime);
                     }
                 }
             }
@@ -383,6 +395,7 @@ class SyncOperations {
                 // only fetch messages if there are some
                 let fetchCompleted = false;
                 let fetchRetryCount = 0;
+                let fetchStartTime = Date.now();
                 while (!fetchCompleted) {
                     // Get fresh imapClient reference inside retry loop
                     let imapClient = this.connection.imapClient;
@@ -483,6 +496,8 @@ class SyncOperations {
                             connErr.statusCode = 503;
                             throw connErr;
                         }
+
+                        checkFetchRetryTimeout(fetchRetryCount, fetchStartTime);
                     }
                 }
             }
@@ -553,6 +568,8 @@ class SyncOperations {
                 let range = false;
                 let lastHighestUid = 0;
                 let batchNumber = 0;
+                let fetchStartTime = Date.now();
+                let totalFetchRetryCount = 0;
                 // process messages in batches
                 while ((range = getFetchRange(mailboxStatus.messages, range))) {
                     batchNumber++;
@@ -738,6 +755,7 @@ class SyncOperations {
                             }
 
                             // Retry with exponential backoff
+                            totalFetchRetryCount++;
                             const fetchRetryDelay = calculateFetchBackoff(++fetchRetryCount);
                             this.logger.error({ msg: `FETCH failed, retrying in ${Math.round(fetchRetryDelay / 1000)}s`, loopId, batchNumber });
                             await new Promise(r => setTimeout(r, fetchRetryDelay));
@@ -776,6 +794,8 @@ class SyncOperations {
                                 newMessages: mailboxStatus.messages,
                                 range
                             });
+
+                            checkFetchRetryTimeout(totalFetchRetryCount, fetchStartTime);
                         }
                     }
                 }

package/lib/email-client/imap-client.js

@@ -299,7 +299,9 @@ class IMAPClient extends BaseClient {
 
             // Set up error handling for the command connection
             const onErr = err => {
-                commandClient?.log.warn({ msg: 'IMAP connection error', cid: commandCid, channel: 'command', account: this.account, err });
+                // Idle timeout (ETIMEOUT) is expected - the command client has no keepalive
+                const logLevel = err.code === 'ETIMEOUT' ? 'debug' : 'warn';
+                commandClient?.log[logLevel]({ msg: 'IMAP connection error', cid: commandCid, channel: 'command', account: this.account, err });
                 commandClient.close();
                 this.commandClient = null;
             };
@@ -1214,9 +1216,14 @@ class IMAPClient extends BaseClient {
         }
 
         // Provider-specific workarounds
-        if (/(\.rambler\.ru|\.163\.com)$/i.test(imapConfig.host)) {
-            // Special case for Rambler and 163. Break IDLE at least once a minute
+        if (/\.rambler\.ru$/i.test(imapConfig.host)) {
+            // Rambler supports IDLE but needs it restarted frequently
             imapConfig.maxIdleTime = 55 * 1000;
+        } else if (/\.163\.com$/i.test(imapConfig.host)) {
+            // 163.com (Coremail) does not support IDLE and ignores NOOP for idle timer reset.
+            // Use STATUS as the keepalive command - it counts as real activity for the server.
+            imapConfig.maxIdleTime = 55 * 1000;
+            imapConfig.missingIdleCommand = 'STATUS';
         } else if (/\.yahoo\.com$/i.test(imapConfig.host)) {
             // Special case for Yahoo. Break IDLE at least once every three minutes
             imapConfig.maxIdleTime = 3 * 60 * 1000;
@@ -2468,6 +2475,9 @@ class IMAPClient extends BaseClient {
         const mailboxes = [];
 
         const listing = await this.getCurrentListing(false, { allowSecondary: true });
+        // Capture reference so we can close it after the await-heavy loop below
+        // without risking closure of a different command client obtained by a concurrent caller
+        const commandClientForListing = this.commandClient;
 
         // Process each configured subconnection
         for (const path of accountData.subconnections || []) {
@@ -2595,6 +2605,12 @@ class IMAPClient extends BaseClient {
             await subconnection.init();
         }
 
+        // Close the command client if it is still the same instance we captured above
+        if (commandClientForListing && this.commandClient === commandClientForListing) {
+            this.commandClient.close();
+            this.commandClient = null;
+        }
+
         return this.subconnections.length;
     }
 

package/lib/email-client/outlook-client.js

@@ -3857,8 +3857,11 @@ class OutlookClient extends BaseClient {
     async processHistory() {
         let event;
         let newMessageOptions;
+        let processedAny = false;
 
         while ((event = await this.accountObject.pullQueueEvent()) !== null) {
+            processedAny = true;
+
             switch (event.type) {
                 case 'updated':
                     await this.processUpdatedMessage(event.message);
@@ -3900,6 +3903,157 @@ class OutlookClient extends BaseClient {
                     break;
             }
         }
+
+        if (processedAny) {
+            await this._updateLastNotificationTime();
+        }
+    }
+
+    async _updateLastNotificationTime() {
+        try {
+            await this.redis.hset(this.getAccountKey(), 'outlookLastNotification', Date.now().toString());
+        } catch (err) {
+            this.logger.error({ msg: 'Failed to update last notification time', err });
+        }
+    }
+
+    /**
+     * Recover messages missed due to failed webhook delivery.
+     * Triggered by MS Graph 'missed' lifecycle event.
+     * Queries recent messages and processes any not already seen via normal webhooks.
+     * @returns {boolean} Whether recovery ran (false if debounced)
+     */
+    async syncMissedMessages() {
+        // Debounce: skip if recovery already ran recently
+        const cooldownKey = `${REDIS_PREFIX}iam:${this.account}:missedRecovery`;
+        const acquired = await this.redis.set(cooldownKey, '1', 'EX', 300, 'NX');
+        if (!acquired) {
+            this.logger.debug({ msg: 'Missed notification recovery skipped (cooldown active)', account: this.account });
+            return false;
+        }
+
+        // Determine lookback window
+        let sinceTime;
+        const lastNotification = await this.redis.hget(this.getAccountKey(), 'outlookLastNotification');
+        if (lastNotification) {
+            // 2-minute safety margin before last known good notification
+            sinceTime = new Date(Number(lastNotification) - 2 * 60 * 1000);
+        } else {
+            sinceTime = new Date(Date.now() - 30 * 60 * 1000);
+        }
+
+        this.logger.info({
+            msg: 'Recovering missed notifications',
+            account: this.account,
+            since: sinceTime.toISOString()
+        });
+
+        let messages = [];
+        let reqUrl = `/${this.oauth2UserPath}/messages`;
+        let reqPayload = {
+            $filter: `receivedDateTime gt ${sinceTime.toISOString()}`,
+            $select: 'id,parentFolderId',
+            $top: 250,
+            $orderby: 'receivedDateTime desc'
+        };
+
+        try {
+            while (reqUrl) {
+                let pageResult = await this.request(reqUrl, 'get', reqPayload);
+
+                if (pageResult?.value?.length) {
+                    messages.push(...pageResult.value);
+                    if (messages.length >= 1000) {
+                        messages.length = 1000;
+                        break;
+                    }
+                }
+
+                if (pageResult['@odata.nextLink']) {
+                    reqUrl = pageResult['@odata.nextLink'];
+                    reqPayload = null; // nextLink includes all query params
+                } else {
+                    reqUrl = null;
+                }
+            }
+        } catch (err) {
+            this.logger.error({ msg: 'Failed to query recent messages for missed notification recovery', account: this.account, err });
+            await this.redis.del(cooldownKey);
+            return false;
+        }
+
+        if (!messages.length) {
+            this.logger.debug({ msg: 'No recent messages found during missed notification recovery', account: this.account });
+            return true;
+        }
+
+        this.logger.info({
+            msg: 'Found recent messages for missed notification recovery',
+            account: this.account,
+            count: messages.length
+        });
+
+        let newMessageOptions = await this.getMessageFetchOptions();
+        newMessageOptions.showPath = true;
+
+        // Pre-resolve unique folder IDs once to avoid per-message Redis lookups
+        let folderMap = new Map();
+        let cachedListing = await this.getCachedMailboxListing();
+        if (cachedListing) {
+            for (let msg of messages) {
+                if (msg.parentFolderId && !folderMap.has(msg.parentFolderId)) {
+                    let folder = cachedListing.find(entry => entry.id === msg.parentFolderId);
+                    if (folder) {
+                        folderMap.set(msg.parentFolderId, folder);
+                    }
+                }
+            }
+        }
+
+        let recoveredCount = 0;
+
+        for (const msg of messages) {
+            try {
+                // Pre-filter: check rollingBucketLock BEFORE the expensive getMessage() call
+                let preFilterChecked = false;
+                let folder = folderMap.get(msg.parentFolderId);
+                if (folder) {
+                    let recentlySeen = await this.rollingBucketLock(`${msg.id}:created`, folder.pathName);
+                    if (recentlySeen) {
+                        continue;
+                    }
+                    preFilterChecked = true;
+                }
+
+                let messageData = await this.prepareNewMessage(msg.id, newMessageOptions);
+                if (!messageData) {
+                    continue;
+                }
+
+                if (!preFilterChecked) {
+                    let recentlySeen = await this.rollingBucketLock(`${messageData.id}:created`, messageData.path);
+                    if (recentlySeen) {
+                        continue;
+                    }
+                }
+
+                await this.processNew(messageData, newMessageOptions);
+                recoveredCount++;
+            } catch (err) {
+                this.logger.error({ msg: 'Failed to process message during missed notification recovery', emailId: msg.id, account: this.account, err });
+            }
+        }
+
+        this.logger.info({
+            msg: 'Missed notification recovery completed',
+            account: this.account,
+            checked: messages.length,
+            recovered: recoveredCount
+        });
+
+        await this._updateLastNotificationTime();
+
+        return true;
     }
 
     /**

package/lib/passkeys.js

@@ -48,6 +48,47 @@ async function fetchCredentialsBySet(setKey) {
     return credentials;
 }
 
+redis.defineCommand('webauthnSaveIfUnderLimit', {
+    numberOfKeys: 3,
+    lua: `
+local userSetKey = KEYS[1]
+local credKey = KEYS[2]
+local allSetKey = KEYS[3]
+local maxCount = tonumber(ARGV[1])
+local credId = ARGV[2]
+
+local currentCount = redis.call('SCARD', userSetKey)
+if currentCount >= maxCount then
+    return 0
+end
+
+redis.call('SADD', userSetKey, credId)
+redis.call('SADD', allSetKey, credId)
+redis.call('HSET', credKey, unpack(ARGV, 3))
+return 1
+`
+});
+
+function serializeCredential({ id, publicKey, counter, transports, name, user }) {
+    return {
+        id,
+        publicKey,
+        counter: String(counter),
+        transports: JSON.stringify(transports || []),
+        name: name || 'Unnamed passkey',
+        user,
+        createdAt: new Date().toISOString()
+    };
+}
+
+function credentialKeys(id, user) {
+    return {
+        credKey: `${KEY_PREFIX}cred:${id}`,
+        userSetKey: `${KEY_PREFIX}creds:${user}`,
+        allSetKey: `${KEY_PREFIX}all`
+    };
+}
+
 module.exports = {
     async getRpConfig() {
         let serviceUrl = await settings.get('serviceUrl');
@@ -73,26 +114,26 @@ module.exports = {
         return challenge || null;
     },
 
-    async saveCredential({ id, publicKey, counter, transports, name, user }) {
-        let credKey = `${KEY_PREFIX}cred:${id}`;
-        let userSetKey = `${KEY_PREFIX}creds:${user}`;
-        let allSetKey = `${KEY_PREFIX}all`;
+    async saveCredential(cred) {
+        let { credKey, userSetKey, allSetKey } = credentialKeys(cred.id, cred.user);
 
         let pipeline = redis.pipeline();
-        pipeline.hset(credKey, {
-            id,
-            publicKey,
-            counter: String(counter),
-            transports: JSON.stringify(transports || []),
-            name: name || 'Unnamed passkey',
-            user,
-            createdAt: new Date().toISOString()
-        });
-        pipeline.sadd(userSetKey, id);
-        pipeline.sadd(allSetKey, id);
+        pipeline.hset(credKey, serializeCredential(cred));
+        pipeline.sadd(userSetKey, cred.id);
+        pipeline.sadd(allSetKey, cred.id);
         await pipeline.exec();
     },
 
+    async saveCredentialIfUnderLimit(cred, maxCount) {
+        let { credKey, userSetKey, allSetKey } = credentialKeys(cred.id, cred.user);
+        let fields = serializeCredential(cred);
+        let fieldArgs = Object.entries(fields).flat();
+
+        let added = await redis.webauthnSaveIfUnderLimit(userSetKey, credKey, allSetKey, maxCount, cred.id, ...fieldArgs);
+
+        return !!added;
+    },
+
     async getCredential(credentialId) {
         if (!credentialId || typeof credentialId !== 'string') {
             return null;

package/lib/routes-ui.js

@@ -4258,20 +4258,22 @@ ${Buffer.from(data.content, 'base64url').toString('base64')}
                 }
 
                 let authData = await settings.get('authData');
-                if (authData && authData.password) {
-                    if (!request.payload.password) {
-                        return h.response({ error: 'Current password is required' }).code(403);
-                    }
-                    let valid;
-                    try {
-                        valid = await pbkdf2.verify(authData.password, request.payload.password);
-                    } catch (err) {
-                        request.logger.error({ msg: 'Failed to verify password for passkey registration', err });
-                        valid = false;
-                    }
-                    if (!valid) {
-                        return h.response({ error: 'Invalid password' }).code(403);
-                    }
+                if (!authData || !authData.password) {
+                    return h.response({ error: 'Account password must be configured before registering passkeys' }).code(403);
+                }
+
+                if (!request.payload.password) {
+                    return h.response({ error: 'Current password is required' }).code(403);
+                }
+                let valid;
+                try {
+                    valid = await pbkdf2.verify(authData.password, request.payload.password);
+                } catch (err) {
+                    request.logger.error({ msg: 'Failed to verify password for passkey registration', err });
+                    valid = false;
+                }
+                if (!valid) {
+                    return h.response({ error: 'Invalid password' }).code(403);
                 }
 
                 let { rpId, origin } = await passkeys.getRpConfig();
@@ -4359,19 +4361,26 @@ ${Buffer.from(data.content, 'base64url').toString('base64')}
                 let { credential } = verification.registrationInfo;
                 let user = (request.auth && request.auth.credentials && request.auth.credentials.user) || 'admin';
 
-                let credentialCount = await passkeys.countCredentials(user);
-                if (credentialCount >= consts.MAX_PASSKEYS_PER_USER) {
-                    return h.response({ error: 'Maximum number of passkeys reached' }).code(400);
+                let authData = await settings.get('authData');
+                if (!authData || !authData.password) {
+                    return h.response({ error: 'Account password must be configured before registering passkeys' }).code(403);
                 }
 
-                await passkeys.saveCredential({
-                    id: credential.id,
-                    publicKey: Buffer.from(credential.publicKey).toString('base64url'),
-                    counter: credential.counter,
-                    transports: credential.transports || [],
-                    name: request.payload.name,
-                    user
-                });
+                let saved = await passkeys.saveCredentialIfUnderLimit(
+                    {
+                        id: credential.id,
+                        publicKey: Buffer.from(credential.publicKey).toString('base64url'),
+                        counter: credential.counter,
+                        transports: credential.transports || [],
+                        name: request.payload.name,
+                        user
+                    },
+                    consts.MAX_PASSKEYS_PER_USER
+                );
+
+                if (!saved) {
+                    return h.response({ error: 'Maximum number of passkeys reached' }).code(400);
+                }
 
                 request.logger.info({
                     msg: 'Passkey registered',

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "emailengine-app",
-    "version": "2.66.0",
+    "version": "2.67.0",
     "private": false,
     "productTitle": "EmailEngine",
     "description": "Email Sync Engine",