emailengine-app 2.68.1 → 2.69.0

package/lib/email-client/imap/sync-operations.js

@@ -1,9 +1,9 @@
 'use strict';
 
 const crypto = require('crypto');
-const { compareExisting, calculateFetchBackoff, readEnvValue } = require('../../tools');
+const { compareExisting, calculateFetchBackoff, readEnvValue, validUidValidity } = require('../../tools');
 const config = require('@zone-eu/wild-config');
-const { DEFAULT_FETCH_BATCH_SIZE, FETCH_RETRY_MAX_TIME, FETCH_RETRY_MIN_ATTEMPTS } = require('../../consts');
+const { DEFAULT_FETCH_BATCH_SIZE, FETCH_RETRY_MAX_TIME, FETCH_RETRY_MIN_ATTEMPTS, MAILBOX_RESET_NOTIFY } = require('../../consts');
 
 // Configurable batch size for fetching messages (default: 250)
 const FETCH_BATCH_SIZE = Number(readEnvValue('EENGINE_FETCH_BATCH_SIZE') || config.service.fetchBatchSize) || DEFAULT_FETCH_BATCH_SIZE;
@@ -111,6 +111,32 @@ function canSkipSync(storedStatus, mailboxStatus) {
     return storedStatus.messages === mailboxStatus.messages && storedStatus.uidNext === mailboxStatus.uidNext && isRecentFullSync(storedStatus);
 }
 
+/**
+ * Determines whether a folder with no stored sync state should be rebuilt SILENTLY
+ * (recording existing messages without emitting messageNew) instead of replayed as new mail.
+ *
+ * This is the "lost index recovery" case: the account has already completed a previous
+ * connected session, yet this folder's mailbox hash is gone entirely while the server
+ * still reports messages. That mismatch means EmailEngine's per-mailbox state was lost
+ * (e.g. Redis evicted it) - replaying those already-synced messages as new would flood
+ * the webhook queue. A genuine first sync keeps `previouslyConnected <= 1` and is left
+ * to the normal notifyFrom-bounded path so intentional backfills still work.
+ *
+ * The decision keys on `hasStoredState` (did the hash hold ANY field), not on individual
+ * fields: Redis eviction removes whole keys, never single hash fields, while individual
+ * fields can legitimately stay absent forever - most notably `uidNext` on servers that
+ * omit UIDNEXT from the SELECT response (updateStoredStatus skips persisting falsy
+ * values), which must not re-trigger the reseed on every open.
+ *
+ * @param {Object} storedStatus - Stored mailbox status from getStoredStatus()
+ * @param {Object} mailboxStatus - Current mailbox status from IMAP
+ * @param {Number} previouslyConnected - Account-level connected-session counter (`state:count:connected`)
+ * @returns {Boolean} True if the index should be silently reseeded
+ */
+function shouldSeedLostIndex(storedStatus, mailboxStatus, previouslyConnected) {
+    return Number(previouslyConnected) > 1 && !(storedStatus && storedStatus.hasStoredState) && Number(mailboxStatus.messages) > 0;
+}
+
 /**
  * Handles the synchronization of IMAP mailboxes
  */
@@ -192,6 +218,107 @@ class SyncOperations {
         return { type: 'full', reason: 'changes_detected' };
     }
 
+    /**
+     * Rebuilds the mailbox baseline from the server's current state WITHOUT emitting
+     * messageNew notifications. Used when the stored sync state was lost (e.g. Redis evicted
+     * it) or invalidated (UIDVALIDITY change), then emits a single mailboxReset event so
+     * integrators can reconcile.
+     *
+     * Full indexer mode records every message on the server in the message index so only
+     * mail arriving afterwards is advertised as new. Fast indexer mode never maintains that
+     * index - the only baseline it needs is the stored uidNext that runFastSync compares
+     * against, so no per-message fetch is performed.
+     *
+     * @param {Object} mailboxStatus - Current mailbox status from IMAP
+     * @param {Object} [options]
+     * @param {String} [options.reason] - Why the reseed happened (included in the mailboxReset payload and logs)
+     * @param {String|Boolean} [options.prevUidValidity] - Previous UIDVALIDITY, when reseeding after a UIDVALIDITY change
+     * @returns {Number} Count of indexed messages (always 0 in fast indexer mode)
+     */
+    async seedMailboxIndex(mailboxStatus, options) {
+        options = options || {};
+        let reason = options.reason || 'syncStateLost';
+
+        let imapClient = this.connection.imapClient;
+        if (!imapClient) {
+            let err = new Error('IMAP connection not available');
+            err.code = 'IMAPConnectionClosing';
+            err.statusCode = 503;
+            throw err;
+        }
+
+        let imapIndexer = this.mailbox.imapIndexer;
+
+        let lock = await this.mailbox.getMailboxLock(null, { description: 'Seed mailbox index' });
+        this.connection.syncing = true;
+        this.mailbox.syncing = true;
+        try {
+            // Drop any stale queued notifications - we are establishing a fresh baseline
+            await this.connection.redis.del(this.mailbox.getNotificationsKey());
+
+            let indexed = 0;
+
+            if (imapIndexer === 'fast') {
+                // Fast mode detects new messages by comparing UIDs against the stored uidNext
+                // (hUpdateBigger in runFastSync), so that value is the baseline to establish.
+                // updateStoredStatus below persists it; when the server omits UIDNEXT from the
+                // SELECT response, derive the baseline from the highest UID on the server -
+                // otherwise the next runFastSync would replay every message as messageNew.
+                if (!mailboxStatus.uidNext && Number(mailboxStatus.messages) > 0) {
+                    let lastMessage = await imapClient.fetchOne('*', { uid: true });
+                    if (lastMessage && lastMessage.uid) {
+                        mailboxStatus = Object.assign({}, mailboxStatus, { uidNext: lastMessage.uid + 1 });
+                    }
+                }
+            } else {
+                let fields = { uid: true, flags: true, modseq: true, emailId: true, labels: true, internalDate: true };
+
+                // Record every message currently on the server as already-seen, without queuing notifications
+                for await (let messageData of imapClient.fetch('1:*', fields, { uid: true })) {
+                    if (!messageData || !messageData.uid) {
+                        continue;
+                    }
+                    // ignore Recent flag
+                    messageData.flags.delete('\\Recent');
+                    await this.mailbox.entryListSet(messageData);
+                    indexed++;
+                }
+            }
+
+            // Persist the current server state as the new baseline. updateStoredStatus also sets
+            // initialUidNext (via hSetNew) so future arrivals are detected normally.
+            await this.mailbox.updateStoredStatus(mailboxStatus);
+
+            this.logger.warn({
+                msg: 'Rebuilt mailbox baseline without notifications',
+                action: 'lost_index_recovery',
+                path: this.mailbox.path,
+                imapIndexer,
+                messages: imapIndexer === 'fast' ? Number(mailboxStatus.messages) || 0 : indexed,
+                reason
+            });
+
+            // Inform integrators that this folder was reset so they can reconcile if needed
+            let resetPayload = {
+                path: this.mailbox.listingEntry.path,
+                name: this.mailbox.listingEntry.name,
+                specialUse: this.mailbox.listingEntry.specialUse || false,
+                uidValidity: validUidValidity(mailboxStatus.uidValidity) ? mailboxStatus.uidValidity.toString() : false,
+                reason
+            };
+            if (typeof options.prevUidValidity !== 'undefined') {
+                resetPayload.prevUidValidity = options.prevUidValidity;
+            }
+            await this.connection.notify(this.mailbox, MAILBOX_RESET_NOTIFY, resetPayload);
+
+            return indexed;
+        } finally {
+            lock.release();
+            this.connection.syncing = false;
+            this.mailbox.syncing = false;
+        }
+    }
+
     /**
      * Fast sync mode - only tracks new messages, doesn't maintain full message list
      * More efficient for large mailboxes where we only care about new messages
@@ -856,6 +983,7 @@ module.exports = {
     canUseCondstorePartialSync,
     canUseSimplePartialSync,
     canSkipSync,
+    shouldSeedLostIndex,
     FETCH_BATCH_SIZE,
     FULL_SYNC_DELAY
 };

package/lib/email-client/imap-client.js

@@ -790,69 +790,116 @@ class IMAPClient extends BaseClient {
             this.main = mainList.sort((a, b) => a.index - b.index)[0].entry;
         }
 
-        // Set up event handlers for IMAP notifications
+        // Set up event handlers for IMAP notifications. Each listener delegates to a
+        // method that wraps the whole body (including normalizePath/lookup) in
+        // try/catch, so a malformed event can never become an unhandled rejection
+        // that kills the worker.
 
         // Process untagged EXISTS responses (new messages)
-        imapClient.on('exists', async event => {
-            if (!event || !event.path || !this.mailboxes.has(normalizePath(event.path))) {
-                return; //?
-            }
-
-            let mailbox = this.mailboxes.get(normalizePath(event.path));
-            try {
-                await mailbox.onExists(event);
-            } catch (err) {
-                imapClient.log.error({ msg: 'Exists error', err });
-            }
-        });
+        imapClient.on('exists', event => this.handleExistsEvent(imapClient, event));
 
         // Handle mailbox open events
-        imapClient.on('mailboxOpen', async event => {
-            if (!event || !event.path || !this.mailboxes.has(normalizePath(event.path))) {
-                return; //?
-            }
-
-            let mailbox = this.mailboxes.get(normalizePath(event.path));
-            try {
-                await mailbox.onOpen(event);
-            } catch (err) {
-                if (err.code === 'IMAPConnectionClosing') {
-                    imapClient.log.debug({ msg: 'Open skipped, connection closing', err });
-                } else {
-                    imapClient.log.error({ msg: 'Open error', err });
-                }
-            }
-        });
+        imapClient.on('mailboxOpen', event => this.handleMailboxOpenEvent(imapClient, event));
 
         // Handle mailbox close events
-        imapClient.on('mailboxClose', async event => {
-            if (!event || !event.path || !this.mailboxes.has(normalizePath(event.path))) {
-                return; //?
-            }
+        imapClient.on('mailboxClose', event => this.handleMailboxCloseEvent(imapClient, event));
 
-            let mailbox = this.mailboxes.get(normalizePath(event.path));
-            try {
-                await mailbox.onClose(event);
-            } catch (err) {
-                imapClient.log.error({ msg: 'Close error', err });
+        // Handle flag changes
+        imapClient.on('flags', event => this.handleFlagsEvent(imapClient, event));
+
+        return response;
+    }
+
+    /**
+     * Resolves the tracked Mailbox for an IMAP event, or null when the event has no
+     * tracked path. May throw on a malformed path - callers must run it inside a
+     * try/catch (see the handle*Event methods below).
+     * @param {Object} event - IMAP event with a `path` property
+     * @returns {Mailbox|null} Tracked mailbox or null
+     */
+    resolveEventMailbox(event) {
+        if (!event || !event.path) {
+            return null;
+        }
+
+        let path = normalizePath(event.path);
+        if (!this.mailboxes.has(path)) {
+            return null;
+        }
+
+        return this.mailboxes.get(path);
+    }
+
+    /**
+     * Handles an untagged EXISTS notification (new messages in a mailbox)
+     * @param {Object} imapClient - IMAP connection that emitted the event
+     * @param {Object} event - Event payload
+     */
+    async handleExistsEvent(imapClient, event) {
+        try {
+            let mailbox = this.resolveEventMailbox(event);
+            if (!mailbox) {
+                return;
             }
-        });
+            await mailbox.onExists(event);
+        } catch (err) {
+            imapClient.log.error({ msg: 'Exists error', err });
+        }
+    }
 
-        // Handle flag changes
-        imapClient.on('flags', async event => {
-            if (!event || !event.path || !this.mailboxes.has(normalizePath(event.path))) {
-                return; //?
+    /**
+     * Handles a mailbox open notification
+     * @param {Object} imapClient - IMAP connection that emitted the event
+     * @param {Object} event - Event payload
+     */
+    async handleMailboxOpenEvent(imapClient, event) {
+        try {
+            let mailbox = this.resolveEventMailbox(event);
+            if (!mailbox) {
+                return;
             }
+            await mailbox.onOpen(event);
+        } catch (err) {
+            if (err.code === 'IMAPConnectionClosing') {
+                imapClient.log.debug({ msg: 'Open skipped, connection closing', err });
+            } else {
+                imapClient.log.error({ msg: 'Open error', err });
+            }
+        }
+    }
 
-            let mailbox = this.mailboxes.get(normalizePath(event.path));
-            try {
-                await mailbox.onFlags(event);
-            } catch (err) {
-                imapClient.log.error({ msg: 'Flags error', err });
+    /**
+     * Handles a mailbox close notification
+     * @param {Object} imapClient - IMAP connection that emitted the event
+     * @param {Object} event - Event payload
+     */
+    async handleMailboxCloseEvent(imapClient, event) {
+        try {
+            let mailbox = this.resolveEventMailbox(event);
+            if (!mailbox) {
+                return;
             }
-        });
+            await mailbox.onClose(event);
+        } catch (err) {
+            imapClient.log.error({ msg: 'Close error', err });
+        }
+    }
 
-        return response;
+    /**
+     * Handles a flag change notification
+     * @param {Object} imapClient - IMAP connection that emitted the event
+     * @param {Object} event - Event payload
+     */
+    async handleFlagsEvent(imapClient, event) {
+        try {
+            let mailbox = this.resolveEventMailbox(event);
+            if (!mailbox) {
+                return;
+            }
+            await mailbox.onFlags(event);
+        } catch (err) {
+            imapClient.log.error({ msg: 'Flags error', err });
+        }
     }
 
     /**
@@ -1599,6 +1646,12 @@ class IMAPClient extends BaseClient {
         } finally {
             this.isClosing = false;
             this.isClosed = true;
+
+            // Tear down subconnections just like close() does, otherwise their IMAP
+            // clients keep reconnecting to the now-deleted account. This runs in the
+            // finally so a mailbox or Redis error above cannot leave them alive
+            // (closeSubconnections is internally guarded and never throws).
+            this.closeSubconnections();
         }
 
         this.logger.info({ msg: 'Closed account', account: this.account });
@@ -2390,12 +2443,18 @@ class IMAPClient extends BaseClient {
                 });
             }
 
-            // Pack response data
+            // Pack response data. ImapFlow's append() returns `destination` as the
+            // append target and `path` as the currently selected mailbox (often the
+            // idling INBOX), so prefer `destination` when building the response path
+            // and message id. Otherwise uploads to a non-selected folder would report
+            // an id that points at the selected mailbox and fails to resolve.
+            let destinationPath = uploadResponse.destination || uploadResponse.path || data.path;
+
             if (uploadResponse.uid) {
-                response.id = await this.packUid(uploadResponse.path || data.path, uploadResponse.uid);
+                response.id = await this.packUid(destinationPath, uploadResponse.uid);
             }
 
-            response.path = uploadResponse.path;
+            response.path = destinationPath;
 
             if (uploadResponse.uid) {
                 response.uid = uploadResponse.uid;
@@ -2448,12 +2507,11 @@ class IMAPClient extends BaseClient {
      * @param {Object} payload - Expunge event data
      */
     async expungeHandler(payload) {
-        if (!payload || !payload.path || !this.mailboxes.has(normalizePath(payload.path))) {
-            return; //?
-        }
-
-        let mailbox = this.mailboxes.get(normalizePath(payload.path));
         try {
+            let mailbox = this.resolveEventMailbox(payload);
+            if (!mailbox) {
+                return;
+            }
             await mailbox.onExpunge(payload);
         } catch (err) {
             this.logger.error({ msg: 'Expunge error', err });

package/lib/email-client/outlook-client.js

@@ -41,6 +41,27 @@ const MAX_BATCH_SIZE = graphApi.MAX_BATCH_SIZE;
 // Subscription is renewed automatically. But just in case, check once in an hour
 const RENEW_WATCH_TTL = 60 * 60 * 1000; // 1h
 
+// MS Graph folder hierarchy is defined by id/parentFolderId, not by "/" in folder
+// names. To build an unambiguous, reversible pathName we percent-encode "%" and "/"
+// inside a single displayName segment, keeping the real "/" as the delimiter BETWEEN
+// segments. Encode "%" first then "/"; decode "%2F" first then "%25" so that a folder
+// name literally containing "%2F" or "%25" round-trips correctly.
+// Both helpers are pure and exception-safe: the typeof guard returns non-string input
+// unchanged, and replaceAll on a string uses literal (not regex) replacement.
+function encodeFolderSegment(name) {
+    if (typeof name !== 'string') {
+        return name;
+    }
+    return name.replaceAll('%', '%25').replaceAll('/', '%2F');
+}
+
+function decodeFolderSegment(segment) {
+    if (typeof segment !== 'string') {
+        return segment;
+    }
+    return segment.replaceAll('%2F', '/').replaceAll('%25', '%');
+}
+
 /*
 Supported operations status:
 ✅ listMessages - with paging (cursor + page nr) and search queries (no support for to/cc/bcc queries)
@@ -625,7 +646,8 @@ class OutlookClient extends BaseClient {
                 'ccRecipients',
                 'bccRecipients',
                 'internetMessageId',
-                'bodyPreview'
+                'bodyPreview',
+                'categories'
             ];
             expandFields = 'attachments($select=id,name,contentType,size,isInline,microsoft.graph.fileAttachment/contentId)';
         }
@@ -647,6 +669,7 @@ class OutlookClient extends BaseClient {
 
         let useOutlookSearch = false;
         let skipToken = null;
+        let labelFilterActive = false;
 
         // Handle search queries
         if (query.search) {
@@ -668,15 +691,48 @@ class OutlookClient extends BaseClient {
                     // we need to have receivedDateTime as the first filtering property, otherwise ordering will fail
                     requestQuery.$filter = `receivedDateTime gt 1970-01-01T00:00:00.000Z and ${$filter}`;
                 }
+                // Category (label) filters compile to a categories/any() lambda. Some mailboxes refuse
+                // to combine that with $orderBy; remember it so we can retry without ordering below.
+                labelFilterActive = !!(query.search.labels && [].concat(query.search.labels.has || [], query.search.labels.not || []).some(Boolean));
             }
         }
 
         let messages;
         let totalMessages;
 
+        let messagesUrl = `/${this.oauth2UserPath}/${folder ? `mailFolders/${folder.id}/` : ''}messages`;
+        // The "not" form of a category filter may require advanced query capabilities; the header is
+        // harmless for the "has" form and for queries without a label filter we omit it entirely.
+        let requestOptions = labelFilterActive ? { headers: { ConsistencyLevel: 'eventual' } } : undefined;
+
         // Execute the message list request
         try {
-            let listing = await this.request(`/${this.oauth2UserPath}/${folder ? `mailFolders/${folder.id}/` : ''}messages`, 'get', requestQuery);
+            let listing;
+            try {
+                listing = await this.request(messagesUrl, 'get', requestQuery, requestOptions);
+            } catch (err) {
+                // A categories/any() filter combined with $orderBy can be rejected as too complex
+                // ("InefficientFilter"). Retry once without server-side ordering and sort the page locally.
+                let graphCode = err?.oauthRequest?.response?.error?.code || '';
+                let graphMessage = err?.oauthRequest?.response?.error?.message || '';
+                let inefficientFilter = /inefficientfilter/i.test(graphCode) || /restriction or sort order is too complex/i.test(graphMessage);
+
+                if (labelFilterActive && requestQuery.$orderBy && inefficientFilter) {
+                    this.logger.warn({
+                        msg: 'Graph rejected ordered category filter, retrying without server-side ordering',
+                        account: this.account,
+                        path
+                    });
+                    delete requestQuery.$orderBy;
+                    listing = await this.request(messagesUrl, 'get', requestQuery, requestOptions);
+                    // Without $orderBy the server no longer guarantees order; sort this page newest-first
+                    if (listing && Array.isArray(listing.value)) {
+                        listing.value.sort((a, b) => new Date(b.receivedDateTime) - new Date(a.receivedDateTime));
+                    }
+                } else {
+                    throw err;
+                }
+            }
 
             totalMessages = !isNaN(listing['@odata.count']) ? Number(listing['@odata.count']) : undefined;
 
@@ -2500,7 +2556,9 @@ class OutlookClient extends BaseClient {
 
         let subPaths = path.split('/');
 
-        let displayName = subPaths.pop();
+        // Decode the leaf so the literal folder name is sent to Graph; parent segments
+        // stay encoded for resolveFolder (which matches against the encoded pathName).
+        let displayName = decodeFolderSegment(subPaths.pop());
         let parentPath = subPaths.join('/');
 
         // Resolve parent folder if specified
@@ -2570,7 +2628,7 @@ class OutlookClient extends BaseClient {
             mailboxId: mailbox.id,
             path: []
                 .concat(parentFolder?.pathName || [])
-                .concat(mailbox.displayName)
+                .concat(encodeFolderSegment(mailbox.displayName))
                 .join('/'),
             created: true
         };
@@ -2637,8 +2695,10 @@ class OutlookClient extends BaseClient {
         if (sourceName !== destinationName) {
             let mailbox;
             try {
+                // sourceName/destinationName stay encoded for the comparison above;
+                // decode only here so Graph receives the literal folder name.
                 mailbox = await this.request(`/${this.oauth2UserPath}/mailFolders/${sourceFolder.id}`, 'patch', {
-                    displayName: destinationName
+                    displayName: decodeFolderSegment(destinationName)
                 });
                 if (!mailbox) {
                     throw new Error('Failed to rename mailbox');
@@ -3077,7 +3137,7 @@ class OutlookClient extends BaseClient {
                             if (pathNamePrefix) {
                                 entry.parentPath = pathNamePrefix;
                             }
-                            entry.pathName = `${pathNamePrefix ? `${pathNamePrefix}/` : ''}${entry.displayName}`;
+                            entry.pathName = `${pathNamePrefix ? `${pathNamePrefix}/` : ''}${encodeFolderSegment(entry.displayName)}`;
                             return entry;
                         })
                     );
@@ -3094,8 +3154,8 @@ class OutlookClient extends BaseClient {
 
             // Traverse child folders
             for (let entry of list) {
-                // do not traverse subfolders for folders with a slash in the name (would create ambiguous paths)
-                if (entry.childFolderCount && entry.displayName.indexOf('/') < 0) {
+                // pathName percent-encodes any literal "/" in the segment, so child paths stay unambiguous
+                if (entry.childFolderCount) {
                     await traverse(entry.pathName, entry.id);
                 }
             }
@@ -3103,10 +3163,8 @@ class OutlookClient extends BaseClient {
 
         await traverse();
 
-        // keep only real folders and folders that do not contain slash in the name
-        mailboxListing = mailboxListing.filter(
-            entry => (!entry['@odata.type'] || /^#?microsoft\.graph\.mailFolder$/.test(entry['@odata.type'])) && entry.displayName.indexOf('/') < 0
-        );
+        // keep only real mail folders (drop search folders and other non-mailFolder types)
+        mailboxListing = mailboxListing.filter(entry => !entry['@odata.type'] || /^#?microsoft\.graph\.mailFolder$/.test(entry['@odata.type']));
 
         return mailboxListing;
     }
@@ -4385,6 +4443,20 @@ class OutlookClient extends BaseClient {
             filterParts.push(`receivedDateTime gt ${this.formatSearchTerm(search.since || search.sentSince, false)}`);
         }
 
+        // Category (label) filters - "has" matches messages tagged with the category, "not" excludes them
+        if (search.labels && typeof search.labels === 'object') {
+            for (let category of [].concat(search.labels.has || [])) {
+                if (category) {
+                    filterParts.push(`categories/any(c:c eq ${this.formatSearchTerm(category)})`);
+                }
+            }
+            for (let category of [].concat(search.labels.not || [])) {
+                if (category) {
+                    filterParts.push(`not (categories/any(c:c eq ${this.formatSearchTerm(category)}))`);
+                }
+            }
+        }
+
         // Limited header search support
         for (let headerKey of Object.keys(search.header || {})) {
             switch (headerKey.toLowerCase().trim()) {
@@ -4522,4 +4594,4 @@ class OutlookClient extends BaseClient {
     }
 }
 
-module.exports = { OutlookClient };
+module.exports = { OutlookClient, encodeFolderSegment, decodeFolderSegment };