emailengine-app 2.61.1 → 2.61.2

package/lib/email-client/outlook-client.js

@@ -2,6 +2,7 @@
 
 const { BaseClient, metricsMeta } = require('./base-client');
 const { Account } = require('../account');
+const { checkAccountScopes } = require('../oauth/scope-checker');
 const settings = require('../settings');
 const { oauth2Apps } = require('../oauth2-apps');
 const getSecret = require('../get-secret');
@@ -13,6 +14,9 @@ const crypto = require('crypto');
 const { Gateway } = require('../gateway');
 const { detectMimeType, detectExtension } = require('nodemailer/lib/mime-funcs/mime-types');
 
+// Import Graph API module for MS Graph API handling
+const graphApi = require('./outlook/graph-api');
+
 const {
     REDIS_PREFIX,
     AUTH_ERROR_NOTIFY,
@@ -20,13 +24,17 @@ const {
     EMAIL_SENT_NOTIFY,
     OUTLOOK_EXPIRATION_TIME,
     OUTLOOK_EXPIRATION_RENEW_TIME,
+    OUTLOOK_SUBSCRIPTION_LOCK_TTL,
+    OUTLOOK_MAX_RETRY_ATTEMPTS,
+    OUTLOOK_RETRY_BASE_DELAY,
+    OUTLOOK_RETRY_MAX_DELAY,
     MESSAGE_UPDATED_NOTIFY,
     MESSAGE_DELETED_NOTIFY,
     MESSAGE_MISSING_NOTIFY
 } = require('../consts');
 
 // Maximum number of operations in a single batch request to Microsoft Graph API
-const MAX_BATCH_SIZE = 20;
+const MAX_BATCH_SIZE = graphApi.MAX_BATCH_SIZE;
 
 // Subscription is renewed automatically. But just in case, check once in an hour
 const RENEW_WATCH_TTL = 60 * 60 * 1000; // 1h
@@ -76,6 +84,7 @@ class OutlookClient extends BaseClient {
         // Flags for background processing
         this.processingHistory = null;
         this.renewWatchTimer = null;
+        this.renewalInProgress = false; // In-memory flag to prevent concurrent subscription renewals
 
         // MS Graph webhook subscription state (for metrics)
         // Possible values: 'unset', 'valid', 'expired', 'failed', 'pending'
@@ -86,87 +95,22 @@ class OutlookClient extends BaseClient {
      * Makes authenticated requests to Microsoft Graph API
      * Handles token management and error responses
      */
-    async request(...args) {
-        let result, accessToken;
-        try {
-            accessToken = await this.getToken();
-        } catch (err) {
-            this.logger.error({ msg: 'Failed to load access token', account: this.account, err });
-            throw err;
-        }
-
-        let [url, method, payload, options = {}] = args;
-
-        try {
-            if (!this.oAuth2Client) {
-                await this.getClient();
-            }
-
-            options.headers = options.headers || {};
-
-            // Build Prefer header with multiple preferences
-            // Request immutable IDs that don't change when messages are moved between folders
-            // https://learn.microsoft.com/en-us/graph/outlook-immutable-id
-            let preferValues = ['IdType="ImmutableId"'];
-
-            // If caller already set a Prefer header, merge it
-            if (options.headers.Prefer) {
-                preferValues.push(options.headers.Prefer);
-            }
-
-            options.headers.Prefer = preferValues.join(', ');
-
-            // Construct full API URL if not already absolute
-            let apiUrl = /^https:/.test(url) ? url : new URL(`/v1.0${url}`, this.oAuth2Client.apiBase).href;
-
-            result = await this.oAuth2Client.request(accessToken, apiUrl, method, payload, options);
-
-            // Track successful API request
-            metricsMeta({ account: this.account }, this.logger, 'oauth2ApiRequest', 'inc', { status: 'success', provider: 'outlook', statusCode: '200' });
-        } catch (err) {
-            // Track failed API request
-            const statusCode = String(err.oauthRequest?.status || 0);
-            metricsMeta({ account: this.account }, this.logger, 'oauth2ApiRequest', 'inc', { status: 'failure', provider: 'outlook', statusCode });
-
-            // Handle specific Graph API error codes
-            switch (err.oauthRequest?.response?.error?.code) {
-                case 'ErrorExecuteSearchStaleData': {
-                    // Search cursor has expired
-                    this.logger.error({ msg: 'Invalid or expired paging cursor', account: this.account, err });
-                    let error = new Error('Invalid or expired paging cursor');
-                    error.code = 'InvalidPagingCursor';
-                    error.statusCode = err.oauthRequest?.status || 500;
-                    throw error;
-                }
-            }
-
-            // Handle HTTP status codes
-            const status = err.oauthRequest?.status;
-            const isClientError = status >= 400 && status < 500;
-
-            switch (status) {
-                case 401:
-                    this.logger.error({ msg: 'Failed to authenticate API request', account: this.account, accessToken, err });
-                    throw err;
-
-                case 429:
-                    // Rate limiting
-                    this.logger.error({ msg: 'API request was throttled', account: this.account, err });
-                    throw err;
-
-                default:
-                    // Log client errors (4xx) at debug level - these are expected operational errors
-                    // Log server errors (5xx) and other failures at error level
-                    if (isClientError) {
-                        this.logger.debug({ msg: 'API request failed with client error', account: this.account, err });
-                    } else {
-                        this.logger.error({ msg: 'Failed to run API request', account: this.account, err });
-                    }
-                    throw err;
-            }
-        }
+    async request(url, method, payload, options = {}) {
+        return graphApi.request(this, url, method, payload, options);
+    }
 
-        return result;
+    /**
+     * Makes authenticated requests to Microsoft Graph API with automatic retry on rate limiting
+     * Implements exponential backoff using Retry-After header or default delays
+     * @param {string} url - API endpoint URL
+     * @param {string} method - HTTP method
+     * @param {object} payload - Request payload
+     * @param {object} options - Request options
+     * @param {number} options.maxRetries - Maximum number of retries (default: 3)
+     * @returns {Promise<object>} API response
+     */
+    async requestWithRetry(url, method, payload, options = {}) {
+        return graphApi.requestWithRetry(this, url, method, payload, options);
     }
 
     // PUBLIC METHODS
@@ -191,7 +135,7 @@ class OutlookClient extends BaseClient {
         // reinitialized to detect the change. Consider checking scopes periodically if dynamic
         // scope changes become common.
         const scopes = accountData.oauth2?.accessToken?.scope || accountData.oauth2?.scope || [];
-        const { hasSendScope, hasReadScope } = this.accountObject.checkAccountScopes('outlook', scopes);
+        const { hasSendScope, hasReadScope } = checkAccountScopes('outlook', scopes, this.logger);
         const isSendOnly = hasSendScope && !hasReadScope;
 
         this.logger.debug({
@@ -776,24 +720,37 @@ class OutlookClient extends BaseClient {
 
         /**
          * Submit a batch of delete operations to Graph API
+         * Uses requestWithRetry for automatic retry on rate limiting
          */
         let submitBatch = async () => {
             let responseData;
             try {
-                responseData = await this.request(`/$batch`, 'post', {
+                responseData = await this.requestWithRetry(`/$batch`, 'post', {
                     requests: batch
                 });
                 for (let response of responseData?.responses || []) {
+                    let emailId = messageMap.get(response.id);
                     if (response?.status >= 200 && response?.status < 300) {
-                        let emailId = messageMap.get(response.id);
                         if (emailId) {
                             updatedEmailIds.push(emailId);
                         }
+                    } else {
+                        // Log individual batch item failures for debugging
+                        this.logger.warn({
+                            msg: 'Batch item failed',
+                            account: this.account,
+                            operation: 'delete',
+                            emailId,
+                            status: response?.status,
+                            error: response?.body?.error
+                        });
                     }
                 }
             } catch (err) {
                 this.logger.error({
                     msg: 'Failed to run batch operation',
+                    account: this.account,
+                    operation: 'delete',
                     err
                 });
                 throw err;
@@ -810,12 +767,19 @@ class OutlookClient extends BaseClient {
             let reqId = `msg_${++idGen}`;
             messageMap.set(reqId, emailId);
 
+            // Common headers for batch requests - include ImmutableId preference
+            const batchHeaders = {
+                'Content-Type': 'application/json',
+                Prefer: 'IdType="ImmutableId"'
+            };
+
             if (force) {
                 // Permanent delete
                 return {
                     id: reqId,
                     method: 'DELETE',
-                    url: `/${this.oauth2UserPath}/messages/${emailId}`
+                    url: `/${this.oauth2UserPath}/messages/${emailId}`,
+                    headers: batchHeaders
                 };
             } else {
                 // Move to trash
@@ -824,9 +788,7 @@ class OutlookClient extends BaseClient {
                     method: 'POST',
                     url: `/${this.oauth2UserPath}/messages/${emailId}/move`,
                     body: { destinationId: 'deleteditems' },
-                    headers: {
-                        'Content-Type': 'application/json'
-                    }
+                    headers: batchHeaders
                 };
             }
         };
@@ -1044,20 +1006,31 @@ class OutlookClient extends BaseClient {
             let submitFetchBatch = async () => {
                 let responseData;
                 try {
-                    responseData = await this.request(`/$batch`, 'post', {
+                    responseData = await this.requestWithRetry(`/$batch`, 'post', {
                         requests: fetchBatch
                     });
                     for (let response of responseData?.responses || []) {
+                        let emailId = fetchMessageMap.get(response.id);
                         if (response?.status >= 200 && response?.status < 300 && response.body) {
-                            let emailId = fetchMessageMap.get(response.id);
                             if (emailId) {
                                 currentCategories.set(emailId, response.body.categories || []);
                             }
+                        } else if (emailId) {
+                            // Log individual batch item failures
+                            this.logger.warn({
+                                msg: 'Batch item failed',
+                                account: this.account,
+                                operation: 'fetchCategories',
+                                emailId,
+                                status: response?.status,
+                                error: response?.body?.error
+                            });
                         }
                     }
                 } catch (err) {
                     this.logger.error({
                         msg: 'Failed to fetch current categories',
+                        account: this.account,
                         err
                     });
                 }
@@ -1071,7 +1044,10 @@ class OutlookClient extends BaseClient {
                 fetchBatch.push({
                     id: reqId,
                     method: 'GET',
-                    url: `/${this.oauth2UserPath}/messages/${emailId}?$select=categories`
+                    url: `/${this.oauth2UserPath}/messages/${emailId}?$select=categories`,
+                    headers: {
+                        Prefer: 'IdType="ImmutableId"'
+                    }
                 });
 
                 if (fetchBatch.length >= MAX_BATCH_SIZE) {
@@ -1092,20 +1068,32 @@ class OutlookClient extends BaseClient {
         let submitBatch = async () => {
             let responseData;
             try {
-                responseData = await this.request(`/$batch`, 'post', {
+                responseData = await this.requestWithRetry(`/$batch`, 'post', {
                     requests: batch
                 });
                 for (let response of responseData?.responses || []) {
+                    let emailId = messageMap.get(response.id);
                     if (response?.status >= 200 && response?.status < 300) {
-                        let emailId = messageMap.get(response.id);
                         if (emailId) {
                             updatedEmailIds.push(emailId);
                         }
+                    } else {
+                        // Log individual batch item failures for debugging
+                        this.logger.warn({
+                            msg: 'Batch item failed',
+                            account: this.account,
+                            operation: 'update',
+                            emailId,
+                            status: response?.status,
+                            error: response?.body?.error
+                        });
                     }
                 }
             } catch (err) {
                 this.logger.error({
                     msg: 'Failed to run batch operation',
+                    account: this.account,
+                    operation: 'update',
                     err
                 });
                 throw err;
@@ -1150,7 +1138,8 @@ class OutlookClient extends BaseClient {
                 url: `/${this.oauth2UserPath}/messages/${emailId}`,
                 body: bodyUpdates,
                 headers: {
-                    'Content-Type': 'application/json'
+                    'Content-Type': 'application/json',
+                    Prefer: 'IdType="ImmutableId"'
                 }
             };
         };
@@ -1272,20 +1261,32 @@ class OutlookClient extends BaseClient {
         let submitBatch = async () => {
             let responseData;
             try {
-                responseData = await this.request(`/$batch`, 'post', {
+                responseData = await this.requestWithRetry(`/$batch`, 'post', {
                     requests: batch
                 });
                 for (let response of responseData?.responses || []) {
+                    let emailId = messageMap.get(response.id);
                     if (response?.status >= 200 && response?.status < 300) {
-                        let emailId = messageMap.get(response.id);
                         if (emailId) {
                             updatedEmailIds.push(emailId);
                         }
+                    } else {
+                        // Log individual batch item failures for debugging
+                        this.logger.warn({
+                            msg: 'Batch item failed',
+                            account: this.account,
+                            operation: 'move',
+                            emailId,
+                            status: response?.status,
+                            error: response?.body?.error
+                        });
                     }
                 }
             } catch (err) {
                 this.logger.error({
                     msg: 'Failed to run batch operation',
+                    account: this.account,
+                    operation: 'move',
                     err
                 });
                 throw err;
@@ -1304,7 +1305,8 @@ class OutlookClient extends BaseClient {
                 url: `/${this.oauth2UserPath}/messages/${emailId}/move`,
                 body: { destinationId: targetFolder.id },
                 headers: {
-                    'Content-Type': 'application/json'
+                    'Content-Type': 'application/json',
+                    Prefer: 'IdType="ImmutableId"'
                 }
             };
         };
@@ -2522,8 +2524,24 @@ class OutlookClient extends BaseClient {
             try {
                 deltaRes = await this.request(deltaReqUrl);
             } catch (err) {
-                this.logger.error({ msg: 'Failed to check mailbox folder delta', err });
-                // might be faulty entry, so clear it
+                const errorCode = err.oauthRequest?.response?.error?.code;
+
+                // Handle delta token expiration - MS Graph returns these codes when delta is stale
+                if (errorCode === 'resyncRequired' || errorCode === 'syncStateNotFound' || errorCode === 'InvalidSyncStateData') {
+                    this.logger.info({
+                        msg: 'Delta token expired, starting fresh sync',
+                        account: this.account,
+                        errorCode
+                    });
+                } else {
+                    this.logger.error({
+                        msg: 'Failed to check mailbox folder delta',
+                        account: this.account,
+                        err
+                    });
+                }
+
+                // Clear delta URL to start fresh
                 await this.redis.hdel(this.getAccountKey(), 'outlookMailFoldersDeltaUrl');
 
                 return true;
@@ -2734,8 +2752,20 @@ class OutlookClient extends BaseClient {
      * @returns {Object} Result with success status and details
      */
     async renewSubscription(force = false) {
+        // In-memory check to prevent concurrent renewals from lifecycle events
+        // This is faster than Redis lock and reduces noise from duplicate requests
+        if (this.renewalInProgress) {
+            this.logger.debug({
+                msg: 'Subscription renewal skipped',
+                reason: 'renewal_in_progress',
+                account: this.account
+            });
+            return { success: false, reason: 'renewal_in_progress' };
+        }
+
         const lockKey = `${this.getAccountKey()}:subscription-renew-lock`;
-        const lockTTL = 30; // 30 seconds TTL for the lock to prevent blocking on crash
+        // Use constant for lock TTL (60 seconds) to handle slow network conditions
+        const lockTTL = OUTLOOK_SUBSCRIPTION_LOCK_TTL;
 
         // Try to acquire lock using Redis SET NX with expiry
         const lockAcquired = await this.redis.set(lockKey, Date.now(), 'EX', lockTTL, 'NX');
@@ -2749,6 +2779,9 @@ class OutlookClient extends BaseClient {
             return { success: false, reason: 'lock_exists' };
         }
 
+        // Set in-memory flag to prevent concurrent calls
+        this.renewalInProgress = true;
+
         try {
             // Get current subscription
             let outlookSubscription = await this.redis.hget(this.getAccountKey(), 'outlookSubscription');
@@ -2774,14 +2807,17 @@ class OutlookClient extends BaseClient {
             }
 
             const now = Date.now();
+            // Add 60 second buffer for clock skew between EmailEngine and MS Graph servers
+            const CLOCK_SKEW_BUFFER = 60 * 1000;
 
-            // Check if already expired
-            if (existingExpirationDateTime && existingExpirationDateTime.getTime() < now) {
+            // Check if already expired (with buffer for clock skew)
+            if (existingExpirationDateTime && existingExpirationDateTime.getTime() < now + CLOCK_SKEW_BUFFER) {
                 this.logger.warn({
-                    msg: 'Subscription already expired',
+                    msg: 'Subscription already expired or expiring imminently',
                     subscriptionId: outlookSubscription.id,
                     expirationDateTime: outlookSubscription.expirationDateTime,
-                    account: this.account
+                    account: this.account,
+                    msUntilExpiry: existingExpirationDateTime.getTime() - now
                 });
                 // Clear the expired subscription
                 await this.redis.hdel(this.getAccountKey(), 'outlookSubscription');
@@ -2868,11 +2904,11 @@ class OutlookClient extends BaseClient {
                 await this.redis.hSetExists(this.getAccountKey(), 'outlookSubscription', JSON.stringify(outlookSubscription));
                 this.subscriptionState = 'failed';
 
-                // Schedule retry with exponential backoff (max 3 retries)
+                // Schedule retry with exponential backoff
                 const retryCount = outlookSubscription.state.retryCount;
-                if (retryCount <= 3) {
-                    // Exponential backoff: 30s, 60s, 120s
-                    const retryDelay = Math.min(30 * Math.pow(2, retryCount - 1), 120) * 1000;
+                if (retryCount <= OUTLOOK_MAX_RETRY_ATTEMPTS) {
+                    // Exponential backoff: 30s, 60s, 120s (capped)
+                    const retryDelay = Math.min(OUTLOOK_RETRY_BASE_DELAY * Math.pow(2, retryCount - 1), OUTLOOK_RETRY_MAX_DELAY) * 1000;
 
                     setTimeout(() => {
                         // Retry will also acquire lock
@@ -2903,7 +2939,8 @@ class OutlookClient extends BaseClient {
                 return { success: false, reason: 'renewal_failed', error: err.message };
             }
         } finally {
-            // Always release the lock
+            // Always release the lock and clear in-memory flag
+            this.renewalInProgress = false;
             await this.redis.del(lockKey);
         }
     }
@@ -3030,9 +3067,17 @@ class OutlookClient extends BaseClient {
                     throw new Error('Empty server response');
                 }
             } catch (err) {
+                const errorMessage = err.oauthRequest?.response?.error?.message || err.message;
+                this.logger.error({
+                    msg: 'Failed to create MS Graph subscription',
+                    account: this.account,
+                    error: errorMessage,
+                    code: err.oauthRequest?.response?.error?.code,
+                    statusCode: err.oauthRequest?.status
+                });
                 outlookSubscription.state = {
                     state: 'error',
-                    error: `Subscription failed: ${err.oauthRequest?.response?.error?.message || err.message}`,
+                    error: `Subscription failed: ${errorMessage}`,
                     time: Date.now()
                 };
                 this.subscriptionState = 'failed';
@@ -3488,10 +3533,11 @@ class OutlookClient extends BaseClient {
                 { metadataOnly: true }
             );
 
-            if (messageListResult?.messages) {
-                messages = messages.concat(messageListResult?.messages);
+            if (messageListResult?.messages?.length) {
+                // Use push instead of concat to avoid O(n^2) memory allocation
+                messages.push(...messageListResult.messages);
                 if (messages.length >= maxMessages) {
-                    messages = messages.slice(0, maxMessages);
+                    messages.length = maxMessages; // Truncate in place
                     notDone = false;
                     break;
                 }