emailengine-app 2.63.4 → 2.65.0

package/lib/email-client/outlook-client.js

@@ -28,6 +28,7 @@ const {
     OUTLOOK_MAX_RETRY_ATTEMPTS,
     OUTLOOK_RETRY_BASE_DELAY,
     OUTLOOK_RETRY_MAX_DELAY,
+    OUTLOOK_CLOCK_SKEW_BUFFER,
     MESSAGE_UPDATED_NOTIFY,
     MESSAGE_DELETED_NOTIFY,
     MESSAGE_MISSING_NOTIFY,
@@ -85,6 +86,9 @@ class OutlookClient extends BaseClient {
         // Flags for background processing
         this.processingHistory = null;
         this.renewWatchTimer = null;
+        // Shared retry timer for subscription operations (creation, renewal, lock errors).
+        // Only one retry can be pending at a time; the lock serializes these operations.
+        this.renewRetryTimer = null;
         this.renewalInProgress = false; // In-memory flag to prevent concurrent subscription renewals
 
         // MS Graph webhook subscription state (for metrics)
@@ -92,6 +96,90 @@ class OutlookClient extends BaseClient {
         this.subscriptionState = 'unset';
     }
 
+    /**
+     * Get the lock ID used for subscription operations.
+     * Both renewSubscription and ensureSubscription coordinate on this lock.
+     * @returns {string}
+     */
+    getSubscriptionLockId() {
+        return ['outlook', 'subscription', this.account].join(':');
+    }
+
+    /**
+     * Load and parse the outlookSubscription data from Redis.
+     * Returns a parsed object, or an empty object if missing/invalid.
+     * @returns {Promise<Object>}
+     */
+    async getStoredSubscription() {
+        let raw = await this.redis.hget(this.getAccountKey(), 'outlookSubscription');
+        if (!raw) {
+            return {};
+        }
+        try {
+            return JSON.parse(raw);
+        } catch (err) {
+            this.logger.warn({ msg: 'Failed to parse stored subscription data', account: this.account, err });
+            return {};
+        }
+    }
+
+    /**
+     * Persist subscription data back to Redis (only if the key already exists).
+     * @param {Object} subscriptionData
+     * @returns {Promise<void>}
+     */
+    async saveStoredSubscription(subscriptionData) {
+        await this.redis.hSetExists(this.getAccountKey(), 'outlookSubscription', JSON.stringify(subscriptionData));
+    }
+
+    /**
+     * Parse an expiration date string, returning null for missing or invalid values.
+     * @param {string} dateStr
+     * @returns {Date|null}
+     */
+    parseExpirationDate(dateStr) {
+        if (!dateStr) {
+            return null;
+        }
+        let date = new Date(dateStr);
+        if (date.toString() === 'Invalid Date') {
+            return null;
+        }
+        return date;
+    }
+
+    /**
+     * Schedule a subscription retry on the shared retry timer.
+     * Replaces any previously pending retry. The timer is unref'd so it
+     * does not keep the process alive.
+     * @param {Object} opts
+     * @param {number} opts.delayMs - Delay before firing the retry
+     * @param {string} opts.reason - Short label for log messages (e.g. 'renewal_failure')
+     * @param {string} opts.errorMsg - Log message if the retry callback throws
+     * @param {Function} opts.fn - Async function to call on retry
+     * @param {number} [opts.retryAttempt] - Current attempt number (for logging)
+     */
+    scheduleSubscriptionRetry({ delayMs, reason, errorMsg, fn, retryAttempt }) {
+        if (this.renewRetryTimer) {
+            this.logger.debug({ msg: 'Replacing pending subscription retry timer', account: this.account, reason });
+        }
+        clearTimeout(this.renewRetryTimer);
+        this.renewRetryTimer = setTimeout(() => {
+            if (this.closed) {
+                return;
+            }
+            fn().catch(err => {
+                this.logger.error({
+                    msg: errorMsg,
+                    account: this.account,
+                    retryAttempt,
+                    err
+                });
+            });
+        }, delayMs);
+        this.renewRetryTimer.unref();
+    }
+
     /**
      * Makes authenticated requests to Microsoft Graph API
      * Handles token management and error responses
@@ -122,21 +210,29 @@ class OutlookClient extends BaseClient {
      */
     async init() {
         this.state = 'connecting';
+        clearTimeout(this.renewWatchTimer);
+        clearTimeout(this.renewRetryTimer);
         await this.setStateVal();
 
         await this.getAccount();
-        await this.prepareDelegatedAccount();
         await this.getClient(true);
+        await this.prepareDelegatedAccount();
 
         let accountData = await this.accountObject.loadAccountData();
 
         // Check if send-only mode
-        // Note: Scopes are checked at initialization and after account updates. If scopes change
-        // during token refresh (rare - typically requires re-authorization), the account must be
-        // reinitialized to detect the change. Consider checking scopes periodically if dynamic
-        // scope changes become common.
+        // For outlookService (client_credentials), app permissions are configured in Entra.
+        // The .default scope doesn't enumerate individual permissions, so assume full access.
+        // For delegated accounts, scopes are checked at initialization and after account updates.
         const scopes = accountData.oauth2?.accessToken?.scope || accountData.oauth2?.scope || [];
-        const { hasSendScope, hasReadScope } = checkAccountScopes('outlook', scopes, this.logger);
+        let hasSendScope, hasReadScope;
+        if (this.oAuth2Client?.useClientCredentials) {
+            hasSendScope = true;
+            hasReadScope = true;
+        } else {
+            ({ hasSendScope, hasReadScope } = checkAccountScopes('outlook', scopes, this.logger));
+        }
+
         const isSendOnly = hasSendScope && !hasReadScope;
 
         this.logger.debug({
@@ -180,7 +276,12 @@ class OutlookClient extends BaseClient {
         }
 
         // Update username if it has changed (e.g., after email address change)
-        if (profileRes.userPrincipalName && accountData.oauth2.auth?.user !== profileRes.userPrincipalName && !accountData.oauth2.auth?.delegatedUser) {
+        if (
+            profileRes.userPrincipalName &&
+            accountData.oauth2.auth?.user !== profileRes.userPrincipalName &&
+            !accountData.oauth2.auth?.delegatedUser &&
+            !this.oAuth2Client?.useClientCredentials
+        ) {
             updates.oauth2 = {
                 partial: true,
                 auth: Object.assign(accountData.oauth2.auth || {}, {
@@ -245,6 +346,15 @@ class OutlookClient extends BaseClient {
         if (!isSendOnly) {
             // additional operations for full access accounts
 
+            // Reset subscription retry counts on reconnect — fresh auth = fresh slate
+            let storedSub = await this.getStoredSubscription();
+            if (storedSub.state && (storedSub.state.createRetryCount > 0 || storedSub.state.retryCount > 0)) {
+                storedSub.state.createRetryCount = 0;
+                storedSub.state.retryCount = 0;
+                storedSub.state.error = null;
+                await this.saveStoredSubscription(storedSub);
+            }
+
             // Set up webhook subscription for real-time updates
             await this.ensureSubscription();
             this.setupRenewWatchTimer();
@@ -266,6 +376,7 @@ class OutlookClient extends BaseClient {
      */
     async close() {
         clearTimeout(this.renewWatchTimer);
+        clearTimeout(this.renewRetryTimer);
         this.closed = true;
 
         if (['init', 'connecting', 'syncing', 'connected'].includes(this.state)) {
@@ -289,8 +400,19 @@ class OutlookClient extends BaseClient {
      */
     async delete() {
         clearTimeout(this.renewWatchTimer);
+        clearTimeout(this.renewRetryTimer);
         this.closed = true;
 
+        // Best-effort: delete MS Graph subscription
+        try {
+            let outlookSubscription = await this.getStoredSubscription();
+            if (outlookSubscription.id) {
+                await this.request(`/subscriptions/${outlookSubscription.id}`, 'DELETE');
+            }
+        } catch (err) {
+            this.logger.error({ msg: 'Failed to delete MS Graph subscription', account: this.account, err });
+        }
+
         if (['init', 'connecting', 'syncing', 'connected'].includes(this.state)) {
             this.state = 'disconnected';
             await this.setStateVal();
@@ -2665,6 +2787,20 @@ class OutlookClient extends BaseClient {
 
         let accountData = await this.accountObject.loadAccountData();
 
+        // For outlookService (client_credentials), always use /users/{email} path
+        if (this.oAuth2Client?.useClientCredentials) {
+            let email = accountData.oauth2?.auth?.delegatedUser || accountData.oauth2?.auth?.user;
+            if (email) {
+                this.oauth2UserPath = `users/${encodeURIComponent(email)}`;
+            } else {
+                let err = new Error('Application credentials require a target user email address');
+                err.code = 'MissingTargetUser';
+                err.authenticationFailed = true;
+                throw err;
+            }
+            return;
+        }
+
         if (accountData?.oauth2?.auth?.delegatedUser && accountData?.oauth2?.auth?.delegatedAccount) {
             await this.getDelegatedAccount(accountData);
             if (this.delegatedAccountObject) {
@@ -2694,13 +2830,21 @@ class OutlookClient extends BaseClient {
 
             // Track successful token refresh (only if token was actually refreshed, not cached)
             if (!tokenData.cached) {
-                metricsMeta({ account: this.account }, this.logger, 'oauth2TokenRefresh', 'inc', { status: 'success', provider: 'outlook', statusCode: '200' });
+                metricsMeta({ account: this.account }, this.logger, 'oauth2TokenRefresh', 'inc', {
+                    status: 'success',
+                    provider: this.oAuth2Client?.provider || 'outlook',
+                    statusCode: '200'
+                });
             }
         } catch (E) {
             if (E.code === 'ETokenRefresh') {
                 // Track failed token refresh
                 const statusCode = String(E.statusCode || 0);
-                metricsMeta({ account: this.account }, this.logger, 'oauth2TokenRefresh', 'inc', { status: 'failure', provider: 'outlook', statusCode });
+                metricsMeta({ account: this.account }, this.logger, 'oauth2TokenRefresh', 'inc', {
+                    status: 'failure',
+                    provider: this.oAuth2Client?.provider || 'outlook',
+                    statusCode
+                });
 
                 // treat as authentication failure
                 this.state = 'authenticationError';
@@ -2743,8 +2887,8 @@ class OutlookClient extends BaseClient {
      */
     async prepare() {
         await this.getAccount();
-        await this.prepareDelegatedAccount();
         await this.getClient();
+        await this.prepareDelegatedAccount();
     }
 
     /**
@@ -2762,11 +2906,20 @@ class OutlookClient extends BaseClient {
 
             try {
                 // First try to renew existing subscription
-                const renewalResult = await this.renewSubscription(false);
+                const renewalResult = await this.renewSubscription({ force: false });
 
                 if (!renewalResult.success && (renewalResult.reason === 'expired' || renewalResult.reason === 'no_subscription')) {
-                    // If subscription expired or doesn't exist, ensure we have one
-                    await this.ensureSubscription();
+                    // Check if we've exceeded max creation retries
+                    let subscriptionData = await this.getStoredSubscription();
+                    if (subscriptionData?.state?.createRetryCount >= OUTLOOK_MAX_RETRY_ATTEMPTS) {
+                        this.logger.error({
+                            msg: 'Max subscription creation retries exceeded, waiting for reconnect',
+                            account: this.account,
+                            createRetryCount: subscriptionData.state.createRetryCount
+                        });
+                    } else {
+                        await this.ensureSubscription();
+                    }
                 }
             } catch (err) {
                 this.logger.error({ msg: 'Failed to renew MS Graph change subscription', account: this.account, err });
@@ -3019,13 +3172,19 @@ class OutlookClient extends BaseClient {
     /**
      * Unified method to renew MS Graph webhook subscription with proper locking
      * Can be called from timer or lifecycle notification handler
-     * @param {boolean} force - Force renewal even if not expired soon
+     * @param {Object} [opts] - Options
+     * @param {boolean} [opts.force=false] - Force renewal even if not expired soon
+     * @param {boolean} [opts.skipLock=false] - Skip lock acquisition (when caller already holds the subscription lock)
      * @returns {Object} Result with success status and details
      */
-    async renewSubscription(force = false) {
+    async renewSubscription({ force = false, skipLock = false } = {}) {
+        if (this.closed) {
+            return { success: false, reason: 'closed' };
+        }
+
         // In-memory check to prevent concurrent renewals from lifecycle events
         // This is faster than Redis lock and reduces noise from duplicate requests
-        if (this.renewalInProgress) {
+        if (this.renewalInProgress && !force) {
             this.logger.debug({
                 msg: 'Subscription renewal skipped',
                 reason: 'renewal_in_progress',
@@ -3034,20 +3193,31 @@ class OutlookClient extends BaseClient {
             return { success: false, reason: 'renewal_in_progress' };
         }
 
-        const lockKey = `${this.getAccountKey()}:subscription-renew-lock`;
-        // Use constant for lock TTL (60 seconds) to handle slow network conditions
-        const lockTTL = OUTLOOK_SUBSCRIPTION_LOCK_TTL;
-
-        // Try to acquire lock using Redis SET NX with expiry
-        const lockAcquired = await this.redis.set(lockKey, Date.now(), 'EX', lockTTL, 'NX');
+        const lock = this.accountObject.getLock();
+        const lockId = this.getSubscriptionLockId();
+        let renewLock;
+        if (!skipLock) {
+            try {
+                renewLock = await lock.acquireLock(lockId, OUTLOOK_SUBSCRIPTION_LOCK_TTL);
+            } catch (err) {
+                this.logger.error({ msg: 'Failed to acquire subscription renew lock', account: this.account, err });
+                this.scheduleSubscriptionRetry({
+                    delayMs: OUTLOOK_RETRY_BASE_DELAY * 1000,
+                    reason: 'lock_error',
+                    errorMsg: 'Subscription renew retry after lock error failed',
+                    fn: () => this.renewSubscription({ force: false })
+                });
+                return { success: false, reason: 'lock_error' };
+            }
 
-        if (!lockAcquired) {
-            this.logger.info({
-                msg: 'Subscription renewal skipped',
-                reason: 'lock_exists',
-                account: this.account
-            });
-            return { success: false, reason: 'lock_exists' };
+            if (!renewLock.success) {
+                this.logger.info({
+                    msg: 'Subscription renewal skipped',
+                    reason: 'lock_exists',
+                    account: this.account
+                });
+                return { success: false, reason: 'lock_exists' };
+            }
         }
 
         // Set in-memory flag to prevent concurrent calls
@@ -3055,16 +3225,7 @@ class OutlookClient extends BaseClient {
 
         try {
             // Get current subscription
-            let outlookSubscription = await this.redis.hget(this.getAccountKey(), 'outlookSubscription');
-            if (outlookSubscription) {
-                try {
-                    outlookSubscription = JSON.parse(outlookSubscription);
-                } catch (err) {
-                    outlookSubscription = {};
-                }
-            } else {
-                outlookSubscription = {};
-            }
+            let outlookSubscription = await this.getStoredSubscription();
 
             // Check if subscription exists and is valid
             if (!outlookSubscription.id) {
@@ -3072,17 +3233,12 @@ class OutlookClient extends BaseClient {
                 return { success: false, reason: 'no_subscription' };
             }
 
-            let existingExpirationDateTime = new Date(outlookSubscription.expirationDateTime);
-            if (existingExpirationDateTime.toString() === 'Invalid Date') {
-                existingExpirationDateTime = null;
-            }
+            let existingExpirationDateTime = this.parseExpirationDate(outlookSubscription.expirationDateTime);
 
             const now = Date.now();
-            // Add 60 second buffer for clock skew between EmailEngine and MS Graph servers
-            const CLOCK_SKEW_BUFFER = 60 * 1000;
 
             // Check if already expired (with buffer for clock skew)
-            if (existingExpirationDateTime && existingExpirationDateTime.getTime() < now + CLOCK_SKEW_BUFFER) {
+            if (existingExpirationDateTime && existingExpirationDateTime.getTime() < now + OUTLOOK_CLOCK_SKEW_BUFFER) {
                 this.logger.warn({
                     msg: 'Subscription already expired or expiring imminently',
                     subscriptionId: outlookSubscription.id,
@@ -3116,41 +3272,39 @@ class OutlookClient extends BaseClient {
                 expirationDateTime: expirationDateTime.toISOString()
             };
 
+            // Capture previous retry counts before overwriting state
+            const previousRetryCount = outlookSubscription.state?.retryCount || 0;
+            const previousCreateRetryCount = outlookSubscription.state?.createRetryCount || 0;
+
             // Update state to renewing
             outlookSubscription.state = {
                 state: 'renewing',
                 time: Date.now()
             };
-            await this.redis.hSetExists(this.getAccountKey(), 'outlookSubscription', JSON.stringify(outlookSubscription));
+            await this.saveStoredSubscription(outlookSubscription);
             this.subscriptionState = 'pending';
 
             try {
                 const subscriptionRes = await this.request(`/subscriptions/${outlookSubscription.id}`, 'PATCH', subscriptionPayload);
 
                 if (subscriptionRes?.expirationDateTime) {
-                    // Check if this was a retry before clearing state
-                    const wasRetry = outlookSubscription.state?.retryCount > 0;
-                    const previousRetryCount = outlookSubscription.state?.retryCount || 0;
-
                     outlookSubscription.expirationDateTime = subscriptionRes.expirationDateTime;
                     outlookSubscription.state = {
                         state: 'created',
                         time: Date.now(),
-                        // Clear any previous error state and retry count
                         retryCount: 0,
+                        createRetryCount: 0,
                         error: null
                     };
 
-                    await this.redis.hSetExists(this.getAccountKey(), 'outlookSubscription', JSON.stringify(outlookSubscription));
+                    await this.saveStoredSubscription(outlookSubscription);
                     this.subscriptionState = 'valid';
 
-                    // Log if this was a successful retry after errors
                     this.logger.info({
-                        msg: wasRetry ? 'Subscription renewed successfully after retry' : 'Subscription renewed successfully',
+                        msg: previousRetryCount > 0 ? 'Subscription renewed successfully after retry' : 'Subscription renewed successfully',
                         subscriptionId: outlookSubscription.id,
                         newExpirationDateTime: subscriptionRes.expirationDateTime,
                         account: this.account,
-                        wasRetry,
                         previousRetryCount
                     });
 
@@ -3170,28 +3324,24 @@ class OutlookClient extends BaseClient {
                     state: 'error',
                     error: `Subscription renewal failed: ${err.oauthRequest?.response?.error?.message || err.message}`,
                     time: Date.now(),
-                    retryCount: (outlookSubscription.state?.retryCount || 0) + 1
+                    retryCount: previousRetryCount + 1,
+                    createRetryCount: previousCreateRetryCount
                 };
-                await this.redis.hSetExists(this.getAccountKey(), 'outlookSubscription', JSON.stringify(outlookSubscription));
+                await this.saveStoredSubscription(outlookSubscription);
                 this.subscriptionState = 'failed';
 
                 // Schedule retry with exponential backoff
                 const retryCount = outlookSubscription.state.retryCount;
-                if (retryCount <= OUTLOOK_MAX_RETRY_ATTEMPTS) {
-                    // Exponential backoff: 30s, 60s, 120s (capped)
+                if (retryCount < OUTLOOK_MAX_RETRY_ATTEMPTS) {
                     const retryDelay = Math.min(OUTLOOK_RETRY_BASE_DELAY * Math.pow(2, retryCount - 1), OUTLOOK_RETRY_MAX_DELAY) * 1000;
 
-                    setTimeout(() => {
-                        // Retry will also acquire lock
-                        this.renewSubscription(force).catch(err => {
-                            this.logger.error({
-                                msg: 'Subscription renewal retry failed',
-                                account: this.account,
-                                retryAttempt: retryCount,
-                                err
-                            });
-                        });
-                    }, retryDelay);
+                    this.scheduleSubscriptionRetry({
+                        delayMs: retryDelay,
+                        reason: 'renewal_failure',
+                        errorMsg: 'Subscription renewal retry failed',
+                        fn: () => this.renewSubscription({ force }),
+                        retryAttempt: retryCount
+                    });
 
                     this.logger.info({
                         msg: 'Scheduling subscription renewal retry',
@@ -3212,7 +3362,13 @@ class OutlookClient extends BaseClient {
         } finally {
             // Always release the lock and clear in-memory flag
             this.renewalInProgress = false;
-            await this.redis.del(lockKey);
+            if (!skipLock && renewLock?.success) {
+                try {
+                    await lock.releaseLock(renewLock);
+                } catch (err) {
+                    this.logger.error({ msg: 'Failed to release subscription renew lock', account: this.account, err });
+                }
+            }
         }
     }
 
@@ -3220,7 +3376,11 @@ class OutlookClient extends BaseClient {
      * Ensure webhook subscription exists and is valid
      * Creates new subscription or renews existing one
      */
-    async ensureSubscription() {
+    async ensureSubscription({ clearExisting = false } = {}) {
+        if (this.closed) {
+            return;
+        }
+
         let serviceUrl = (await settings.get('serviceUrl')) || null;
         let notificationBaseUrl = (await settings.get('notificationBaseUrl')) || serviceUrl || '';
 
@@ -3229,131 +3389,200 @@ class OutlookClient extends BaseClient {
             return false;
         }
 
-        let outlookSubscription = await this.redis.hget(this.getAccountKey(), 'outlookSubscription');
-        if (outlookSubscription) {
-            try {
-                outlookSubscription = JSON.parse(outlookSubscription);
-            } catch (err) {
-                outlookSubscription = {};
-            }
-        }
-
-        if (!outlookSubscription) {
-            outlookSubscription = {};
+        // Acquire distributed lock to prevent concurrent subscription operations
+        const lock = this.accountObject.getLock();
+        const lockId = this.getSubscriptionLockId();
+        let ensureLock;
+        try {
+            ensureLock = await lock.acquireLock(lockId, OUTLOOK_SUBSCRIPTION_LOCK_TTL);
+        } catch (err) {
+            this.logger.error({ msg: 'Failed to acquire subscription ensure lock', account: this.account, err });
+            // Schedule retry for transient lock errors (e.g., Redis connectivity).
+            // Do not forward clearExisting on retry: by the time the retry fires,
+            // another path may have created a valid subscription that should not be cleared.
+            this.scheduleSubscriptionRetry({
+                delayMs: OUTLOOK_RETRY_BASE_DELAY * 1000,
+                reason: 'lock_error',
+                errorMsg: 'Subscription ensure retry after lock error failed',
+                fn: () => this.ensureSubscription()
+            });
+            return;
         }
-
-        // Check if subscription is being created/renewed
-        if (['creating', 'renewing'].includes(outlookSubscription.state?.state) && outlookSubscription.state.time > Date.now() - 30 * 60 * 1000) {
-            // allow previous operation to finish (30 minute timeout)
+        if (!ensureLock.success) {
             this.logger.info({
-                msg: 'Subscription renewal skipped',
-                reason: 'pending',
-                subscriptionId: outlookSubscription?.id,
-                state: outlookSubscription.state?.state,
-                created: outlookSubscription.state?.time,
-                expected: Date.now() - 30 * 60 * 1000
+                msg: 'Subscription creation skipped',
+                reason: 'lock_exists',
+                account: this.account
             });
-            this.subscriptionState = 'pending';
             return;
         }
 
-        let now = Date.now();
-        let expirationDateTime = new Date(now + OUTLOOK_EXPIRATION_TIME);
-
-        // Renew existing subscription if needed
-        if (outlookSubscription.id) {
-            let existingExpirationDateTime = new Date(outlookSubscription.expirationDateTime);
-            if (existingExpirationDateTime.toString() === 'Invalid Date') {
-                existingExpirationDateTime = null;
+        try {
+            if (clearExisting) {
+                await this.redis.hdel(this.getAccountKey(), 'outlookSubscription');
+                this.logger.info({
+                    msg: 'Cleared existing subscription data for re-creation',
+                    account: this.account
+                });
             }
 
-            if (existingExpirationDateTime && existingExpirationDateTime.getTime() < now) {
-                //  already expired
-                outlookSubscription = {};
-            } else if (existingExpirationDateTime && existingExpirationDateTime.getTime() < now + OUTLOOK_EXPIRATION_RENEW_TIME) {
-                // Use the unified renewal method
-                const renewalResult = await this.renewSubscription(false);
-                if (!renewalResult.success && renewalResult.reason === 'expired') {
-                    // Subscription expired, clear it to create a new one
-                    outlookSubscription = {};
-                } else {
-                    return renewalResult.success;
-                }
-            } else {
-                // Subscription is valid, do nothing
-                this.logger.info({
-                    msg: 'Subscription renewal skipped',
-                    reason: 'valid',
+            let outlookSubscription = await this.getStoredSubscription();
+
+            // If state is creating/renewing, the previous lock holder crashed before completing.
+            // Since we hold the lock, no other operation is in progress — reset to a clean state.
+            if (['creating', 'renewing'].includes(outlookSubscription.state?.state)) {
+                this.logger.warn({
+                    msg: 'Found stale subscription state from crashed operation, resetting state',
                     subscriptionId: outlookSubscription?.id,
                     state: outlookSubscription.state?.state,
-                    created: outlookSubscription.state?.time,
-                    expirationDateTime: outlookSubscription.expirationDateTime
+                    stateTime: outlookSubscription.state?.time,
+                    account: this.account
                 });
-                this.subscriptionState = 'valid';
-                return;
-            }
-        }
 
-        // Create new subscription if needed
-        if (!outlookSubscription.id) {
-            const notificationUrl = prepareUrl('/oauth/msg/notification', notificationBaseUrl, {
-                account: this.account
-            });
-
-            const lifecycleNotificationUrl = prepareUrl('/oauth/msg/lifecycle', notificationBaseUrl, {
-                account: this.account
-            });
+                // Reset transient state so downstream logic sees a clean state
+                outlookSubscription.state.state = outlookSubscription.id ? 'created' : 'error';
+                outlookSubscription.state.time = Date.now();
+                await this.saveStoredSubscription(outlookSubscription);
+            }
 
-            const subscriptionPayload = {
-                changeType: 'created,updated,deleted',
-                notificationUrl,
-                lifecycleNotificationUrl,
-                resource: `/${this.oauth2UserPath}/messages`,
-                expirationDateTime: expirationDateTime.toISOString(),
-                clientState: crypto.randomUUID()
-            };
+            let now = Date.now();
+            let expirationDateTime = new Date(now + OUTLOOK_EXPIRATION_TIME);
 
-            outlookSubscription.state = {
-                state: 'creating',
-                time: Date.now()
-            };
-            await this.redis.hSetExists(this.getAccountKey(), 'outlookSubscription', JSON.stringify(outlookSubscription));
-            this.subscriptionState = 'pending';
+            // Renew existing subscription if needed
+            if (outlookSubscription.id) {
+                let existingExpirationDateTime = this.parseExpirationDate(outlookSubscription.expirationDateTime);
 
-            let subscriptionRes;
-            try {
-                subscriptionRes = await this.request(`/subscriptions`, 'post', subscriptionPayload);
-                if (subscriptionRes?.expirationDateTime) {
-                    outlookSubscription = {
-                        id: subscriptionRes.id,
-                        expirationDateTime: subscriptionRes.expirationDateTime,
-                        clientState: subscriptionRes.clientState,
-                        state: {
-                            state: 'created',
-                            time: Date.now()
-                        }
-                    };
-                    this.subscriptionState = 'valid';
+                if (existingExpirationDateTime && existingExpirationDateTime.getTime() < now + OUTLOOK_CLOCK_SKEW_BUFFER) {
+                    //  already expired (with buffer for clock skew)
+                    outlookSubscription = {};
+                } else if (existingExpirationDateTime && existingExpirationDateTime.getTime() < now + OUTLOOK_EXPIRATION_RENEW_TIME) {
+                    // Use the unified renewal method (skip lock since we already hold it)
+                    const renewalResult = await this.renewSubscription({ force: false, skipLock: true });
+                    if (!renewalResult.success && renewalResult.reason === 'expired') {
+                        // Subscription expired, clear it to create a new one
+                        outlookSubscription = {};
+                    } else {
+                        return renewalResult.success;
+                    }
                 } else {
-                    throw new Error('Empty server response');
+                    // Subscription is valid, do nothing
+                    this.logger.info({
+                        msg: 'Subscription renewal skipped',
+                        reason: 'valid',
+                        subscriptionId: outlookSubscription?.id,
+                        state: outlookSubscription.state?.state,
+                        created: outlookSubscription.state?.time,
+                        expirationDateTime: outlookSubscription.expirationDateTime
+                    });
+                    this.subscriptionState = 'valid';
+                    return;
                 }
-            } catch (err) {
-                const errorMessage = err.oauthRequest?.response?.error?.message || err.message;
-                this.logger.error({
-                    msg: 'Failed to create MS Graph subscription',
-                    account: this.account,
-                    error: errorMessage,
-                    code: err.oauthRequest?.response?.error?.code,
-                    statusCode: err.oauthRequest?.status
+            }
+
+            // Create new subscription if needed
+            if (!outlookSubscription.id) {
+                const notificationUrl = prepareUrl('/oauth/msg/notification', notificationBaseUrl, {
+                    account: this.account
+                });
+
+                const lifecycleNotificationUrl = prepareUrl('/oauth/msg/lifecycle', notificationBaseUrl, {
+                    account: this.account
                 });
+
+                const subscriptionPayload = {
+                    changeType: 'created,updated,deleted',
+                    notificationUrl,
+                    lifecycleNotificationUrl,
+                    resource: `/${this.oauth2UserPath}/messages`,
+                    expirationDateTime: expirationDateTime.toISOString(),
+                    clientState: crypto.randomUUID()
+                };
+
+                // Capture previous create retry count before overwriting state
+                const previousCreateRetryCount = outlookSubscription.state?.createRetryCount || 0;
+
                 outlookSubscription.state = {
-                    state: 'error',
-                    error: `Subscription failed: ${errorMessage}`,
+                    state: 'creating',
                     time: Date.now()
                 };
-                this.subscriptionState = 'failed';
-            } finally {
-                await this.redis.hSetExists(this.getAccountKey(), 'outlookSubscription', JSON.stringify(outlookSubscription));
+                await this.saveStoredSubscription(outlookSubscription);
+                this.subscriptionState = 'pending';
+
+                let subscriptionRes;
+                try {
+                    subscriptionRes = await this.request(`/subscriptions`, 'post', subscriptionPayload);
+                    if (subscriptionRes?.expirationDateTime) {
+                        outlookSubscription = {
+                            id: subscriptionRes.id,
+                            expirationDateTime: subscriptionRes.expirationDateTime,
+                            clientState: subscriptionRes.clientState,
+                            state: {
+                                state: 'created',
+                                time: Date.now(),
+                                createRetryCount: 0,
+                                retryCount: 0
+                            }
+                        };
+                        this.subscriptionState = 'valid';
+                    } else {
+                        throw new Error('Empty server response');
+                    }
+                } catch (err) {
+                    const errorMessage = err.oauthRequest?.response?.error?.message || err.message;
+                    this.logger.error({
+                        msg: 'Failed to create MS Graph subscription',
+                        account: this.account,
+                        error: errorMessage,
+                        code: err.oauthRequest?.response?.error?.code,
+                        statusCode: err.oauthRequest?.status
+                    });
+                    outlookSubscription.state = {
+                        state: 'error',
+                        error: `Subscription failed: ${errorMessage}`,
+                        time: Date.now(),
+                        createRetryCount: previousCreateRetryCount + 1
+                    };
+                    this.subscriptionState = 'failed';
+                } finally {
+                    await this.saveStoredSubscription(outlookSubscription);
+                }
+
+                // Schedule retry with exponential backoff if creation failed
+                if (outlookSubscription.state?.state === 'error') {
+                    const createRetryCount = outlookSubscription.state.createRetryCount || 0;
+                    if (createRetryCount < OUTLOOK_MAX_RETRY_ATTEMPTS) {
+                        const retryDelay = Math.min(OUTLOOK_RETRY_BASE_DELAY * Math.pow(2, createRetryCount - 1), OUTLOOK_RETRY_MAX_DELAY) * 1000;
+
+                        this.scheduleSubscriptionRetry({
+                            delayMs: retryDelay,
+                            reason: 'creation_failure',
+                            errorMsg: 'Subscription creation retry failed',
+                            fn: () => this.ensureSubscription(),
+                            retryAttempt: createRetryCount
+                        });
+
+                        this.logger.info({
+                            msg: 'Scheduling subscription creation retry',
+                            account: this.account,
+                            retryAttempt: createRetryCount,
+                            retryDelayMs: retryDelay
+                        });
+                    } else {
+                        this.logger.error({
+                            msg: 'Max subscription creation retries exceeded',
+                            account: this.account,
+                            createRetryCount
+                        });
+                    }
+                }
+            }
+        } finally {
+            if (ensureLock?.success) {
+                try {
+                    await lock.releaseLock(ensureLock);
+                } catch (err) {
+                    this.logger.error({ msg: 'Failed to release subscription ensure lock', account: this.account, err });
+                }
             }
         }
     }