emailengine-app 2.61.0 → 2.61.2

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## [2.61.2](https://github.com/postalsys/emailengine/compare/v2.61.1...v2.61.2) (2026-01-12)
+
+
+### Bug Fixes
+
+* add error logging for MS Graph subscription creation failures ([ba928e4](https://github.com/postalsys/emailengine/commit/ba928e495acb7fcc8c18e6cf34c917a2f4ad3337))
+* add forced exit to prevent test timeout in CI ([aeb7261](https://github.com/postalsys/emailengine/commit/aeb726169b26e7263acb55041112f5a97a818b8a))
+* handle empty or invalid JSON responses from OAuth APIs ([57d8886](https://github.com/postalsys/emailengine/commit/57d8886a0d29b20df4b575bac8210f6ed97f7fa3))
+
+## [2.61.1](https://github.com/postalsys/emailengine/compare/v2.61.0...v2.61.1) (2025-12-28)
+
+
+### Bug Fixes
+
+* Memory leak fixes for IMAP client and webhooks worker ([b749f96](https://github.com/postalsys/emailengine/commit/b749f964f7e8de6828d23b8c3c5a3ca11e15898a))
+
 ## [2.61.0](https://github.com/postalsys/emailengine/compare/v2.60.1...v2.61.0) (2025-12-22)
 
 

package/data/google-crawlers.json

@@ -1,5 +1,5 @@
 {
-    "creationTime": "2025-12-18T15:45:44.000000",
+    "creationTime": "2026-01-09T15:45:52.000000",
     "prefixes": [
         {
             "ipv6Prefix": "2001:4860:4801:2008::/64"

package/lib/account/account-state.js

@@ -0,0 +1,248 @@
+'use strict';
+
+const Boom = require('@hapi/boom');
+const { REDIS_PREFIX } = require('../consts');
+
+// Account state constants
+const ACCOUNT_STATES = {
+    INIT: 'init',
+    UNSET: 'unset',
+    CONNECTED: 'connected',
+    CONNECTING: 'connecting',
+    SYNCING: 'syncing',
+    AUTHENTICATION_ERROR: 'authenticationError',
+    CONNECT_ERROR: 'connectError'
+};
+
+// States that indicate the account is operational
+const VALID_STATES = [ACCOUNT_STATES.CONNECTED, ACCOUNT_STATES.CONNECTING, ACCOUNT_STATES.SYNCING];
+
+// States that bypass runIndex checking
+const BYPASS_RUN_INDEX_STATES = [ACCOUNT_STATES.INIT, ACCOUNT_STATES.UNSET];
+
+/**
+ * Calculates the effective account state based on runIndex and current state
+ * @param {Object} accountData - The account data object
+ * @param {number} runIndex - Current run index from the worker
+ * @returns {string} The effective state value
+ */
+function calculateEffectiveState(accountData, runIndex) {
+    const currentState = accountData.state;
+    const accountRunIndex = accountData.runIndex;
+    const isApiAccount = accountData.isApi;
+
+    // API accounts and special states bypass runIndex checking
+    if (!runIndex || runIndex <= accountRunIndex || BYPASS_RUN_INDEX_STATES.includes(currentState) || isApiAccount) {
+        return currentState;
+    }
+
+    // Account hasn't been processed by current worker yet
+    return ACCOUNT_STATES.INIT;
+}
+
+/**
+ * Validates that an account is in a state suitable for operations
+ * @param {Object} accountData - The account data object with state
+ * @throws {Boom} When the account state is not valid for operations
+ */
+function validateAccountState(accountData) {
+    if (VALID_STATES.includes(accountData.state)) {
+        return;
+    }
+
+    let err;
+    switch (accountData.state) {
+        case ACCOUNT_STATES.INIT:
+            err = new Error('Requested account is not yet initialized');
+            err.code = 'NotYetConnected';
+            break;
+
+        case ACCOUNT_STATES.AUTHENTICATION_ERROR:
+            err = new Error('Requested account can not be authenticated');
+            err.code = 'AuthenticationFails';
+            break;
+
+        case ACCOUNT_STATES.CONNECT_ERROR:
+            err = new Error('Can not establish server connection for requested account');
+            err.code = 'ConnectionError';
+            break;
+
+        case ACCOUNT_STATES.UNSET:
+            err = new Error('Syncing is disabled for the requested account');
+            err.code = 'NotSyncing';
+            break;
+
+        default:
+            err = new Error('Requested account currently not available');
+            err.code = 'NoAvailable';
+            break;
+    }
+
+    const error = Boom.boomify(err, { statusCode: 503 });
+    if (accountData.state) {
+        error.output.payload.state = accountData.state;
+    }
+    if (err.code) {
+        error.output.payload.code = err.code;
+    }
+    throw error;
+}
+
+/**
+ * Gets the account state from Redis
+ * @param {Object} redis - Redis client
+ * @param {string} account - Account ID
+ * @returns {Promise<string|null>} The current state value
+ */
+async function getAccountState(redis, account) {
+    const accountKey = `${REDIS_PREFIX}iad:${account}`;
+    return await redis.hget(accountKey, 'state');
+}
+
+/**
+ * Sets the account state in Redis
+ * Only updates if the account exists
+ * @param {Object} redis - Redis client
+ * @param {string} account - Account ID
+ * @param {string} state - New state value
+ * @returns {Promise<boolean>} True if state was set
+ */
+async function setAccountState(redis, account, state) {
+    const accountKey = `${REDIS_PREFIX}iad:${account}`;
+    const result = await redis.hSetExists(accountKey, 'state', state);
+    return result === 1;
+}
+
+/**
+ * Gets the last error state for an account
+ * @param {Object} redis - Redis client
+ * @param {string} account - Account ID
+ * @returns {Promise<Object|null>} Parsed error state object or null
+ */
+async function getLastErrorState(redis, account) {
+    const accountKey = `${REDIS_PREFIX}iad:${account}`;
+    const errorState = await redis.hget(accountKey, 'lastErrorState');
+
+    if (!errorState) {
+        return null;
+    }
+
+    try {
+        return JSON.parse(errorState);
+    } catch (err) {
+        return null;
+    }
+}
+
+/**
+ * Sets the last error state for an account
+ * @param {Object} redis - Redis client
+ * @param {string} account - Account ID
+ * @param {Object} errorData - Error state data
+ * @returns {Promise<boolean>} True if error state was set
+ */
+async function setLastErrorState(redis, account, errorData) {
+    const accountKey = `${REDIS_PREFIX}iad:${account}`;
+    const result = await redis.hSetExists(accountKey, 'lastErrorState', JSON.stringify(errorData));
+    return result === 1;
+}
+
+/**
+ * Clears the last error state for an account
+ * @param {Object} redis - Redis client
+ * @param {string} account - Account ID
+ * @returns {Promise<number>} Number of fields removed
+ */
+async function clearLastErrorState(redis, account) {
+    const accountKey = `${REDIS_PREFIX}iad:${account}`;
+    return await redis.hdel(accountKey, 'lastErrorState');
+}
+
+/**
+ * Gets the connection count for a specific state
+ * @param {Object} redis - Redis client
+ * @param {string} account - Account ID
+ * @param {string} state - State to get count for
+ * @returns {Promise<number>} Connection count
+ */
+async function getStateCount(redis, account, state) {
+    const accountKey = `${REDIS_PREFIX}iad:${account}`;
+    const count = await redis.hget(accountKey, `state:count:${state}`);
+    return parseInt(count, 10) || 0;
+}
+
+/**
+ * Resets the connection count for a specific state
+ * @param {Object} redis - Redis client
+ * @param {string} account - Account ID
+ * @param {string} state - State to reset count for
+ * @returns {Promise<boolean>} True if count was reset
+ */
+async function resetStateCount(redis, account, state) {
+    const accountKey = `${REDIS_PREFIX}iad:${account}`;
+    const result = await redis.hset(accountKey, `state:count:${state}`, '0');
+    return result >= 0;
+}
+
+/**
+ * Formats the last error for API responses
+ * Returns null if account is connected or no error exists
+ * @param {Object} accountData - Account data object
+ * @returns {Object|null} Formatted error object or null
+ */
+function formatLastError(accountData) {
+    if (accountData.state === ACCOUNT_STATES.CONNECTED) {
+        return null;
+    }
+
+    if (!accountData.lastErrorState || !Object.keys(accountData.lastErrorState).length) {
+        return null;
+    }
+
+    return accountData.lastErrorState;
+}
+
+/**
+ * Determines if account state should be reported as 'init' based on runIndex
+ * Used for account listings when the worker hasn't processed the account yet
+ * @param {Object} accountData - Account data with state and runIndex
+ * @param {number} currentRunIndex - Current worker run index
+ * @returns {string} Effective state for display
+ */
+function getDisplayState(accountData, currentRunIndex) {
+    const currentState = accountData.state;
+    const accountRunIndex = accountData.runIndex;
+    const isApiAccount = accountData.isApi;
+
+    if (!currentRunIndex || currentRunIndex <= accountRunIndex || BYPASS_RUN_INDEX_STATES.includes(currentState) || isApiAccount) {
+        return currentState;
+    }
+
+    return ACCOUNT_STATES.INIT;
+}
+
+module.exports = {
+    // Constants
+    ACCOUNT_STATES,
+    VALID_STATES,
+    BYPASS_RUN_INDEX_STATES,
+
+    // State calculation
+    calculateEffectiveState,
+    validateAccountState,
+    getDisplayState,
+
+    // Redis operations
+    getAccountState,
+    setAccountState,
+
+    // Error state management
+    getLastErrorState,
+    setLastErrorState,
+    clearLastErrorState,
+    formatLastError,
+
+    // State counters
+    getStateCount,
+    resetStateCount
+};

package/lib/account.js

@@ -26,8 +26,8 @@ const Lock = require('ioredfour');
 const nanoid = customAlphabet('0123456789abcdefghijklmnopqrstuvwxyz', 16);
 const { REDIS_PREFIX, ACCOUNT_DELETED_NOTIFY, MAILBOX_HASH } = require('./consts');
 const { mimeHtml } = require('@postalsys/email-text-tools');
-const { GMAIL_API_SCOPES } = require('./oauth/gmail');
-const { OUTLOOK_API_SCOPES } = require('./oauth/outlook');
+const { checkAccountScopes: checkScopes } = require('./oauth/scope-checker');
+const { ACCOUNT_STATES, calculateEffectiveState, validateAccountState, getDisplayState, formatLastError } = require('./account/account-state');
 
 class Account {
     constructor(options) {
@@ -58,127 +58,15 @@ class Account {
     }
 
     /**
-     * Checks OAuth2 scopes to determine account capabilities
+     * Checks OAuth2 scopes to determine account capabilities.
+     * Delegates to centralized scope-checker module.
      *
-     * @param {string} provider - OAuth2 provider name (e.g., 'gmail', 'outlook')
+     * @param {string} provider - OAuth2 provider name ('gmail' or 'outlook')
      * @param {Array<string>} scopes - Array of OAuth2 scope strings
      * @returns {{hasSendScope: boolean, hasReadScope: boolean}} Object indicating send and read capabilities
-     *
-     * @example
-     * // Gmail send-only account
-     * checkAccountScopes('gmail', ['https://www.googleapis.com/auth/gmail.send'])
-     * // Returns: { hasSendScope: true, hasReadScope: false }
-     *
-     * @example
-     * // Gmail full access account
-     * checkAccountScopes('gmail', ['https://www.googleapis.com/auth/gmail.modify'])
-     * // Returns: { hasSendScope: false, hasReadScope: true }
-     *
-     * @example
-     * // Outlook send-only account (global cloud)
-     * checkAccountScopes('outlook', ['https://graph.microsoft.com/Mail.Send', 'offline_access'])
-     * // Returns: { hasSendScope: true, hasReadScope: false }
-     *
-     * @example
-     * // Outlook full access account (GCC-High cloud)
-     * checkAccountScopes('outlook', ['https://graph.microsoft.us/Mail.ReadWrite', 'https://graph.microsoft.us/Mail.Send'])
-     * // Returns: { hasSendScope: true, hasReadScope: true }
-     *
-     * @example
-     * // Outlook send-only account (DoD cloud)
-     * checkAccountScopes('outlook', ['https://dod-graph.microsoft.us/Mail.Send', 'offline_access'])
-     * // Returns: { hasSendScope: true, hasReadScope: false }
      */
     checkAccountScopes(provider, scopes) {
-        if (!scopes || !Array.isArray(scopes)) {
-            return { hasSendScope: false, hasReadScope: false };
-        }
-
-        if (provider === 'gmail') {
-            const hasSendScope = scopes.some(s => s.includes(GMAIL_API_SCOPES.send));
-            const hasReadScope = scopes.some(
-                s =>
-                    s.includes(GMAIL_API_SCOPES.modify) ||
-                    s.includes(GMAIL_API_SCOPES.readonly) ||
-                    s.includes(GMAIL_API_SCOPES.labels) ||
-                    s.includes('mail.google.com')
-            );
-            return { hasSendScope, hasReadScope };
-        }
-
-        if (provider === 'outlook') {
-            // Known Microsoft Graph API endpoints across different cloud environments
-            const msGraphDomains = [
-                'graph.microsoft.com', // Global cloud
-                'graph.microsoft.us', // GCC-High cloud
-                'dod-graph.microsoft.us', // DoD cloud
-                'microsoftgraph.chinacloudapi.cn' // China cloud
-            ];
-
-            // Normalize scopes by extracting the scope name from the full URL
-            // Supports multiple MS Graph endpoints (global, GCC-High, DoD, China)
-            // Examples:
-            //   https://graph.microsoft.com/Mail.Send -> Mail.Send
-            //   https://graph.microsoft.us/Mail.ReadWrite -> Mail.ReadWrite
-            //   https://dod-graph.microsoft.us/Mail.Send -> Mail.Send
-            //   https://microsoftgraph.chinacloudapi.cn/Mail.Send -> Mail.Send
-            //   offline_access -> offline_access (passed through)
-            const normalizedScopes = scopes.map(s => {
-                // Handle plain scope names (e.g., offline_access, openid)
-                // These are not URLs and should be passed through as-is
-                if (!s.includes('://')) {
-                    return s;
-                }
-
-                // Try to parse as URL to validate it's a Microsoft Graph endpoint
-                try {
-                    const url = new URL(s);
-
-                    // Validate protocol - must be https
-                    if (url.protocol !== 'https:') {
-                        this.logger.warn({
-                            msg: 'Invalid protocol in MS Graph scope URL, expected https',
-                            scope: s,
-                            protocol: url.protocol
-                        });
-                        return s; // Return as-is for non-https URLs
-                    }
-
-                    // Check if this is a recognized MS Graph domain
-                    if (msGraphDomains.includes(url.hostname)) {
-                        // Extract scope name from path only (ignoring query params and fragments)
-                        // Examples:
-                        //   /Mail.Send -> Mail.Send
-                        //   /Mail.Send?foo=bar -> Mail.Send
-                        //   /Mail.Send#section -> Mail.Send
-                        //   /Mail.Send/ -> Mail.Send (removes trailing slash)
-                        const scopeName = url.pathname.substring(1).replace(/\/$/, '');
-                        if (scopeName) {
-                            return scopeName;
-                        }
-                        this.logger.warn({
-                            msg: 'MS Graph scope URL has no scope name in path',
-                            scope: s
-                        });
-                    }
-                } catch (err) {
-                    // Invalid URL format
-                    this.logger.warn({
-                        msg: 'Failed to parse MS Graph scope URL',
-                        scope: s,
-                        err: err.message
-                    });
-                }
-                // Return as-is if not a recognized Graph URL or parsing failed
-                return s;
-            });
-
-            const hasSendScope = normalizedScopes.some(s => s === OUTLOOK_API_SCOPES.send);
-            const hasReadScope = normalizedScopes.some(s => s === OUTLOOK_API_SCOPES.read || s === OUTLOOK_API_SCOPES.readWrite);
-            return { hasSendScope, hasReadScope };
-        }
-
-        return { hasSendScope: false, hasReadScope: false };
+        return checkScopes(provider, scopes, this.logger);
     }
 
     /**
@@ -342,10 +230,7 @@ class Account {
                     accountData.oauth2 && accountData.oauth2.provider && accountData.oauth2.provider !== accountData.type
                         ? accountData.oauth2.provider
                         : undefined,
-                state:
-                    !runIndex || runIndex <= accountData.runIndex || ['init', 'unset'].includes(accountData.state) || accountData.isApi
-                        ? accountData.state
-                        : 'init',
+                state: getDisplayState(accountData, runIndex),
 
                 webhooks: accountData.webhooks || undefined,
                 proxy: accountData.proxy || undefined,
@@ -355,10 +240,7 @@ class Account {
 
                 syncTime: accountData.sync,
 
-                lastError:
-                    accountData.state === 'connected' || !accountData.lastErrorState || !Object.keys(accountData.lastErrorState).length
-                        ? null
-                        : accountData.lastErrorState
+                lastError: formatLastError(accountData)
             }))
         };
 
@@ -819,53 +701,10 @@ class Account {
             }
         }
 
-        accountData.state =
-            !runIndex || runIndex <= accountData.runIndex || ['init', 'unset'].includes(accountData.state) || accountData.isApi ? accountData.state : 'init';
+        accountData.state = calculateEffectiveState(accountData, runIndex);
 
-        if (requireValid && !['connected', 'connecting', 'syncing'].includes(accountData.state)) {
-            let err;
-            switch (accountData.state) {
-                case 'init':
-                    err = new Error('Requested account is not yet initialized');
-                    err.code = 'NotYetConnected';
-                    break;
-                /*
-                // Check disabled for the following states - allow commands to go through.
-                // A secondary IMAP connection is opened if possible.
-                */
-                /*
-                case 'connecting':
-                case 'syncing':
-                    err = new Error('Requested account is not yet connected');
-                    err.code = 'NotYetConnected';
-                    break;
-                */
-                case 'authenticationError':
-                    err = new Error('Requested account can not be authenticated');
-                    err.code = 'AuthenticationFails';
-                    break;
-                case 'connectError':
-                    err = new Error('Can not establish server connection for requested account');
-                    err.code = 'ConnectionError';
-                    break;
-                case 'unset':
-                    err = new Error('Syncing is disabled for the requested account');
-                    err.code = 'NotSyncing';
-                    break;
-                default:
-                    err = new Error('Requested account currently not available');
-                    err.code = 'NoAvailable';
-                    break;
-            }
-
-            let error = Boom.boomify(err, { statusCode: 503 });
-            if (accountData.state) {
-                error.output.payload.state = accountData.state;
-            }
-            if (err.code) {
-                error.output.payload.code = err.code;
-            }
-            throw error;
+        if (requireValid) {
+            validateAccountState(accountData);
         }
 
         return accountData;
@@ -1059,9 +898,9 @@ class Account {
             .multi()
             .hgetall(this.getAccountKey())
             .hmset(this.getAccountKey(), this.serializeAccountData(accountData))
-            .hsetnx(this.getAccountKey(), 'state', 'init')
+            .hsetnx(this.getAccountKey(), 'state', ACCOUNT_STATES.INIT)
             .hsetnx(this.getAccountKey(), 'runIndex', runIndex.toString())
-            .hsetnx(this.getAccountKey(), `state:count:connected`, '0')
+            .hsetnx(this.getAccountKey(), `state:count:${ACCOUNT_STATES.CONNECTED}`, '0')
             .sadd(`${REDIS_PREFIX}ia:accounts`, this.account);
 
         if (accountData.oauth2 && accountData.oauth2.provider) {
@@ -1282,7 +1121,7 @@ class Account {
 
             // just pass through, do nothing
             return mailboxListing;
-        } else if (accountData.state === 'connected' || query.counters) {
+        } else if (accountData.state === ACCOUNT_STATES.CONNECTED || query.counters) {
             // run LIST
             mailboxListing = await this.listMailboxes(query);
             if (mailboxListing && mailboxListing.error) {
@@ -1292,7 +1131,7 @@ class Account {
                 }
                 throw error;
             }
-        } else if (accountData.state === 'unset') {
+        } else if (accountData.state === ACCOUNT_STATES.UNSET) {
             // account has not been set up yet
             let error = Boom.boomify(new Error('Syncing is disabled for the requested account'), { statusCode: 503 });
             if (accountData.state) {
@@ -1300,7 +1139,7 @@ class Account {
             }
             error.output.payload.code = 'NotSyncing';
             throw error;
-        } else if (accountData.state === 'init' || !(await this.redis.exists(this.getMailboxListKey()))) {
+        } else if (accountData.state === ACCOUNT_STATES.INIT || !(await this.redis.exists(this.getMailboxListKey()))) {
             // account has not been set up yet
             let error = Boom.boomify(new Error('Requested account is not yet initialized'), { statusCode: 503 });
             if (accountData.state) {
@@ -2413,7 +2252,7 @@ class Account {
                     // start syncing new messages from current time
                     .hset(this.getAccountKey(), 'notifyFrom', notifyFrom.toISOString())
                     // mark connection count to 0 to trigger `accountInitialized` event
-                    .hset(this.getAccountKey(), `state:count:connected`, '0')
+                    .hset(this.getAccountKey(), `state:count:${ACCOUNT_STATES.CONNECTED}`, '0')
                     .del(this.getMailboxListKey()) // mailbox list
                     // Note: mailbox ID hash (iah:) and listRegistry are intentionally NOT deleted
                     // to preserve message ID stability across reconnections. Message IDs depend on