emailengine-app 2.61.0 → 2.61.2

package/lib/email-client/gmail/gmail-api.js

@@ -0,0 +1,243 @@
+'use strict';
+
+const { metricsMeta } = require('../base-client');
+
+// Gmail API configuration
+const GMAIL_API_BASE = 'https://gmail.googleapis.com';
+
+// Maximum concurrent listing requests
+const LIST_BATCH_SIZE = 10;
+
+// Rate limiting configuration
+const MAX_RETRY_ATTEMPTS = 3;
+const RETRY_BASE_DELAY = 1000; // 1 second base delay
+
+// Gmail API error code mapping to internal error codes
+// https://developers.google.com/gmail/api/reference/rest#error-codes
+const GMAIL_ERROR_MAP = {
+    INVALID_ARGUMENT: { code: 'InvalidArgument', status: 400 },
+    FAILED_PRECONDITION: { code: 'FailedPrecondition', status: 400 },
+    NOT_FOUND: { code: 'NotFound', status: 404 },
+    PERMISSION_DENIED: { code: 'PermissionDenied', status: 403 },
+    RESOURCE_EXHAUSTED: { code: 'RateLimitExceeded', status: 429 },
+    UNAUTHENTICATED: { code: 'Unauthenticated', status: 401 },
+    INTERNAL: { code: 'InternalError', status: 500 },
+    UNAVAILABLE: { code: 'ServiceUnavailable', status: 503 }
+};
+
+/**
+ * Creates an error from a Gmail API error response
+ * @param {Object} gmailError - The error object from Gmail API
+ * @param {string} gmailErrorStatus - The error status from Gmail API
+ * @returns {Error|null} A formatted error object or null if not mappable
+ */
+function createGmailError(gmailError, gmailErrorStatus) {
+    const mappedError = GMAIL_ERROR_MAP[gmailErrorStatus];
+    if (!mappedError) {
+        return null;
+    }
+
+    const error = new Error(gmailError?.message || gmailErrorStatus);
+    error.code = mappedError.code;
+    error.statusCode = mappedError.status;
+    error.gmailErrorStatus = gmailErrorStatus;
+    return error;
+}
+
+/**
+ * Checks if an error indicates rate limiting
+ * @param {Object} err - The error object
+ * @returns {boolean} True if rate limited
+ */
+function isRateLimitError(err) {
+    const status = err.oauthRequest?.status;
+    const errorReason = err.oauthRequest?.response?.error?.errors?.[0]?.reason;
+
+    return status === 429 || errorReason === 'rateLimitExceeded' || errorReason === 'userRateLimitExceeded';
+}
+
+/**
+ * Calculates retry delay with exponential backoff and jitter
+ * @param {Object} err - The error object (may contain Retry-After header)
+ * @param {number} attempt - Current attempt number (0-indexed)
+ * @returns {number} Delay in milliseconds
+ */
+function calculateRetryDelay(err, attempt) {
+    const retryAfter = err.oauthRequest?.headers?.['retry-after'];
+
+    // Use Retry-After header if available, otherwise exponential backoff
+    let delay = retryAfter ? parseInt(retryAfter, 10) * 1000 : RETRY_BASE_DELAY * Math.pow(2, attempt);
+
+    // Add jitter (0-500ms) to prevent synchronized retries
+    delay += Math.random() * 500;
+
+    return delay;
+}
+
+/**
+ * Makes authenticated requests to Gmail API with automatic retry on rate limiting
+ * Implements exponential backoff with jitter
+ *
+ * @param {Object} context - The client context (GmailClient instance)
+ * @param {string} url - API endpoint URL
+ * @param {string} [method='get'] - HTTP method
+ * @param {*} [payload] - Request payload
+ * @param {Object} [options={}] - Request options
+ * @returns {Promise<*>} API response
+ */
+async function request(context, url, method, payload, options = {}) {
+    const maxRetries = options.maxRetries ?? MAX_RETRY_ATTEMPTS;
+    let lastError;
+
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        let result, accessToken;
+
+        try {
+            accessToken = await context.getToken();
+        } catch (err) {
+            context.logger.error({ msg: 'Failed to load access token', account: context.account, err });
+            throw err;
+        }
+
+        try {
+            if (!context.oAuth2Client) {
+                await context.getClient();
+            }
+
+            result = await context.oAuth2Client.request(accessToken, url, method, payload, options);
+
+            // Track successful API request
+            metricsMeta({ account: context.account }, context.logger, 'oauth2ApiRequest', 'inc', {
+                status: 'success',
+                provider: 'gmail',
+                statusCode: '200'
+            });
+
+            return result;
+        } catch (err) {
+            lastError = err;
+
+            // Check if this is a rate limit error
+            if (isRateLimitError(err) && attempt < maxRetries) {
+                const delay = calculateRetryDelay(err, attempt);
+
+                context.logger.warn({
+                    msg: 'Rate limited by Gmail API, retrying',
+                    account: context.account,
+                    attempt: attempt + 1,
+                    maxRetries,
+                    delay,
+                    errorReason: err.oauthRequest?.response?.error?.errors?.[0]?.reason
+                });
+
+                metricsMeta({ account: context.account }, context.logger, 'oauth2ApiRequest', 'inc', {
+                    status: 'rate_limited',
+                    provider: 'gmail',
+                    statusCode: '429'
+                });
+
+                await new Promise(resolve => setTimeout(resolve, delay));
+                continue;
+            }
+
+            // Log client errors (4xx) at debug level - these are expected operational errors
+            // Log server errors (5xx) and other failures at error level
+            const status = err.oauthRequest?.status;
+            const isClientError = status >= 400 && status < 500;
+
+            if (isClientError) {
+                context.logger.debug({ msg: 'API request failed with client error', account: context.account, err });
+            } else {
+                context.logger.error({ msg: 'Failed to run API request', account: context.account, err });
+            }
+
+            // Track failed API request
+            const statusCode = String(err.oauthRequest?.status || 0);
+            metricsMeta({ account: context.account }, context.logger, 'oauth2ApiRequest', 'inc', {
+                status: 'failure',
+                provider: 'gmail',
+                statusCode
+            });
+
+            throw err;
+        }
+    }
+
+    // If we exhausted all retries, throw the last error
+    throw lastError;
+}
+
+/**
+ * Builds a Gmail API URL for a specific endpoint
+ * @param {string} endpoint - The API endpoint path (e.g., '/users/me/messages')
+ * @returns {string} Full API URL
+ */
+function buildApiUrl(endpoint) {
+    // Remove leading slash if present to avoid double slashes
+    const path = endpoint.startsWith('/') ? endpoint : '/' + endpoint;
+    return `${GMAIL_API_BASE}/gmail/v1${path}`;
+}
+
+/**
+ * Executes batch API requests with concurrency control
+ *
+ * @param {Object} context - The client context (GmailClient instance)
+ * @param {Array<Object>} items - Array of items to process
+ * @param {Function} requestFn - Function that takes an item and returns a promise
+ * @param {number} [batchSize=LIST_BATCH_SIZE] - Maximum concurrent requests
+ * @returns {Promise<Array>} Array of results
+ */
+async function executeBatchRequests(context, items, requestFn, batchSize = LIST_BATCH_SIZE) {
+    const results = [];
+    let batch = [];
+
+    const processBatch = async () => {
+        if (batch.length === 0) {
+            return;
+        }
+
+        const batchResults = await Promise.allSettled(batch);
+
+        for (const entry of batchResults) {
+            if (entry.status === 'rejected') {
+                throw entry.reason;
+            }
+            if (entry.value) {
+                results.push(entry.value);
+            }
+        }
+
+        batch = [];
+    };
+
+    for (const item of items) {
+        batch.push(requestFn(item));
+
+        if (batch.length >= batchSize) {
+            await processBatch();
+        }
+    }
+
+    await processBatch();
+
+    return results;
+}
+
+module.exports = {
+    // Constants
+    GMAIL_API_BASE,
+    LIST_BATCH_SIZE,
+    MAX_RETRY_ATTEMPTS,
+    RETRY_BASE_DELAY,
+    GMAIL_ERROR_MAP,
+
+    // Request functions
+    request,
+    buildApiUrl,
+    executeBatchRequests,
+
+    // Error handling
+    createGmailError,
+    isRateLimitError,
+    calculateRetryDelay
+};

package/lib/email-client/gmail-client.js

@@ -2,6 +2,7 @@
 
 const { Account } = require('../account');
 const { oauth2Apps } = require('../oauth2-apps');
+const { checkAccountScopes } = require('../oauth/scope-checker');
 const getSecret = require('../get-secret');
 const msgpack = require('msgpack5')();
 const addressparser = require('nodemailer/lib/addressparser');
@@ -22,9 +23,7 @@ const {
     AUTH_SUCCESS_NOTIFY
 } = require('../consts');
 
-// Gmail API configuration
-const GMAIL_API_BASE = 'https://gmail.googleapis.com';
-const LIST_BATCH_SIZE = 10; // how many listing requests to run at the same time
+const { GMAIL_API_BASE, LIST_BATCH_SIZE, request: gmailApiRequest } = require('./gmail/gmail-api');
 
 // Labels to exclude from folder listings
 const SKIP_LABELS = ['UNREAD', 'STARRED', 'IMPORTANT', 'CHAT', 'CATEGORY_PERSONAL'];
@@ -61,6 +60,7 @@ for (let label of Object.keys(SYSTEM_LABELS)) {
 // Timing constants for Gmail Pub/Sub watch
 const RENEW_WATCH_TTL = 60 * 60 * 1000; // 1h - how often to check if watch needs renewal
 const MIN_WATCH_TTL = 24 * 3600 * 1000; // 1day - minimum time before renewing watch
+const FALLBACK_POLLING_INTERVAL = 10 * 60 * 1000; // 10min - fallback polling if no Pub/Sub notifications
 
 /*
 Gmail API implementation status:
@@ -251,47 +251,15 @@ class GmailClient extends BaseClient {
 
     /**
      * Makes authenticated request to Gmail API
-     * Handles token refresh automatically
-     * @param {...any} args - Request parameters
+     * Handles token refresh and rate limit retries automatically
+     * @param {string} url - API endpoint URL
+     * @param {string} [method='get'] - HTTP method
+     * @param {*} [payload] - Request payload
+     * @param {Object} [options={}] - Request options
      * @returns {Object} API response
      */
-    async request(...args) {
-        let result, accessToken;
-        try {
-            accessToken = await this.getToken();
-        } catch (err) {
-            this.logger.error({ msg: 'Failed to load access token', account: this.account, err });
-            throw err;
-        }
-
-        try {
-            if (!this.oAuth2Client) {
-                await this.getClient();
-            }
-            result = await this.oAuth2Client.request(accessToken, ...args);
-
-            // Track successful API request
-            metricsMeta({ account: this.account }, this.logger, 'oauth2ApiRequest', 'inc', { status: 'success', provider: 'gmail', statusCode: '200' });
-        } catch (err) {
-            // Log client errors (4xx) at debug level - these are expected operational errors
-            // Log server errors (5xx) and other failures at error level
-            const status = err.oauthRequest?.status;
-            const isClientError = status >= 400 && status < 500;
-
-            if (isClientError) {
-                this.logger.debug({ msg: 'API request failed with client error', account: this.account, err });
-            } else {
-                this.logger.error({ msg: 'Failed to run API request', account: this.account, err });
-            }
-
-            // Track failed API request
-            const statusCode = String(err.oauthRequest?.status || 0);
-            metricsMeta({ account: this.account }, this.logger, 'oauth2ApiRequest', 'inc', { status: 'failure', provider: 'gmail', statusCode });
-
-            throw err;
-        }
-
-        return result;
+    async request(url, method, payload, options) {
+        return gmailApiRequest(this, url, method, payload, options);
     }
 
     // PUBLIC METHODS
@@ -326,7 +294,7 @@ class GmailClient extends BaseClient {
 
         // Check if send-only mode
         const scopes = accountData.oauth2?.accessToken?.scope || accountData.oauth2?.scope || [];
-        const { hasSendScope, hasReadScope } = this.accountObject.checkAccountScopes('gmail', scopes);
+        const { hasSendScope, hasReadScope } = checkAccountScopes('gmail', scopes);
         const isSendOnly = hasSendScope && !hasReadScope;
 
         this.logger.info({
@@ -453,6 +421,10 @@ class GmailClient extends BaseClient {
 
             // Schedule periodic watch renewal (only for full access accounts)
             this.setupRenewWatchTimer();
+
+            // Schedule fallback polling in case Pub/Sub notifications are dropped
+            this.lastNotificationTime = Date.now();
+            this.setupFallbackPollingTimer();
         }
 
         // Determine if this is a reconnection after error
@@ -501,8 +473,14 @@ class GmailClient extends BaseClient {
      */
     async close() {
         clearTimeout(this.renewWatchTimer);
+        clearTimeout(this.fallbackPollingTimer);
         this.closed = true;
 
+        // Clean up cached data
+        this.cachedLabels = null;
+        this.cachedLabelsTime = null;
+        this.pendingHistoryId = null;
+
         if (['init', 'connecting', 'syncing', 'connected'].includes(this.state)) {
             this.state = 'disconnected';
             await this.setStateVal();
@@ -518,8 +496,14 @@ class GmailClient extends BaseClient {
 
     async delete() {
         clearTimeout(this.renewWatchTimer);
+        clearTimeout(this.fallbackPollingTimer);
         this.closed = true;
 
+        // Clean up cached data
+        this.cachedLabels = null;
+        this.cachedLabelsTime = null;
+        this.pendingHistoryId = null;
+
         if (['init', 'connecting', 'syncing', 'connected'].includes(this.state)) {
             this.state = 'disconnected';
             await this.setStateVal();
@@ -1074,7 +1058,7 @@ class GmailClient extends BaseClient {
             }
 
             for (let flag of [].concat(updates.flags.delete || [])) {
-                labelUpdates.push(this.flagToLabel(flag), true);
+                labelUpdates.push(this.flagToLabel(flag, true));
             }
 
             labelUpdates
@@ -1564,9 +1548,9 @@ class GmailClient extends BaseClient {
 
         let gmailMessageId;
         if (submitInfo?.id) {
-            // Detect send-only mode - use same scope resolution as in account.js
+            // Detect send-only mode using centralized scope checker
             const scopes = accountData.oauth2?.accessToken?.scope || accountData.oauth2?.scope || [];
-            const { hasSendScope, hasReadScope } = this.accountObject.checkAccountScopes('gmail', scopes);
+            const { hasSendScope, hasReadScope } = checkAccountScopes('gmail', scopes);
             const isSendOnly = hasSendScope && !hasReadScope;
 
             if (!isSendOnly) {
@@ -1797,6 +1781,9 @@ class GmailClient extends BaseClient {
      * @returns {boolean} Processing result
      */
     async externalNotify(message) {
+        // Track notification time for fallback polling
+        this.lastNotificationTime = Date.now();
+
         let { historyId } = message || {};
 
         let existingHistoryId = Number(await this.redis.hget(this.getAccountKey(), 'googleHistoryId')) || null;
@@ -1882,28 +1869,93 @@ class GmailClient extends BaseClient {
 
     /**
      * Sets up timer to periodically renew Gmail watch subscription
+     * Uses actual watch expiration if available for smarter scheduling
      */
     setupRenewWatchTimer() {
         if (this.closed) {
             return;
         }
         clearTimeout(this.renewWatchTimer);
+
+        // Calculate optimal delay based on watch expiration
+        let delay = RENEW_WATCH_TTL;
+        if (this.watchExpiration) {
+            // Renew 1 hour before expiration
+            const renewAt = this.watchExpiration - 60 * 60 * 1000;
+            const timeUntilRenewal = renewAt - Date.now();
+            // Use the calculated time, but not less than RENEW_WATCH_TTL
+            delay = Math.max(timeUntilRenewal, RENEW_WATCH_TTL);
+        }
+
         this.renewWatchTimer = setTimeout(() => {
             if (this.closed) {
                 return;
             }
+            let authError = false;
             this.renewWatch()
                 .catch(err => {
                     this.logger.error({ msg: 'Failed to renew Gmail subscription watch', account: this.account, err });
+                    // Check if this is a permanent auth failure
+                    if (err.code === 'ETokenRefresh' || this.state === 'authenticationError') {
+                        authError = true;
+                    }
                 })
                 .finally(() => {
-                    // restart timer
-                    this.setupRenewWatchTimer();
+                    // Don't restart timer on auth failures - let the auth flow handle recovery
+                    if (!this.closed && !authError && this.state !== 'authenticationError') {
+                        this.setupRenewWatchTimer();
+                    }
                 });
-        }, RENEW_WATCH_TTL);
+        }, delay);
         this.renewWatchTimer.unref();
     }
 
+    /**
+     * Sets up fallback polling timer to check for missed notifications
+     * If no Pub/Sub notifications received within the interval, triggers a proactive sync
+     */
+    setupFallbackPollingTimer() {
+        if (this.closed) {
+            return;
+        }
+        clearTimeout(this.fallbackPollingTimer);
+        this.fallbackPollingTimer = setTimeout(async () => {
+            if (this.closed) {
+                return;
+            }
+
+            const timeSinceNotification = Date.now() - (this.lastNotificationTime || 0);
+            if (timeSinceNotification >= FALLBACK_POLLING_INTERVAL) {
+                // No notifications received within the interval, do a proactive sync
+                this.logger.info({
+                    msg: 'No Pub/Sub notifications received, triggering fallback sync',
+                    account: this.account,
+                    timeSinceNotification
+                });
+
+                try {
+                    const historyId = await this.redis.hget(this.getAccountKey(), 'googleHistoryId');
+                    if (historyId) {
+                        // Trigger sync to check for any missed changes
+                        this.triggerSync(Number(historyId), Number(historyId));
+                    }
+                } catch (err) {
+                    this.logger.error({
+                        msg: 'Failed to trigger fallback sync',
+                        account: this.account,
+                        err
+                    });
+                }
+            }
+
+            // Restart the timer
+            if (!this.closed && this.state !== 'authenticationError') {
+                this.setupFallbackPollingTimer();
+            }
+        }, FALLBACK_POLLING_INTERVAL);
+        this.fallbackPollingTimer.unref();
+    }
+
     /**
      * Renews Gmail Pub/Sub watch subscription
      * @param {Object} accountData - Account data
@@ -1930,15 +1982,22 @@ class GmailClient extends BaseClient {
                         topicName: appData?.pubSubTopic
                     });
                     // { historyId: '3663748', expiration: '1720183655953' }
+
+                    // Store expiration for smarter renewal scheduling
+                    const watchExpiration = watchResponse?.expiration ? Number(watchResponse.expiration) : null;
+                    this.watchExpiration = watchExpiration;
+
                     await this.accountObject.update({
                         lastWatch: new Date(now),
                         watchResponse,
+                        watchExpiration,
                         watchFailure: null
                     });
                     this.logger.info({
                         msg: 'Renewed Gmail pubsub watch',
                         account: this.account,
-                        watchResponse
+                        watchResponse,
+                        watchExpiration: watchExpiration ? new Date(watchExpiration).toISOString() : null
                     });
                 } catch (err) {
                     await this.accountObject.update({
@@ -2486,15 +2545,37 @@ class GmailClient extends BaseClient {
      */
     triggerSync(currentHistoryId, updatedHistoryId) {
         if (this.processingHistory) {
+            // Queue the latest historyId instead of dropping the notification
+            this.pendingHistoryId = Math.max(this.pendingHistoryId || 0, updatedHistoryId);
+            this.logger.debug({
+                msg: 'Sync already in progress, queued pending historyId',
+                account: this.account,
+                pendingHistoryId: this.pendingHistoryId
+            });
             return;
         }
         this.processingHistory = true;
+        const processedHistoryId = updatedHistoryId;
         this.processHistory(currentHistoryId, updatedHistoryId)
             .catch(err => {
                 this.logger.error({ msg: 'Failed to process account history', currentHistoryId, updatedHistoryId, account: this.account, err });
             })
             .finally(() => {
                 this.processingHistory = false;
+                // Process any queued sync notifications
+                if (this.pendingHistoryId && this.pendingHistoryId > processedHistoryId) {
+                    const pending = this.pendingHistoryId;
+                    this.pendingHistoryId = null;
+                    this.logger.debug({
+                        msg: 'Processing queued historyId',
+                        account: this.account,
+                        fromHistoryId: processedHistoryId,
+                        toHistoryId: pending
+                    });
+                    this.triggerSync(processedHistoryId, pending);
+                } else {
+                    this.pendingHistoryId = null;
+                }
             });
     }
 
@@ -2668,9 +2749,20 @@ class GmailClient extends BaseClient {
             } catch (err) {
                 switch (err?.oauthRequest?.response?.error?.code) {
                     case 404: {
-                        // History ID too old
-                        this.logger.info({ msg: 'Provided history ID is too old', account: this.account, historyId: currentHistoryId, updatedHistoryId, err });
-                        // set to newest known value, ignore missed entries
+                        // History ID too old - some changes may have been missed
+                        this.logger.warn({
+                            msg: 'History ID too old, some email changes may have been missed',
+                            account: this.account,
+                            currentHistoryId,
+                            updatedHistoryId,
+                            err
+                        });
+                        // Emit warning event so the account can be flagged for attention
+                        await emitChangeEvent(this.logger, this.account, 'syncWarning', {
+                            type: 'historyIdExpired',
+                            message: 'Some email changes may have been missed due to expired history ID'
+                        });
+                        // Set to newest known value
                         newestHistoryId = updatedHistoryId;
                         return;
                     }