emailengine-app 2.63.3 → 2.64.0

package/workers/api.js

@@ -163,7 +163,8 @@ const {
     googleSubscriptionNameSchema,
     messageReferenceSchema,
     idempotencyKeySchema,
-    headerTimeoutSchema
+    headerTimeoutSchema,
+    pubSubErrorSchema
 } = require('../lib/schemas');
 
 const OAuth2ProviderSchema = Joi.string()
@@ -180,6 +181,21 @@ const AccountTypeSchema = Joi.string()
     .required()
     .label('AccountType');
 
+function flattenOAuthAppMeta(app) {
+    if (!app.meta) {
+        return;
+    }
+    let authFlag = app.meta.authFlag;
+    let pubSubFlag = app.meta.pubSubFlag;
+    delete app.meta;
+    if (authFlag && authFlag.message) {
+        app.lastError = { response: authFlag.message };
+    }
+    if (pubSubFlag && pubSubFlag.message) {
+        app.pubSubError = { message: pubSubFlag.message, description: pubSubFlag.description || null };
+    }
+}
+
 const SUPPORTED_LOCALES = locales.map(locale => locale.locale);
 
 const FLAG_SORT_ORDER = ['\\Inbox', '\\Flagged', '\\Sent', '\\Drafts', '\\All', '\\Archive', '\\Junk', '\\Trash'];
@@ -362,6 +378,7 @@ async function call(message, transferList) {
             err.statusCode = 504;
             err.code = 'Timeout';
             err.ttl = ttl;
+            callQueue.delete(mid);
             reject(err);
         }, ttl);
 
@@ -1557,11 +1574,12 @@ Include your token in requests using one of these methods:
             let checkKey = `${REDIS_PREFIX}test:${Date.now()}`;
             let expected = crypto.randomBytes(8).toString('hex');
             let res = await redis.multi().set(checkKey, expected).get(checkKey).del(checkKey).exec();
-            if (res[1] && res[1][1] === expected && res[2] && res[2][1] === 1) {
-                return { success: true };
+            if (!(res[1] && res[1][1] === expected && res[2] && res[2][1] === 1)) {
+                let error = Boom.boomify(new Error('Database check failed'), { statusCode: 500 });
+                throw error;
             }
-            let error = Boom.boomify(new Error('Database check failed'), { statusCode: 500 });
-            throw error;
+
+            return { success: true };
         },
         options: {
             description: 'Health check',
@@ -1570,6 +1588,99 @@ Include your token in requests using one of these methods:
         }
     });
 
+    server.route({
+        method: 'GET',
+        path: '/v1/pubsub/status',
+
+        async handler(request) {
+            try {
+                let response = await oauth2Apps.list(request.query.page, request.query.pageSize, { pubsub: true });
+
+                let apps = response.apps.map(app => {
+                    flattenOAuthAppMeta(app);
+                    return { id: app.id, name: app.name || null, lastError: app.lastError || null, pubSubError: app.pubSubError || null };
+                });
+
+                return {
+                    total: response.total,
+                    page: response.page,
+                    pages: response.pages,
+                    apps
+                };
+            } catch (err) {
+                request.logger.error({ msg: 'API request failed', err });
+                if (Boom.isBoom(err)) {
+                    throw err;
+                }
+                let error = Boom.boomify(err, { statusCode: err.statusCode || 500 });
+                if (err.code) {
+                    error.output.payload.code = err.code;
+                }
+                throw error;
+            }
+        },
+
+        options: {
+            description: 'List Pub/Sub status',
+            notes: 'Lists Pub/Sub enabled OAuth2 applications and their subscription status',
+            tags: ['api', 'OAuth2 Applications'],
+
+            plugins: {},
+
+            auth: {
+                strategy: 'api-token',
+                mode: 'required'
+            },
+            cors: CORS_CONFIG,
+
+            validate: {
+                options: {
+                    stripUnknown: false,
+                    abortEarly: false,
+                    convert: true
+                },
+                failAction,
+
+                query: Joi.object({
+                    page: Joi.number()
+                        .integer()
+                        .min(0)
+                        .max(1024 * 1024)
+                        .default(0)
+                        .example(0)
+                        .description('Page number (zero indexed, so use 0 for first page)')
+                        .label('PageNumber'),
+                    pageSize: Joi.number().integer().min(1).max(1000).default(20).example(20).description('How many entries per page').label('PageSize')
+                }).label('PubSubStatusFilter')
+            },
+
+            response: {
+                schema: Joi.object({
+                    total: Joi.number().integer().example(120).description('How many matching entries').label('TotalNumber'),
+                    page: Joi.number().integer().example(0).description('Current page (0-based index)').label('PageNumber'),
+                    pages: Joi.number().integer().example(24).description('Total page count').label('PagesNumber'),
+
+                    apps: Joi.array()
+                        .items(
+                            Joi.object({
+                                id: Joi.string().max(256).required().example('AAABhaBPHscAAAAH').description('OAuth2 application ID'),
+                                name: Joi.string().allow(null).max(256).example('My Gmail App').description('Display name for the app'),
+                                lastError: Joi.object({
+                                    response: Joi.string().example('Enable the Cloud Pub/Sub API').description('Setup error message')
+                                })
+                                    .allow(null)
+                                    .description('Setup error from ensurePubsub, if any')
+                                    .label('PubSubSetupError'),
+                                pubSubError: pubSubErrorSchema.allow(null)
+                            }).label('PubSubAppStatus')
+                        )
+                        .label('PubSubAppStatusList')
+                }).label('PubSubStatusResponse'),
+                failAction: 'log'
+            }
+        }
+    });
+
     server.route({
         method: 'GET',
         path: '/redirect',
@@ -1936,6 +2047,10 @@ Include your token in requests using one of these methods:
 
             const outlookSubscription = accountData.outlookSubscription;
 
+            // Deduplicate lifecycle events within the same batch to prevent
+            // concurrent handlers racing (e.g., two subscriptionRemoved entries)
+            const seenLifecycleEvents = new Set();
+
             for (let entry of (request.payload && request.payload.value) || []) {
                 request.logger.debug({
                     msg: 'MS Graph subscription event',
@@ -1960,82 +2075,43 @@ Include your token in requests using one of these methods:
                     continue;
                 }
 
-                switch (entry.lifecycleEvent) {
-                    case 'reauthorizationRequired': {
-                        // Microsoft is requesting reauthorization - force renewal immediately
-                        request.logger.info({
-                            msg: 'Received reauthorizationRequired lifecycle event',
-                            subscriptionId: outlookSubscription.id,
+                // Route recognized lifecycle events to the IMAP worker
+                // so the live client with its OAuth state handles them
+                if (entry.lifecycleEvent === 'reauthorizationRequired' || entry.lifecycleEvent === 'subscriptionRemoved') {
+                    const dedupeKey = `${entry.lifecycleEvent}:${entry.subscriptionId}`;
+                    if (seenLifecycleEvents.has(dedupeKey)) {
+                        request.logger.debug({
+                            msg: 'Skipping duplicate lifecycle event in batch',
+                            lifecycleEvent: entry.lifecycleEvent,
+                            subscriptionId: entry.subscriptionId,
                             account: request.query.account
                         });
-
-                        // Use the unified renewal method from OutlookClient
-                        // We need to create a client instance to call the renewal method
-                        const { OutlookClient } = require('../lib/email-client/outlook-client');
-                        const client = new OutlookClient(accountData, {
-                            redis,
-                            secret: await getSecret(),
-                            logger: request.logger
-                        });
-
-                        try {
-                            // Force renewal when we get reauthorizationRequired
-                            const renewalResult = await client.renewSubscription(true);
-
-                            if (renewalResult.success) {
-                                request.logger.info({
-                                    msg: 'Successfully renewed subscription from lifecycle event',
-                                    subscriptionId: outlookSubscription.id,
-                                    account: request.query.account,
-                                    newExpirationDateTime: renewalResult.expirationDateTime
-                                });
-                            } else {
-                                request.logger.error({
-                                    msg: 'Failed to renew subscription from lifecycle event',
-                                    subscriptionId: outlookSubscription.id,
-                                    account: request.query.account,
-                                    reason: renewalResult.reason,
-                                    error: renewalResult.error
-                                });
-                            }
-                        } catch (err) {
-                            request.logger.error({
-                                msg: 'Exception while renewing subscription from lifecycle event',
-                                subscriptionId: outlookSubscription.id,
-                                account: request.query.account,
-                                err
-                            });
-                        } finally {
-                            // Clean up client instance
-                            if (client && typeof client.close === 'function') {
-                                try {
-                                    await client.close();
-                                } catch (cleanupErr) {
-                                    request.logger.debug({
-                                        msg: 'Error closing client after lifecycle renewal',
-                                        account: request.query.account,
-                                        err: cleanupErr
-                                    });
-                                }
-                            }
-                        }
-
-                        break;
+                        continue;
                     }
+                    seenLifecycleEvents.add(dedupeKey);
 
-                    case 'subscriptionRemoved': {
-                        // subscription was removed, should we recreate it?
-                        await accountObject.update({
-                            outlookSubscription: {
-                                state: {
-                                    state: 'error',
-                                    error: `Subscription removed`,
-                                    time: Date.now()
-                                }
-                            }
+                    request.logger.info({
+                        msg: 'Received lifecycle event',
+                        lifecycleEvent: entry.lifecycleEvent,
+                        subscriptionId: outlookSubscription.id,
+                        account: request.query.account
+                    });
+
+                    // Fire-and-forget: return HTTP 202 immediately so Microsoft
+                    // does not time out the lifecycle webhook delivery
+                    call({
+                        cmd: 'subscriptionLifecycle',
+                        account: request.query.account,
+                        event: entry.lifecycleEvent,
+                        timeout: consts.OUTLOOK_SUBSCRIPTION_LOCK_TTL
+                    }).catch(err => {
+                        request.logger.error({
+                            msg: 'Failed to handle lifecycle event via worker',
+                            account: request.query.account,
+                            lifecycleEvent: entry.lifecycleEvent,
+                            err
                         });
-                        break;
-                    }
+                    });
                 }
             }
 
@@ -4876,13 +4952,7 @@ Include your token in requests using one of these methods:
                         delete app.app;
                     }
 
-                    if (app.meta) {
-                        let authFlag = app.meta.authFlag;
-                        delete app.meta;
-                        if (authFlag && authFlag.message) {
-                            app.lastError = { response: authFlag.message };
-                        }
-                    }
+                    flattenOAuthAppMeta(app);
                 }
 
                 return response;
@@ -5002,7 +5072,8 @@ Include your token in requests using one of these methods:
                                     .example('******')
                                     .description('PEM formatted service secret for 2-legged OAuth2 applications. Actual value is not revealed.'),
 
-                                lastError: lastErrorSchema.allow(null)
+                                lastError: lastErrorSchema.allow(null),
+                                pubSubError: pubSubErrorSchema.allow(null)
                             }).label('OAuth2ResponseItem')
                         )
                         .label('OAuth2Entries')
@@ -5035,13 +5106,7 @@ Include your token in requests using one of these methods:
                     delete app.app;
                 }
 
-                if (app.meta) {
-                    let authFlag = app.meta.authFlag;
-                    delete app.meta;
-                    if (authFlag && authFlag.message) {
-                        app.lastError = { response: authFlag.message };
-                    }
-                }
+                flattenOAuthAppMeta(app);
 
                 return app;
             } catch (err) {
@@ -5145,7 +5210,8 @@ Include your token in requests using one of these methods:
                         .example(12)
                         .description('The number of accounts registered with this application. Not available for legacy apps.'),
 
-                    lastError: lastErrorSchema.allow(null)
+                    lastError: lastErrorSchema.allow(null),
+                    pubSubError: pubSubErrorSchema.allow(null)
                 }).label('ApplicationResponse'),
                 failAction: 'log'
             }
@@ -5160,8 +5226,9 @@ Include your token in requests using one of these methods:
             try {
                 let result = await oauth2Apps.create(request.payload);
 
-                if (result && result.pubsubUpdates && result.pubsubUpdates.pubSubSubscription) {
+                if (result && result.pubsubUpdates && Object.keys(result.pubsubUpdates).length > 0) {
                     await call({ cmd: 'googlePubSub', app: result.id });
+                    delete result.pubsubUpdates;
                 }
 
                 return result;
@@ -5220,8 +5287,9 @@ Include your token in requests using one of these methods:
             try {
                 let result = await oauth2Apps.update(request.params.app, request.payload);
 
-                if (result && result.pubsubUpdates && result.pubsubUpdates.pubSubSubscription) {
+                if (result && result.pubsubUpdates && Object.keys(result.pubsubUpdates).length > 0) {
                     await call({ cmd: 'googlePubSub', app: result.id });
+                    delete result.pubsubUpdates;
                 }
 
                 return result;
@@ -5369,7 +5437,15 @@ Include your token in requests using one of these methods:
 
         async handler(request) {
             try {
-                return await oauth2Apps.del(request.params.app);
+                let result = await oauth2Apps.del(request.params.app);
+
+                try {
+                    await call({ cmd: 'googlePubSubRemove', app: request.params.app });
+                } catch (err) {
+                    request.logger.error({ msg: 'Failed to notify workers about OAuth2 app deletion', err, app: request.params.app });
+                }
+
+                return result;
             } catch (err) {
                 request.logger.error({ msg: 'API request failed', err });
                 if (Boom.isBoom(err)) {
@@ -6942,11 +7018,13 @@ ${now}`,
         // Replace error with friendly HTML
         const error = response;
         const ctx = {
+            statusCode: error.output.statusCode,
             message:
                 error.output.statusCode === 404
                     ? request.app.gt.gettext('Requested page not found')
                     : (error.output && error.output.payload && error.output.payload.message) || request.app.gt.gettext('Something went wrong'),
-            details: error.output && error.output.payload && error.output.payload.details
+            details: error.output && error.output.payload && error.output.payload.details,
+            templateLocale: request.app.locale
         };
 
         if (error.output && error.output.payload) {

package/workers/documents.js

@@ -72,6 +72,7 @@ async function call(message, transferList) {
             err.statusCode = 504;
             err.code = 'Timeout';
             err.ttl = ttl;
+            callQueue.delete(mid);
             reject(err);
         }, ttl);
 

package/workers/imap.js

@@ -78,10 +78,6 @@ class ConnectionHandler {
         this.mids = 0;
 
         this.accounts = new Map();
-
-        // Reconnection metrics tracking
-        this.reconnectMetrics = new Map(); // Track metrics per account
-        this.metricsWindow = 60000; // 1-minute window
     }
 
     async init() {
@@ -639,6 +635,34 @@ class ConnectionHandler {
         return await accountData.connection.externalNotify(message);
     }
 
+    async subscriptionLifecycle(message) {
+        if (!this.accounts.has(message.account)) {
+            throw NO_ACTIVE_HANDLER_RESP_ERR;
+        }
+
+        let accountData = this.accounts.get(message.account);
+        if (!accountData.connection) {
+            throw NO_ACTIVE_HANDLER_RESP_ERR;
+        }
+
+        let connection = accountData.connection;
+
+        switch (message.event) {
+            case 'reauthorizationRequired':
+                return await connection.renewSubscription({ force: true });
+
+            case 'subscriptionRemoved':
+                // Clear stored subscription since MS deleted it server-side, under lock
+                logger.info({ msg: 'Handling subscriptionRemoved lifecycle event', account: message.account });
+                await connection.ensureSubscription({ clearExisting: true });
+                return true;
+
+            default:
+                logger.warn({ msg: 'Unknown subscription lifecycle event', event: message.event, account: message.account });
+                return false;
+        }
+    }
+
     async getQuota(message) {
         if (!this.accounts.has(message.account)) {
             throw NO_ACTIVE_HANDLER_RESP_ERR;
@@ -725,49 +749,6 @@ class ConnectionHandler {
         };
     }
 
-    /**
-     * Track reconnection attempts for monitoring (without blocking)
-     * @param {string} account - Account identifier
-     */
-    trackReconnection(account) {
-        const now = Date.now();
-        const metrics = this.reconnectMetrics.get(account) || {
-            attempts: [],
-            warnings: 0
-        };
-
-        // Clean old attempts outside window
-        metrics.attempts = metrics.attempts.filter(t => now - t < this.metricsWindow);
-        metrics.attempts.push(now);
-
-        // Log warning if excessive reconnections
-        if (metrics.attempts.length > 20) {
-            // More than 20 per minute
-            metrics.warnings++;
-            logger.warn({
-                msg: 'Excessive reconnection rate detected',
-                account,
-                rate: `${metrics.attempts.length}/min`,
-                totalWarnings: metrics.warnings
-            });
-
-            // Emit metrics for monitoring/alerting
-            try {
-                parentPort.postMessage({
-                    cmd: 'metrics',
-                    key: 'imap.reconnect.excessive',
-                    method: 'inc',
-                    args: [1],
-                    meta: { account }
-                });
-            } catch (err) {
-                logger.error({ msg: 'Failed to send metrics', err });
-            }
-        }
-
-        this.reconnectMetrics.set(account, metrics);
-    }
-
     async getAttachment(message) {
         if (!this.accounts.has(message.account)) {
             throw NO_ACTIVE_HANDLER_RESP_ERR;
@@ -892,6 +873,7 @@ class ConnectionHandler {
             case 'uploadMessage':
             case 'subconnections':
             case 'externalNotify':
+            case 'subscriptionLifecycle':
             case 'listSignatures':
                 return await this[message.cmd](message);
 
@@ -940,6 +922,7 @@ class ConnectionHandler {
                 err.statusCode = 504;
                 err.code = 'Timeout';
                 err.ttl = ttl;
+                this.callQueue.delete(mid);
                 reject(err);
             }, ttl);
 

package/workers/smtp.js

@@ -77,6 +77,7 @@ async function call(message, transferList) {
             err.statusCode = 504;
             err.code = 'Timeout';
             err.ttl = ttl;
+            callQueue.delete(mid);
             reject(err);
         }, ttl);
 

package/workers/submit.js

@@ -85,6 +85,7 @@ async function call(message, transferList) {
             err.statusCode = 504;
             err.code = 'Timeout';
             err.ttl = ttl;
+            callQueue.delete(mid);
             reject(err);
         }, ttl);
 

package/workers/webhooks.js

@@ -67,6 +67,7 @@ async function call(message, transferList) {
             err.statusCode = 504;
             err.code = 'Timeout';
             err.ttl = ttl;
+            callQueue.delete(mid);
             reject(err);
         }, ttl);
 
@@ -117,6 +118,14 @@ async function onCommand(command) {
         case 'googlePubSub':
             await googlePubSub.update(command.app);
             return true;
+        case 'googlePubSubRemove':
+            googlePubSub.remove(command.app);
+            return true;
+        case 'close':
+            clearTimeout(startRetryTimer);
+            googlePubSub.stopAll();
+            await notifyWorker.close(true);
+            return true;
         default:
             logger.debug({ msg: 'Unhandled command', command });
             return 999;
@@ -132,6 +141,18 @@ setInterval(() => {
     }
 }, 10 * 1000).unref();
 
+// Clean up Pub/Sub instances when parent port closes
+parentPort.on('close', () => {
+    clearTimeout(startRetryTimer);
+    googlePubSub.stopAll();
+    // notifyWorker.close() may throw synchronously if not yet initialized
+    try {
+        notifyWorker.close(true).catch(() => {});
+    } catch {
+        // ignore
+    }
+});
+
 // Send initial ready signal
 parentPort.postMessage({ cmd: 'ready' });
 
@@ -461,28 +482,6 @@ const notifyWorker = new Worker(
                 status: 'success'
             });
         } catch (err) {
-            /*
-            // do not disable by default
-            if (err.status === 410) {
-                // disable webhook
-                logger.error({
-                    msg: 'Webhooks were disabled by server',
-                    action: 'webhook',
-                    queue: job.queue.name,
-                    code: 'disabled_by_server',
-                    job: job.id,
-                    webhooks,
-                    accountWebhooks: !!accountWebhooks,
-                    event: job.name,
-                    status: err.status,
-                    account: job.data.account,
-route: customRoute && customRoute.id,
-                    err
-                });
-                await settings.set('webhooksEnabled', false);
-                return;
-            }
-            */
             logger.error({
                 msg: 'Failed posting webhook',
                 action: 'webhook',
@@ -606,13 +605,26 @@ notifyWorker.on('failed', async job => {
     });
 });
 
-googlePubSub
-    .start()
-    .then(() => {
-        logger.info({ msg: 'Started processing Google pub/sub' });
-    })
-    .catch(err => {
-        logger.fatal({ msg: 'Failed to start processing Google pub/sub', err });
-    });
+let startRetryTimer = null;
+
+(function startGooglePubSub(attempt) {
+    googlePubSub
+        .start()
+        .then(() => {
+            logger.info({ msg: 'Started processing Google pub/sub' });
+        })
+        .catch(err => {
+            let maxNormalAttempts = 20;
+            let delay;
+            if (attempt < maxNormalAttempts) {
+                delay = Math.min(5000 * Math.pow(2, Math.min(attempt, 10)), 60000);
+                logger.error({ msg: 'Failed to start processing Google pub/sub', err, attempt: attempt + 1, retryMs: delay });
+            } else {
+                delay = 5 * 60 * 1000;
+                logger.warn({ msg: 'Failed to start processing Google pub/sub (reduced frequency)', err, attempt: attempt + 1, retryMs: delay });
+            }
+            startRetryTimer = setTimeout(() => startGooglePubSub(attempt + 1), delay);
+        });
+})(0);
 
 logger.info({ msg: 'Started Webhooks worker thread', version: packageData.version });