emailengine-app 2.63.3 → 2.64.0

package/lib/oauth2-apps.js

@@ -3,7 +3,7 @@
 const { redis } = require('./db');
 const msgpack = require('msgpack5')();
 const logger = require('./logger');
-const { REDIS_PREFIX } = require('./consts');
+const { REDIS_PREFIX, TRANSIENT_NETWORK_CODES, GMAIL_PUBSUB_DEFAULT_EXPIRATION_TTL, GMAIL_PUBSUB_ACK_DEADLINE_SECONDS } = require('./consts');
 const { encrypt, decrypt } = require('./encrypt');
 const Boom = require('@hapi/boom');
 const settings = require('./settings');
@@ -11,17 +11,6 @@ const Lock = require('ioredfour');
 const getSecret = require('./get-secret');
 const { parentPort } = require('worker_threads');
 
-const TRANSIENT_NETWORK_CODES = new Set([
-    'ENOTFOUND',
-    'EAI_AGAIN',
-    'ETIMEDOUT',
-    'ECONNRESET',
-    'ECONNREFUSED',
-    'UND_ERR_SOCKET',
-    'UND_ERR_CONNECT_TIMEOUT',
-    'UND_ERR_HEADERS_TIMEOUT'
-]);
-
 /**
  * Record metrics for OAuth2 token operations
  * Works in both main thread and worker threads
@@ -54,6 +43,9 @@ const { MailRuOauth, MAIL_RU_SCOPES } = require('./oauth/mail-ru');
 const LEGACY_KEYS = ['gmail', 'gmailService', 'outlook', 'mailRu'];
 const LEGACY_KEYS_REV = JSON.parse(JSON.stringify(LEGACY_KEYS)).reverse();
 
+const PUBSUB_PERM_MANAGE = 'Service client does not have permission to manage Pub/Sub topics. Grant the service user the "Pub/Sub Admin" role.';
+const PUBSUB_PERM_VIEW = 'Service client does not have permission to view Pub/Sub topics. Grant the service user the "Pub/Sub Admin" role.';
+
 const OAUTH_PROVIDERS = {
     gmail: 'Gmail',
     gmailService: 'Gmail Service Accounts',
@@ -176,11 +168,13 @@ function formatExtraScopes(extraScopes, baseScopes, defaultScopesList, skipScope
         defaultScopes = (baseScopes && defaultScopesList[baseScopes]) || defaultScopesList.imap;
     }
 
-    let extras = [];
     if (!extraScopes && !skipScopes.length) {
         return defaultScopes;
     }
 
+    extraScopes = extraScopes || [];
+
+    let extras = [];
     for (let extraScope of extraScopes) {
         if (defaultScopes.includes(extraScope) || (scopePrefix && defaultScopes.includes(`${scopePrefix}/${extraScope}`))) {
             // skip existing
@@ -192,19 +186,16 @@ function formatExtraScopes(extraScopes, baseScopes, defaultScopesList, skipScope
     let result = extras.length ? extras.concat(defaultScopes) : defaultScopes;
 
     if (skipScopes.length) {
-        result = result.filter(scope => {
-            for (let skipScope of skipScopes) {
-                if (
-                    scope === skipScope ||
-                    scope === `https://outlook.office.com/${skipScope}` ||
-                    scope === `https://graph.microsoft.com/${skipScope}` ||
-                    scope === `https://www.googleapis.com/auth/${skipScope}`
-                ) {
-                    return false;
-                }
-            }
-            return true;
-        });
+        result = result.filter(
+            scope =>
+                !skipScopes.some(
+                    skipScope =>
+                        scope === skipScope ||
+                        scope === `https://outlook.office.com/${skipScope}` ||
+                        scope === `https://graph.microsoft.com/${skipScope}` ||
+                        scope === `https://www.googleapis.com/auth/${skipScope}`
+                )
+        );
     }
 
     return result;
@@ -216,6 +207,9 @@ class OAuth2AppsHandler {
         this.redis = this.options.redis;
 
         this.secret = null;
+
+        this._pubSubBackfillDone = false;
+        this._pubSubBackfillPromise = null;
     }
 
     async encrypt(value) {
@@ -252,6 +246,56 @@ class OAuth2AppsHandler {
         return `${REDIS_PREFIX}settings`;
     }
 
+    async backfillPubSubApps() {
+        if (this._pubSubBackfillDone) {
+            return await this.redis.smembers(this.getSubscribersKey());
+        }
+
+        if (this._pubSubBackfillPromise) {
+            return await this._pubSubBackfillPromise;
+        }
+
+        this._pubSubBackfillPromise = this._doBackfillPubSubApps();
+        try {
+            return await this._pubSubBackfillPromise;
+        } finally {
+            this._pubSubBackfillPromise = null;
+        }
+    }
+
+    async _doBackfillPubSubApps() {
+        let [subscriberIds, allIds] = await Promise.all([this.redis.smembers(this.getSubscribersKey()), this.redis.smembers(this.getIndexKey())]);
+        let subscriberSet = new Set(subscriberIds);
+        let missingIds = allIds.filter(id => !subscriberSet.has(id));
+        if (!missingIds.length) {
+            this._pubSubBackfillDone = true;
+            return subscriberIds;
+        }
+        let bufKeys = missingIds.map(id => `${id}:data`);
+        let entries = await this.redis.hmgetBuffer(this.getDataKey(), bufKeys);
+        let toAdd = [];
+        for (let i = 0; i < entries.length; i++) {
+            if (!entries[i]) {
+                continue;
+            }
+            try {
+                let data = msgpack.decode(entries[i]);
+                if (data.baseScopes === 'pubsub') {
+                    toAdd.push(missingIds[i]);
+                    logger.info({ msg: 'Backfilled pubsub app to subscribers set', app: missingIds[i] });
+                }
+            } catch (err) {
+                logger.warn({ msg: 'Failed to decode app data during backfill', app: missingIds[i], err });
+            }
+        }
+        if (toAdd.length) {
+            await this.redis.sadd(this.getSubscribersKey(), ...toAdd);
+            subscriberIds.push(...toAdd);
+        }
+        this._pubSubBackfillDone = true;
+        return subscriberIds;
+    }
+
     async list(page, pageSize, opts) {
         opts = opts || {};
         page = Math.max(Number(page) || 0, 0);
@@ -259,14 +303,12 @@ class OAuth2AppsHandler {
 
         let startPos = page * pageSize;
 
-        let keyFunc;
+        let idList;
         if (opts.pubsub) {
-            keyFunc = 'getSubscribersKey';
+            idList = await this.backfillPubSubApps();
         } else {
-            keyFunc = 'getIndexKey';
+            idList = await this.redis.smembers(this.getIndexKey());
         }
-
-        let idList = await this.redis.smembers(this[keyFunc]());
         idList = [].concat(idList || []).sort((a, b) => -a.localeCompare(b));
 
         if (!opts.pubsub) {
@@ -303,7 +345,7 @@ class OAuth2AppsHandler {
                 let data = await this.get(legacyKey);
                 response.apps.push(data);
             } catch (err) {
-                logger.error({ msg: 'Failed to process legacy app', legacyKey });
+                logger.error({ msg: 'Failed to process legacy app', legacyKey, err });
                 continue;
             }
         }
@@ -311,18 +353,21 @@ class OAuth2AppsHandler {
         if (keys.length) {
             let bufKeys = keys.flatMap(id => [`${id}:data`, `${id}:meta`]);
             let list = await this.redis.hmgetBuffer(this.getDataKey(), bufKeys);
-            for (let i = 0; i < list.length; i++) {
-                let entry = list[i];
+            for (let i = 0; i < list.length; i += 2) {
+                let dataEntry = list[i];
+                let metaEntry = list[i + 1];
                 try {
-                    if (i % 2 === 0) {
-                        let data = msgpack.decode(entry);
-                        response.apps.push(data);
-                    } else if (entry) {
-                        response.apps[response.apps.length - 1].meta = msgpack.decode(entry);
+                    let data = msgpack.decode(dataEntry);
+                    if (metaEntry) {
+                        try {
+                            data.meta = msgpack.decode(metaEntry);
+                        } catch (metaErr) {
+                            logger.error({ msg: 'Failed to process app meta', entryLength: metaEntry?.length || 0 });
+                        }
                     }
+                    response.apps.push(data);
                 } catch (err) {
-                    logger.error({ msg: 'Failed to process app', entry: entry.toString('base64') });
-                    continue;
+                    logger.error({ msg: 'Failed to process app', entryLength: dataEntry?.length || 0 });
                 }
             }
         }
@@ -370,13 +415,11 @@ class OAuth2AppsHandler {
     async generateId() {
         let idNum = await this.redis.hincrby(this.getSettingsKey(), 'idcount', 1);
 
-        let idBuf = Buffer.alloc(8 + 4);
+        let idBuf = Buffer.alloc(12);
         idBuf.writeBigUInt64BE(BigInt(Date.now()), 0);
         idBuf.writeUInt32BE(idNum, 8);
 
-        const id = idBuf.toString('base64url');
-
-        return id;
+        return idBuf.toString('base64url');
     }
 
     async getLegacyApp(id) {
@@ -589,15 +632,15 @@ class OAuth2AppsHandler {
         try {
             data = msgpack.decode(getDataBuf);
         } catch (err) {
-            logger.error({ msg: 'Failed to process app', app: id, entry: getDataBuf.toString('base64') });
+            logger.error({ msg: 'Failed to process app', app: id, entryLength: getDataBuf?.length || 0 });
             throw err;
         }
 
         if (getMetaBuf) {
             try {
-                data = Object.assign(data, { meta: msgpack.decode(getMetaBuf) });
+                data.meta = msgpack.decode(getMetaBuf);
             } catch (err) {
-                logger.error({ msg: 'Failed to process app', app: id, entry: getMetaBuf.toString('base64') });
+                logger.error({ msg: 'Failed to process app meta', app: id, entryLength: getMetaBuf?.length || 0 });
             }
         }
 
@@ -635,7 +678,7 @@ class OAuth2AppsHandler {
                 [`${id}:data`]: msgpack.encode(entry)
             });
 
-        if (data.pubSubSubscription) {
+        if (data.pubSubSubscription || entry.baseScopes === 'pubsub') {
             insertResultReq = insertResultReq.sadd(this.getSubscribersKey(), id);
         }
 
@@ -645,9 +688,9 @@ class OAuth2AppsHandler {
 
         let insertResult = await insertResultReq.exec();
 
-        let hasError = (insertResult[0] && insertResult[0][0]) || (insertResult[1] && insertResult[1][0]);
-        if (hasError) {
-            throw hasError;
+        let errorEntry = insertResult.find(entry => entry && entry[0]);
+        if (errorEntry) {
+            throw errorEntry[0];
         }
 
         const result = {
@@ -655,17 +698,7 @@ class OAuth2AppsHandler {
             created: true
         };
 
-        try {
-            let appData = await this.get(id);
-            if (appData.baseScopes === 'pubsub') {
-                let pubsubUpdates = await this.ensurePubsub(appData);
-                if (Object.keys(pubsubUpdates || {})) {
-                    result.pubsubUpdates = pubsubUpdates;
-                }
-            }
-        } catch (err) {
-            logger.error({ msg: 'Failed to set up pubsub', app: id, err });
-        }
+        await this.tryEnsurePubsub(id, result);
 
         return result;
     }
@@ -687,6 +720,8 @@ class OAuth2AppsHandler {
 
         let existingData = msgpack.decode(existingDataBuf);
 
+        let oldPubSubApp = existingData.pubSubApp || null;
+
         let encryptedValues = {};
         for (let key of ['clientSecret', 'serviceKey', 'accessToken']) {
             if (data[key]) {
@@ -706,22 +741,25 @@ class OAuth2AppsHandler {
 
         let insertResultReq = this.redis.multi().sadd(this.getIndexKey(), id).hmset(this.getDataKey(), updates);
 
-        if (data.pubSubSubscription) {
+        if (data.pubSubSubscription || entry.baseScopes === 'pubsub') {
             insertResultReq = insertResultReq.sadd(this.getSubscribersKey(), id);
         }
 
         if (data.pubSubApp) {
-            if (existingData.pubSubApp && existingData.pubSubApp !== data.pubSubApp) {
-                insertResultReq = insertResultReq.srem(this.getPubsubAppKey(existingData.pubSubApp), id);
+            if (oldPubSubApp && oldPubSubApp !== data.pubSubApp) {
+                insertResultReq = insertResultReq.srem(this.getPubsubAppKey(oldPubSubApp), id);
             }
             insertResultReq = insertResultReq.sadd(this.getPubsubAppKey(data.pubSubApp), id);
+        } else if (oldPubSubApp && 'pubSubApp' in data) {
+            // pubSubApp was explicitly cleared
+            insertResultReq = insertResultReq.srem(this.getPubsubAppKey(oldPubSubApp), id);
         }
 
         let insertResult = await insertResultReq.exec();
 
-        let hasError = (insertResult[0] && insertResult[0][0]) || (insertResult[1] && insertResult[1][0]);
-        if (hasError) {
-            throw hasError;
+        let errorEntry = insertResult.find(entry => entry && entry[0]);
+        if (errorEntry) {
+            throw errorEntry[0];
         }
 
         if (opts.partial) {
@@ -739,17 +777,7 @@ class OAuth2AppsHandler {
             updated: true
         };
 
-        try {
-            let appData = await this.get(id);
-            if (appData.baseScopes === 'pubsub') {
-                let pubsubUpdates = await this.ensurePubsub(appData);
-                if (Object.keys(pubsubUpdates || {})) {
-                    result.pubsubUpdates = pubsubUpdates;
-                }
-            }
-        } catch (err) {
-            logger.error({ msg: 'Failed to set up pubsub', app: id, err });
-        }
+        await this.tryEnsurePubsub(id, result);
 
         return result;
     }
@@ -762,148 +790,224 @@ class OAuth2AppsHandler {
 
         let appData = await this.get(id);
 
-        if (appData.pubSubTopic) {
-            // try to delete topic
+        if (!appData) {
+            return { id, deleted: false, accounts: 0 };
+        }
+
+        // Acquire the same lock used by ensurePubsub to prevent racing with recovery
+        let needsPubSubLock = appData.pubSubTopic || appData.baseScopes === 'pubsub';
+        let delLock;
+
+        if (needsPubSubLock) {
+            let lockKey = ['oauth', 'pubsub', appData.id].join(':');
             try {
-                await this.deleteTopic(appData);
+                delLock = await lock.waitAcquireLock(lockKey, 5 * 60 * 1000, 2 * 60 * 1000);
+                if (!delLock.success) {
+                    throw new Error('Failed to get lock for pubsub app deletion');
+                }
             } catch (err) {
-                logger.error({ msg: 'Failed to delete existing pubsub topic', app: appData.id, topic: appData.pubSubTopic, err });
+                logger.error({ msg: 'Failed to acquire lock for pubsub app deletion', lockKey, err });
+                throw err;
             }
         }
 
-        let pipeline = this.redis
-            .multi()
-            .srem(this.getIndexKey(), id)
-            .hdel(this.getDataKey(), [`${id}:data`, `${id}:meta`])
-            .scard(`${REDIS_PREFIX}oapp:a:${id}`)
-            .del(`${REDIS_PREFIX}oapp:a:${id}`)
-            .del(`${REDIS_PREFIX}oapp:h:${id}`)
-            .srem(this.getSubscribersKey(), id);
-
-        if (appData.pubsubApp) {
-            pipeline = pipeline.srem(this.getPubsubAppKey(appData.pubSubApp), id);
-        }
+        try {
+            if (appData.pubSubTopic) {
+                // try to delete topic
+                try {
+                    await this.deleteTopic(appData);
+                } catch (err) {
+                    logger.error({
+                        msg: 'Failed to delete existing pubsub topic',
+                        app: appData.id,
+                        topic: appData.pubSubTopic,
+                        subscription: appData.pubSubSubscription,
+                        err
+                    });
+                }
+            }
 
-        let deleteResult = await pipeline.exec();
+            let pipeline = this.redis
+                .multi()
+                .srem(this.getIndexKey(), id)
+                .hdel(this.getDataKey(), [`${id}:data`, `${id}:meta`])
+                .scard(`${REDIS_PREFIX}oapp:a:${id}`)
+                .del(`${REDIS_PREFIX}oapp:a:${id}`)
+                .del(`${REDIS_PREFIX}oapp:h:${id}`)
+                .srem(this.getSubscribersKey(), id);
+
+            if (appData.pubSubApp) {
+                pipeline = pipeline.srem(this.getPubsubAppKey(appData.pubSubApp), id);
+            }
 
-        let hasError = (deleteResult[0] && deleteResult[0][0]) || (deleteResult[1] && deleteResult[1][0]);
-        if (hasError) {
-            throw hasError;
-        }
+            if (appData.baseScopes === 'pubsub') {
+                pipeline = pipeline.del(this.getPubsubAppKey(id));
+            }
 
-        let deletedDocs = ((deleteResult[0] && deleteResult[0][1]) || 0) + ((deleteResult[1] && deleteResult[1][1]) || 0);
+            let deleteResult = await pipeline.exec();
 
-        return {
-            id,
-            deleted: deletedDocs >= 2,
-            accounts: Number(deleteResult[2] && deleteResult[2][1]) || 0
-        };
+            let errorEntry = deleteResult.find(entry => entry && entry[0]);
+            if (errorEntry) {
+                throw errorEntry[0];
+            }
+
+            let deletedDocs = ((deleteResult[0] && deleteResult[0][1]) || 0) + ((deleteResult[1] && deleteResult[1][1]) || 0);
+
+            return {
+                id,
+                deleted: deletedDocs >= 2,
+                accounts: Number(deleteResult[2] && deleteResult[2][1]) || 0
+            };
+        } finally {
+            if (delLock?.success) {
+                await lock.releaseLock(delLock);
+            }
+        }
     }
 
+    // Note: not atomic (read-modify-write). Concurrent callers use last-write-wins,
+    // which is acceptable for self-correcting informational metadata.
     async setMeta(id, meta) {
-        let existingMeta;
         let existingMetaBuf = await this.redis.hgetBuffer(this.getDataKey(), `${id}:meta`);
-        if (!existingMetaBuf) {
-            existingMeta = {};
-        } else {
-            existingMeta = msgpack.decode(existingMetaBuf);
-        }
+        let existingMeta = existingMetaBuf ? msgpack.decode(existingMetaBuf) : {};
 
         let entry = Object.assign(existingMeta, meta || {});
 
-        let updates = {
+        await this.redis.hmset(this.getDataKey(), {
             [`${id}:meta`]: msgpack.encode(entry)
-        };
-
-        await this.redis.hmset(this.getDataKey(), updates);
+        });
 
-        return {
-            id,
-            updated: true
-        };
+        return { id, updated: true };
     }
 
-    async deleteTopic(appData) {
-        let topicName = appData.pubSubTopic;
-        let topicUrl = `https://pubsub.googleapis.com/v1/${topicName}`;
-
-        let subscriptionName = appData.pubSubSubscription;
-        let subscriptionUrl = `https://pubsub.googleapis.com/v1/${subscriptionName}`;
-
-        let client = await this.getClient(appData.id);
-
-        // Step 1. Get access token for service client
-        let accessToken = await this.getServiceAccessToken(appData, client);
-        if (!accessToken) {
-            throw new Error('Failed to get access token');
-        }
+    /**
+     * Fire-and-forget setMeta call that logs errors instead of throwing.
+     * Used in ensurePubsub error handlers where we want to record a flag
+     * but must not block or alter the original error flow.
+     */
+    setMetaFireAndForget(appId, meta) {
+        this.setMeta(appId, meta).catch(metaErr => {
+            logger.error({ msg: 'Failed to set metadata', app: appId, err: metaErr });
+        });
+    }
 
+    /**
+     * Try to set up Pub/Sub for the given app and attach results to the result object.
+     * Errors are logged but not thrown so they do not prevent create/update from succeeding.
+     */
+    async tryEnsurePubsub(id, result) {
         try {
-            if (topicName) {
-                // fails if topic does not exist
-                await client.request(accessToken, topicUrl, 'DELETE', Buffer.alloc(0), { returnText: true });
-                /*
-                {}
-            */
+            let appData = await this.get(id);
+            if (appData.baseScopes === 'pubsub') {
+                let pubsubUpdates = await this.ensurePubsub(appData);
+                result.pubsubUpdates = pubsubUpdates || {};
             }
         } catch (err) {
-            switch (err?.oauthRequest?.response?.error?.code) {
-                case 403:
-                    // no permissions
-                    logger.error({
-                        msg: 'Service client does not have permissions to delete Pub/Sub topics. Make sure the role for the service user is "Pub/Sub Admin".',
-                        app: appData.id,
-                        topic: topicName
-                    });
-                    throw err;
-                case 404: {
-                    // does not exist
-                    logger.info({ msg: 'Topic does not exist', app: appData.id, topic: topicName });
-                    break;
-                }
-                default:
-                    throw err;
-            }
+            logger.error({ msg: 'Failed to set up pubsub', app: id, err });
         }
+    }
 
+    async _retryDeleteOnce(doDelete, appData, resourceType, resourceName, delay, reason) {
+        logger.warn({
+            msg: `${reason} deleting Pub/Sub ${resourceType}, retrying in ${delay}ms`,
+            app: appData.id,
+            [resourceType]: resourceName
+        });
+        await new Promise(resolve => setTimeout(resolve, delay));
         try {
-            if (subscriptionName) {
-                // fails if subscription does not exist
-                await client.request(accessToken, subscriptionUrl, 'DELETE', Buffer.alloc(0), { returnText: true });
-                /*
-                {}
-            */
+            await doDelete();
+        } catch (retryErr) {
+            if (retryErr?.oauthRequest?.response?.error?.code === 404) {
+                logger.info({
+                    msg: `${resourceType} no longer exists after retry`,
+                    app: appData.id,
+                    [resourceType]: resourceName
+                });
+                return;
             }
+            logger.error({
+                msg: `Retry failed deleting Pub/Sub ${resourceType}, resource may need manual cleanup`,
+                app: appData.id,
+                [resourceType]: resourceName,
+                code: retryErr.code || retryErr?.oauthRequest?.response?.error?.code
+            });
+            throw retryErr;
+        }
+    }
+
+    async _deletePubSubResource(client, accessToken, appData, resourceType, resourceName) {
+        if (!resourceName) {
+            return;
+        }
+        let url = `https://pubsub.googleapis.com/v1/${resourceName}`;
+        const doDelete = () => client.request(accessToken, url, 'DELETE', Buffer.alloc(0), { returnText: true });
+        try {
+            await doDelete();
         } catch (err) {
             switch (err?.oauthRequest?.response?.error?.code) {
                 case 403:
-                    // no permissions
                     logger.error({
-                        msg: 'Service client does not have permissions to delete Pub/Sub subscriptions. Make sure the role for the service user is "Pub/Sub Admin".',
+                        msg: `Service client does not have permissions to delete Pub/Sub ${resourceType}s. Make sure the role for the service user is "Pub/Sub Admin".`,
                         app: appData.id,
-                        topic: topicName
+                        [resourceType]: resourceName
                     });
                     throw err;
-                case 404: {
-                    // does not exist
-                    logger.info({ msg: 'Subscription does not exist', app: appData.id, topic: topicName });
+                case 404:
+                    logger.info({ msg: `${resourceType} does not exist`, app: appData.id, [resourceType]: resourceName });
+                    break;
+                case 429: {
+                    let retryAfter = err.retryAfter || err.oauthRequest?.retryAfter;
+                    let delay = retryAfter ? Math.min(retryAfter * 1000, 30000) : 2000;
+                    await this._retryDeleteOnce(doDelete, appData, resourceType, resourceName, delay, 'Rate limited');
                     break;
                 }
                 default:
+                    if (TRANSIENT_NETWORK_CODES.has(err.code)) {
+                        await this._retryDeleteOnce(doDelete, appData, resourceType, resourceName, 2000, 'Transient network error');
+                        break;
+                    }
                     throw err;
             }
         }
     }
 
+    async deleteTopic(appData) {
+        let client = await this.getClient(appData.id);
+
+        let accessToken = await this.getServiceAccessToken(appData, client);
+        if (!accessToken) {
+            throw new Error('Failed to get access token');
+        }
+
+        await this._deletePubSubResource(client, accessToken, appData, 'subscription', appData.pubSubSubscription);
+        await this._deletePubSubResource(client, accessToken, appData, 'topic', appData.pubSubTopic);
+    }
+
     async ensurePubsub(appData) {
         let project = appData.googleProjectId;
 
         let topic = appData.googleTopicName || `ee-pub-${appData.id}`;
         let subscription = appData.googleSubscriptionName || `ee-sub-${appData.id}`;
 
+        // Determine desired expirationPolicy from setting
+        let gmailSubscriptionTtl = await settings.get('gmailSubscriptionTtl');
+        let desiredExpirationPolicy;
+        if (gmailSubscriptionTtl === 0) {
+            // Indefinite (no expiration)
+            desiredExpirationPolicy = {};
+        } else if (typeof gmailSubscriptionTtl === 'number' && gmailSubscriptionTtl > 0) {
+            // Convert days to seconds for Google API
+            desiredExpirationPolicy = { ttl: `${gmailSubscriptionTtl * 86400}s` };
+        } else {
+            // Not set - let Google apply its default 31-day TTL
+            desiredExpirationPolicy = null;
+        }
+
         let results = {};
+        let ttlPatchFailed = null;
 
-        if (!project || !topic) {
+        if (!project) {
+            logger.warn({ msg: 'googleProjectId is required for Pub/Sub setup', app: appData.id });
             return results;
         }
 
@@ -916,130 +1020,181 @@ class OAuth2AppsHandler {
         const member = 'serviceAccount:gmail-api-push@system.gserviceaccount.com';
         const role = 'roles/pubsub.publisher';
 
-        let client = await this.getClient(appData.id);
-
-        // Step 1. Get access token for service client
-        let accessToken = await this.getServiceAccessToken(appData, client);
-        if (!accessToken) {
-            throw new Error('Failed to get access token');
+        // Acquire per-app lock to prevent concurrent ensurePubsub races (e.g., user update + background recovery)
+        let lockKey = ['oauth', 'pubsub', appData.id].join(':');
+        let ensureLock;
+        try {
+            ensureLock = await lock.waitAcquireLock(lockKey, 5 * 60 * 1000, 2 * 60 * 1000);
+            if (!ensureLock.success) {
+                logger.error({ msg: 'Failed to get ensurePubsub lock', lockKey });
+                throw new Error('Failed to get ensurePubsub lock');
+            }
+        } catch (err) {
+            logger.error({ msg: 'Failed to get ensurePubsub lock', lockKey, err });
+            throw err;
         }
 
-        // Step 2. Ensure topic
         try {
-            // fails if topic does not exist
-            await client.request(accessToken, topicUrl, 'GET');
-            logger.debug({ msg: 'Topic already exists', app: appData.id, topic: topicName });
-            /*
-                {name: 'projects/...'}
-            */
-        } catch (err) {
-            if (TRANSIENT_NETWORK_CODES.has(err.code)) {
-                logger.warn({ msg: 'Network error checking Pub/Sub topic', app: appData.id, code: err.code });
-                throw err;
+            let client = await this.getClient(appData.id);
+
+            // Step 1. Get access token for service client
+            let accessToken = await this.getServiceAccessToken(appData, client);
+            if (!accessToken) {
+                throw new Error('Failed to get access token');
             }
-            switch (err?.oauthRequest?.response?.error?.code) {
-                case 403:
-                    // no permissions
-                    if (/Cloud Pub\/Sub API has not been used in project/.test(err?.oauthRequest?.response?.error?.message)) {
-                        this.setMeta(appData.id, {
-                            authFlag: {
-                                message:
-                                    'Enable the Cloud Pub/Sub API for your project before using the service client. Check the server response below for details.',
-                                description: err?.oauthRequest?.response?.error?.message
-                            }
-                        });
-                    } else {
-                        this.setMeta(appData.id, {
-                            authFlag: {
-                                message: 'Service client does not have permission to manage Pub/Sub topics. Grant the service user the "Pub/Sub Admin" role.'
-                            }
-                        });
-                    }
 
+            // Step 2. Ensure topic
+            try {
+                // fails if topic does not exist
+                await client.request(accessToken, topicUrl, 'GET');
+                logger.debug({ msg: 'Topic already exists', app: appData.id, topic: topicName });
+                /*
+                {name: 'projects/...'}
+            */
+            } catch (err) {
+                if (TRANSIENT_NETWORK_CODES.has(err.code)) {
+                    logger.warn({ msg: 'Network error checking Pub/Sub topic', app: appData.id, code: err.code });
                     throw err;
-                case 404: {
-                    // does not exist
-                    logger.info({ msg: 'Topic does not exist', app: appData.id, topic: topicName });
-                    try {
-                        let topicCreateRes = await client.request(accessToken, topicUrl, 'PUT', Buffer.alloc(0));
-                        /*
+                }
+                switch (err?.oauthRequest?.response?.error?.code) {
+                    case 403:
+                        // no permissions
+                        if (/Cloud Pub\/Sub API has not been used in project/.test(err?.oauthRequest?.response?.error?.message)) {
+                            this.setMetaFireAndForget(appData.id, {
+                                authFlag: {
+                                    message:
+                                        'Enable the Cloud Pub/Sub API for your project before using the service client. Check the server response below for details.',
+                                    description: err?.oauthRequest?.response?.error?.message
+                                },
+                                ttlWarning: null
+                            });
+                        } else {
+                            this.setMetaFireAndForget(appData.id, {
+                                authFlag: { message: PUBSUB_PERM_MANAGE },
+                                ttlWarning: null
+                            });
+                        }
+
+                        throw err;
+                    case 404: {
+                        // does not exist
+                        logger.info({ msg: 'Topic does not exist', app: appData.id, topic: topicName });
+                        try {
+                            let topicCreateRes = await client.request(accessToken, topicUrl, 'PUT', Buffer.alloc(0));
+                            /*
                             {name: 'projects/...'}
                         */
-                        if (!topicCreateRes?.name) {
-                            throw new Error('Topic was not created');
-                        }
+                            if (!topicCreateRes?.name) {
+                                throw new Error('Topic was not created', { cause: err });
+                            }
 
-                        await this.update(
-                            appData.id,
-                            {
-                                pubSubTopic: topicName
-                            },
-                            { partial: true }
-                        );
-
-                        results.pubSubTopic = topicName;
-                    } catch (err) {
-                        if (TRANSIENT_NETWORK_CODES.has(err.code)) {
-                            logger.warn({ msg: 'Network error creating Pub/Sub topic', app: appData.id, code: err.code });
-                            throw err;
-                        }
-                        switch (err?.oauthRequest?.response?.error?.code) {
-                            case 403:
-                                // no permissions
-                                this.setMeta(appData.id, {
-                                    authFlag: {
-                                        message:
-                                            'Service client does not have permission to manage Pub/Sub topics. Grant the service user the "Pub/Sub Admin" role.'
-                                    }
-                                });
-                                throw err;
-                            case 409:
-                                // already exists
-                                logger.info({ msg: 'Topic already exists', app: appData.id, topic: topicName });
-                                break;
-                            default:
+                            await this.update(
+                                appData.id,
+                                {
+                                    pubSubTopic: topicName
+                                },
+                                { partial: true }
+                            );
+
+                            results.pubSubTopic = topicName;
+                        } catch (err) {
+                            if (TRANSIENT_NETWORK_CODES.has(err.code)) {
+                                logger.warn({ msg: 'Network error creating Pub/Sub topic', app: appData.id, code: err.code });
                                 throw err;
+                            }
+                            switch (err?.oauthRequest?.response?.error?.code) {
+                                case 403:
+                                    // no permissions
+                                    this.setMetaFireAndForget(appData.id, {
+                                        authFlag: { message: PUBSUB_PERM_MANAGE },
+                                        ttlWarning: null
+                                    });
+                                    throw err;
+                                case 409:
+                                    // already exists
+                                    logger.info({ msg: 'Topic already exists', app: appData.id, topic: topicName });
+                                    break;
+                                default:
+                                    throw err;
+                            }
                         }
+                        break;
                     }
-                    break;
+                    default:
+                        throw err;
                 }
-                default:
-                    throw err;
             }
-        }
 
-        // Step 3. Set up subscriber
+            // Step 3. Set up subscriber
 
-        try {
-            // fails if topic does not exist
-            await client.request(accessToken, subscriptionUrl, 'GET');
-            logger.debug({ msg: 'Subscription already exists', app: appData.id, subscription: subscriptionName });
-            /*
+            try {
+                // fails if topic does not exist
+                let subscriptionData = await client.request(accessToken, subscriptionUrl, 'GET');
+                logger.debug({ msg: 'Subscription already exists', app: appData.id, subscription: subscriptionName });
+                /*
                 {name: 'projects/...'}
             */
-        } catch (err) {
-            if (TRANSIENT_NETWORK_CODES.has(err.code)) {
-                logger.warn({ msg: 'Network error checking Pub/Sub subscription', app: appData.id, code: err.code });
-                throw err;
-            }
-            switch (err?.oauthRequest?.response?.error?.code) {
-                case 403:
-                    // no permissions
-                    this.setMeta(appData.id, {
-                        authFlag: {
-                            message: 'Service client does not have permission to view Pub/Sub topics. Grant the service user the "Pub/Sub Admin" role.'
-                        }
-                    });
-                    throw err;
-                case 404: {
-                    // does not exist
-                    logger.info({ msg: 'Subscription does not exist', app: appData.id, subscription: subscriptionName });
+
+                // Patch existing subscription's expiration policy if it differs from desired
+                // When no explicit TTL is configured, use Google's default (31 days) for comparison
+                let effectivePolicy = desiredExpirationPolicy ?? { ttl: GMAIL_PUBSUB_DEFAULT_EXPIRATION_TTL };
+                let currentTtl = subscriptionData?.expirationPolicy?.ttl || null;
+                let desiredTtl = effectivePolicy.ttl || null;
+                if (currentTtl !== desiredTtl) {
                     try {
-                        let subscriptionCreateRes = await client.request(accessToken, subscriptionUrl, 'PUT', {
-                            topic: topicName,
-                            ackDeadlineSeconds: 30
+                        await client.request(accessToken, subscriptionUrl, 'PATCH', {
+                            subscription: { expirationPolicy: effectivePolicy },
+                            updateMask: 'expirationPolicy'
                         });
-                        /*
+                        logger.info({
+                            msg: 'Updated expiration policy on subscription',
+                            app: appData.id,
+                            subscription: subscriptionName,
+                            desiredTtl
+                        });
+                        // Update stored TTL after successful patch
+                        subscriptionData.expirationPolicy = effectivePolicy;
+                    } catch (patchErr) {
+                        ttlPatchFailed = patchErr;
+                        logger.warn({
+                            msg: 'Failed to update expiration policy on subscription',
+                            app: appData.id,
+                            subscription: subscriptionName,
+                            err: patchErr
+                        });
+                    }
+                }
+
+                // Store current expiration info in meta
+                await this.setMeta(appData.id, {
+                    subscriptionExpiration: subscriptionData?.expirationPolicy?.ttl || null
+                });
+            } catch (err) {
+                if (TRANSIENT_NETWORK_CODES.has(err.code)) {
+                    logger.warn({ msg: 'Network error checking Pub/Sub subscription', app: appData.id, code: err.code });
+                    throw err;
+                }
+                switch (err?.oauthRequest?.response?.error?.code) {
+                    case 403:
+                        // no permissions
+                        this.setMetaFireAndForget(appData.id, {
+                            authFlag: { message: PUBSUB_PERM_VIEW },
+                            ttlWarning: null
+                        });
+                        throw err;
+                    case 404: {
+                        // does not exist
+                        logger.info({ msg: 'Subscription does not exist', app: appData.id, subscription: subscriptionName });
+                        try {
+                            let createPayload = {
+                                topic: topicName,
+                                ackDeadlineSeconds: GMAIL_PUBSUB_ACK_DEADLINE_SECONDS
+                            };
+                            if (desiredExpirationPolicy !== null) {
+                                createPayload.expirationPolicy = desiredExpirationPolicy;
+                            }
+                            let subscriptionCreateRes = await client.request(accessToken, subscriptionUrl, 'PUT', createPayload);
+                            /*
                             {
                             name: 'projects/webhooks-425411/subscriptions/ee-sub-AAABkE9_uNMAAAAH',
                             topic: 'projects/webhooks-425411/topics/ee-pub-AAABkE9_uNMAAAAH',
@@ -1051,57 +1206,61 @@ class OAuth2AppsHandler {
                             }
                         */
 
-                        if (!subscriptionCreateRes?.name) {
-                            throw new Error('Topic was not created');
-                        }
+                            if (!subscriptionCreateRes?.name) {
+                                throw new Error('Subscription was not created', { cause: err });
+                            }
 
-                        await this.update(
-                            appData.id,
-                            {
-                                pubSubSubscription: subscriptionName
-                            },
-                            { partial: true }
-                        );
-
-                        results.pubSubSubscription = subscriptionName;
-                    } catch (err) {
-                        if (TRANSIENT_NETWORK_CODES.has(err.code)) {
-                            logger.warn({ msg: 'Network error creating Pub/Sub subscription', app: appData.id, code: err.code });
-                            throw err;
-                        }
-                        switch (err?.oauthRequest?.response?.error?.code) {
-                            case 403:
-                                // no permissions
-                                this.setMeta(appData.id, {
-                                    authFlag: {
-                                        message:
-                                            'Service client does not have permission to manage Pub/Sub topics. Grant the service user the "Pub/Sub Admin" role.'
-                                    }
-                                });
-                                throw err;
-                            case 409:
-                                // already exists
-                                logger.info({ msg: 'Subscription already exists', app: appData.id, subscription: subscriptionName });
-                                break;
-                            default:
+                            await this.update(
+                                appData.id,
+                                {
+                                    pubSubSubscription: subscriptionName
+                                },
+                                { partial: true }
+                            );
+
+                            results.pubSubSubscription = subscriptionName;
+
+                            // Store expiration info from the created subscription
+                            await this.setMeta(appData.id, {
+                                subscriptionExpiration: subscriptionCreateRes?.expirationPolicy?.ttl || null
+                            });
+                        } catch (err) {
+                            if (TRANSIENT_NETWORK_CODES.has(err.code)) {
+                                logger.warn({ msg: 'Network error creating Pub/Sub subscription', app: appData.id, code: err.code });
                                 throw err;
+                            }
+                            switch (err?.oauthRequest?.response?.error?.code) {
+                                case 403:
+                                    // no permissions
+                                    this.setMetaFireAndForget(appData.id, {
+                                        authFlag: { message: PUBSUB_PERM_MANAGE },
+                                        ttlWarning: null
+                                    });
+                                    throw err;
+                                case 409:
+                                    // already exists
+                                    logger.info({ msg: 'Subscription already exists', app: appData.id, subscription: subscriptionName });
+                                    break;
+                                default:
+                                    throw err;
+                            }
                         }
+                        break;
                     }
-                    break;
+                    default:
+                        throw err;
                 }
-                default:
-                    throw err;
             }
-        }
 
-        // Step 4. Grant access to Gmail publisher
+            // Step 4. Grant access to Gmail publisher
 
-        let existingPolicy;
-        try {
-            // Check for an existing policy grant
-            let getIamPolycyRes = await client.request(accessToken, `${topicUrl}:getIamPolicy`, 'GET');
-            existingPolicy = getIamPolycyRes?.bindings?.find(binding => binding?.role === role && binding?.members?.includes(member));
-            /*
+            let existingPolicy;
+            let getIamPolicyRes;
+            try {
+                // Check for an existing policy grant
+                getIamPolicyRes = await client.request(accessToken, `${topicUrl}:getIamPolicy`, 'GET');
+                existingPolicy = getIamPolicyRes?.bindings?.find(binding => binding?.role === role && binding?.members?.includes(member));
+                /*
                 {
                 version: 1,
                 etag: 'BwYbtWmb5c0=',
@@ -1113,77 +1272,93 @@ class OAuth2AppsHandler {
                 ]
                 }
             */
-        } catch (err) {
-            if (TRANSIENT_NETWORK_CODES.has(err.code)) {
-                logger.warn({ msg: 'Network error checking Pub/Sub IAM policy', app: appData.id, code: err.code });
-                throw err;
-            }
-            switch (err?.oauthRequest?.response?.error?.code) {
-                case 403:
-                    // no permissions
-                    this.setMeta(appData.id, {
-                        authFlag: {
-                            message: 'Service client does not have permission to view Pub/Sub topics. Grant the service user the "Pub/Sub Admin" role.'
-                        }
-                    });
-                    throw err;
-                default:
-                    throw err;
-            }
-        }
-
-        if (existingPolicy) {
-            logger.debug({ msg: 'Gmail publisher policy already exists', app: appData.id, topic: topicName });
-        } else {
-            logger.debug({ msg: 'Granting access to Gmail publisher', app: appData.id, topic: topicName });
-            const policyPayload = {
-                policy: {
-                    bindings: [
-                        {
-                            members: [member],
-                            role
-                        }
-                    ]
-                }
-            };
-            try {
-                await client.request(accessToken, `${topicUrl}:setIamPolicy`, 'POST', policyPayload);
-                results.iamPolicy = {
-                    members: [member],
-                    role
-                };
-
-                await this.update(
-                    appData.id,
-                    {
-                        pubSubIamPolicy: results.iamPolicy
-                    },
-                    { partial: true }
-                );
             } catch (err) {
                 if (TRANSIENT_NETWORK_CODES.has(err.code)) {
-                    logger.warn({ msg: 'Network error setting Pub/Sub IAM policy', app: appData.id, code: err.code });
+                    logger.warn({ msg: 'Network error checking Pub/Sub IAM policy', app: appData.id, code: err.code });
                     throw err;
                 }
                 switch (err?.oauthRequest?.response?.error?.code) {
                     case 403:
                         // no permissions
-                        this.setMeta(appData.id, {
-                            authFlag: {
-                                message: 'Service client does not have permission to manage Pub/Sub topics. Grant the service user the "Pub/Sub Admin" role.'
-                            }
+                        this.setMetaFireAndForget(appData.id, {
+                            authFlag: { message: PUBSUB_PERM_VIEW },
+                            ttlWarning: null
                         });
                         throw err;
                     default:
                         throw err;
                 }
             }
-        }
 
-        // clear auth flag if everything worked
-        await this.setMeta(appData.id, { authFlag: null });
+            if (existingPolicy) {
+                logger.debug({ msg: 'Gmail publisher policy already exists', app: appData.id, topic: topicName });
+            } else {
+                logger.debug({ msg: 'Granting access to Gmail publisher', app: appData.id, topic: topicName });
+                let existingBindings = (getIamPolicyRes && getIamPolicyRes.bindings) || [];
+                const policyPayload = {
+                    policy: {
+                        bindings: existingBindings.concat([
+                            {
+                                members: [member],
+                                role
+                            }
+                        ])
+                    }
+                };
+                if (getIamPolicyRes && getIamPolicyRes.etag) {
+                    policyPayload.policy.etag = getIamPolicyRes.etag;
+                }
+                try {
+                    await client.request(accessToken, `${topicUrl}:setIamPolicy`, 'POST', policyPayload);
+                    results.iamPolicy = {
+                        members: [member],
+                        role
+                    };
+
+                    await this.update(
+                        appData.id,
+                        {
+                            pubSubIamPolicy: results.iamPolicy
+                        },
+                        { partial: true }
+                    );
+                } catch (err) {
+                    if (TRANSIENT_NETWORK_CODES.has(err.code)) {
+                        logger.warn({ msg: 'Network error setting Pub/Sub IAM policy', app: appData.id, code: err.code });
+                        throw err;
+                    }
+                    switch (err?.oauthRequest?.response?.error?.code) {
+                        case 403:
+                            // no permissions
+                            this.setMetaFireAndForget(appData.id, {
+                                authFlag: { message: PUBSUB_PERM_MANAGE },
+                                ttlWarning: null
+                            });
+                            throw err;
+                        default:
+                            throw err;
+                    }
+                }
+            }
+
+            // clear auth flag if everything worked; set or clear TTL warning
+            let metaUpdate = { authFlag: null };
+            if (ttlPatchFailed) {
+                metaUpdate.ttlWarning = {
+                    message: 'Failed to update expiration policy on subscription',
+                    description: [ttlPatchFailed.message, ttlPatchFailed.code].filter(Boolean).join('; ')
+                };
+            } else {
+                metaUpdate.ttlWarning = null;
+            }
+            await this.setMeta(appData.id, metaUpdate);
 
-        return results;
+            return results;
+        } finally {
+            if (ensureLock?.success) {
+                await lock.releaseLock(ensureLock);
+            }
+        }
     }
 
     async getClient(id, extraOpts) {
@@ -1202,7 +1377,7 @@ class OAuth2AppsHandler {
                 let redirectUrl = appData.redirectUrl;
                 let scopes = formatExtraScopes(appData.extraScopes, appData.baseScopes, GMAIL_SCOPES, appData.skipScopes);
 
-                let googleProjectId = appData.projectIdv;
+                let googleProjectId = appData.googleProjectId;
                 let workspaceAccounts = appData.googleWorkspaceAccounts;
 
                 if (!clientId || !clientSecret || !redirectUrl) {
@@ -1245,7 +1420,7 @@ class OAuth2AppsHandler {
 
                 let scopes = formatExtraScopes(appData.extraScopes, appData.baseScopes, GMAIL_SCOPES, appData.skipScopes);
 
-                let googleProjectId = appData.projectIdv;
+                let googleProjectId = appData.googleProjectId;
                 let workspaceAccounts = appData.googleWorkspaceAccounts;
 
                 if (!serviceClient || !serviceKey) {
@@ -1445,7 +1620,9 @@ class OAuth2AppsHandler {
             });
             throw err;
         } finally {
-            await lock.releaseLock(renewLock);
+            if (renewLock?.success) {
+                await lock.releaseLock(renewLock);
+            }
         }
     }
 }