emailengine-app 2.63.4 → 2.64.0

package/lib/routes-ui.js

@@ -58,7 +58,6 @@ const { Client: ElasticSearch } = require('@elastic/elasticsearch');
 const { llmPreProcess } = require('./llm-pre-process');
 const { locales } = require('./translations');
 const capa = require('./capa');
-const exampleWebhookPayloads = require('./payload-examples-webhooks.json');
 const exampleDocumentsPayloads = require('./payload-examples-documents.json');
 const { defaultMappings } = require('./es');
 const { getESClient } = require('../lib/document-store');
@@ -719,34 +718,6 @@ async function getOpenAiError(gt) {
     return openAiError;
 }
 
-async function getExampleWebhookPayloads() {
-    let serviceUrl = await settings.get('serviceUrl');
-    let date = new Date().toISOString();
-
-    let examplePayloads = structuredClone(exampleWebhookPayloads);
-
-    examplePayloads.forEach(payload => {
-        if (payload && payload.content) {
-            if (typeof payload.content.serviceUrl === 'string') {
-                payload.content.serviceUrl = serviceUrl;
-            }
-
-            if (typeof payload.content.date === 'string') {
-                payload.content.date = date;
-            }
-
-            if (payload.content.data && typeof payload.content.data.date === 'string') {
-                payload.content.data.date = date;
-            }
-
-            if (payload.content.data && typeof payload.content.data.created === 'string') {
-                payload.content.data.created = date;
-            }
-        }
-    });
-    return examplePayloads;
-}
-
 async function getExampleDocumentsPayloads() {
     let date = new Date().toISOString();
 
@@ -886,6 +857,21 @@ function applyRoutes(server, call) {
         throw error;
     }
 
+    /**
+     * Fetch the list of Pub/Sub apps and mark the one matching selectedId as selected.
+     * Returns the apps array ready for template rendering.
+     */
+    async function getPubSubAppsForSelect(selectedId) {
+        let result = await oauth2Apps.list(0, 1000, { pubsub: true });
+        let apps = (result && result.apps) || [];
+        for (let app of apps) {
+            if (app.id === selectedId) {
+                app.selected = true;
+            }
+        }
+        return apps;
+    }
+
     // List exports for account
     server.route({
         method: 'GET',
@@ -2568,6 +2554,7 @@ return true;`
                     pageTitle: 'OAuth2',
                     menuConfig: true,
                     menuConfigOauth: true,
+                    activeApplications: true,
 
                     newLink: newLink.pathname + newLink.search,
 
@@ -2614,6 +2601,160 @@ return true;`
         }
     });
 
+    // GET /admin/config/oauth/subscriptions - Gmail Pub/Sub subscriptions list
+    server.route({
+        method: 'GET',
+        path: '/admin/config/oauth/subscriptions',
+        async handler(request, h) {
+            try {
+                let data = await oauth2Apps.list(request.query.page - 1, request.query.pageSize, { pubsub: true });
+
+                let gmailSubscriptionTtl = await settings.get('gmailSubscriptionTtl');
+
+                // Compute human-readable expiration for each app
+                // meta.subscriptionExpiration is:
+                //   undefined - no data yet (app predates this feature or ensurePubsub hasn't run)
+                //   null      - indefinite (no TTL set, ensurePubsub confirmed this)
+                //   "Ns"      - TTL in seconds (e.g. "2678400s" for 31 days)
+                let gt = request.app.gt;
+                for (let app of data.apps) {
+                    if (!app.pubSubSubscription) {
+                        app.expirationLabel = '';
+                        continue;
+                    }
+
+                    let meta = app.meta || {};
+                    if (!('subscriptionExpiration' in meta)) {
+                        app.expirationLabel = gt.gettext('Unknown');
+                        continue;
+                    }
+
+                    let seconds = parseInt(meta.subscriptionExpiration, 10);
+                    if (seconds > 0) {
+                        let days = Math.round(seconds / 86400);
+                        app.expirationLabel = util.format(gt.ngettext('%d day', '%d days', days), days);
+                    } else {
+                        app.expirationLabel = gt.gettext('Indefinite');
+                    }
+                }
+
+                let nextPage = false;
+                let prevPage = false;
+
+                let getPagingUrl = page => {
+                    let url = new URL(`admin/config/oauth/subscriptions`, 'http://localhost');
+
+                    if (page) {
+                        url.searchParams.append('page', page);
+                    }
+
+                    if (request.query.pageSize !== DEFAULT_PAGE_SIZE) {
+                        url.searchParams.append('pageSize', request.query.pageSize);
+                    }
+
+                    return url.pathname + url.search;
+                };
+
+                if (data.pages > data.page + 1) {
+                    nextPage = getPagingUrl(data.page + 2);
+                }
+
+                if (data.page > 0) {
+                    prevPage = getPagingUrl(data.page);
+                }
+
+                return h.view(
+                    'config/oauth/subscriptions',
+                    {
+                        pageTitle: 'OAuth2',
+                        menuConfig: true,
+                        menuConfigOauth: true,
+                        activeSubscriptions: true,
+
+                        showPaging: data.pages > 1,
+                        nextPage,
+                        prevPage,
+                        firstPage: data.page === 0,
+                        pageLinks: new Array(data.pages || 1).fill(0).map((z, i) => ({
+                            url: getPagingUrl(i + 1),
+                            title: i + 1,
+                            active: i === data.page
+                        })),
+
+                        apps: data.apps,
+
+                        values: {
+                            gmailSubscriptionTtl: typeof gmailSubscriptionTtl === 'number' ? gmailSubscriptionTtl : ''
+                        }
+                    },
+                    {
+                        layout: 'app'
+                    }
+                );
+            } catch (err) {
+                request.logger.error({ msg: 'Failed to load subscriptions page', err });
+                throwAsBoom(err);
+            }
+        },
+
+        options: {
+            validate: {
+                options: {
+                    stripUnknown: true,
+                    abortEarly: false,
+                    convert: true
+                },
+
+                async failAction(request, h /*, err*/) {
+                    return h.redirect('/admin/config/oauth/subscriptions').takeover();
+                },
+
+                query: Joi.object({
+                    page: Joi.number().integer().min(1).max(1000000).default(1),
+                    pageSize: Joi.number().integer().min(1).max(250).default(DEFAULT_PAGE_SIZE)
+                })
+            }
+        }
+    });
+
+    server.route({
+        method: 'POST',
+        path: '/admin/config/oauth/subscriptions',
+        async handler(request, h) {
+            try {
+                // Joi .empty('').allow(null) ensures this is either null or a number
+                let ttl = request.payload.gmailSubscriptionTtl != null ? request.payload.gmailSubscriptionTtl : null;
+                await settings.set('gmailSubscriptionTtl', ttl);
+
+                await request.flash({ type: 'info', message: 'Configuration updated' });
+                return h.redirect('/admin/config/oauth/subscriptions');
+            } catch (err) {
+                await request.flash({ type: 'danger', message: 'Failed to save settings' });
+                request.logger.error({ msg: 'Failed to save subscription settings', err });
+                return h.redirect('/admin/config/oauth/subscriptions');
+            }
+        },
+        options: {
+            validate: {
+                options: {
+                    stripUnknown: true,
+                    abortEarly: false,
+                    convert: true
+                },
+
+                async failAction(request, h /*, err*/) {
+                    await request.flash({ type: 'danger', message: 'Invalid setting value' });
+                    return h.redirect('/admin/config/oauth/subscriptions').takeover();
+                },
+
+                payload: Joi.object({
+                    gmailSubscriptionTtl: Joi.number().integer().empty('').allow(null).min(0).max(365),
+                    crumb: Joi.string().max(256)
+                })
+            }
+        }
+    });
+
     server.route({
         method: 'GET',
         path: '/admin/config/oauth/app/{app}',
@@ -2724,6 +2865,12 @@ return true;`
             try {
                 await oauth2Apps.del(request.payload.app);
 
+                try {
+                    await call({ cmd: 'googlePubSubRemove', app: request.payload.app });
+                } catch (err) {
+                    request.logger.error({ msg: 'Failed to notify workers about OAuth2 app deletion', err, app: request.payload.app });
+                }
+
                 await request.flash({ type: 'info', message: `OAuth2 app deleted` });
 
                 return h.redirect('/admin/config/oauth');
@@ -2743,7 +2890,7 @@ return true;`
 
                 async failAction(request, h, err) {
                     await request.flash({ type: 'danger', message: `Couldn't delete OAuth2 app. Try again.` });
-                    request.logger.error({ msg: 'Failed to delete delete the OAuth2 application', err });
+                    request.logger.error({ msg: 'Failed to delete the OAuth2 application', err });
 
                     return h.redirect('/admin/config/oauth').takeover();
                 },
@@ -2768,8 +2915,6 @@ return true;`
                 defaultRedirectUrl = defaultRedirectUrl.replace(/^http:\/\/127\.0\.0\.1\b/i, 'http://localhost');
             }
 
-            let pubSubApps = await oauth2Apps.list(0, 1000, { pubsub: true });
-
             return h.view(
                 'config/oauth/new',
                 {
@@ -2787,7 +2932,7 @@ return true;`
                     baseScopesApi: false,
                     baseScopesPubsub: false,
 
-                    pubSubApps: pubSubApps && pubSubApps.apps,
+                    pubSubApps: await getPubSubAppsForSelect(null),
 
                     azureClouds: structuredClone(AZURE_CLOUDS).map(entry => {
                         if (entry.id === 'global') {
@@ -2858,7 +3003,7 @@ return true;`
                     throw new Error('Unexpected result');
                 }
 
-                if (oauth2App && oauth2App.pubsubUpdates && oauth2App.pubsubUpdates.pubSubSubscription) {
+                if (oauth2App && oauth2App.pubsubUpdates && Object.keys(oauth2App.pubsubUpdates).length > 0) {
                     await call({ cmd: 'googlePubSub', app: oauth2App.id });
                 }
 
@@ -2881,8 +3026,6 @@ return true;`
                     defaultRedirectUrl = defaultRedirectUrl.replace(/^http:\/\/127\.0\.0\.1\b/i, 'http://localhost');
                 }
 
-                let pubSubApps = await oauth2Apps.list(0, 1000, { pubsub: true });
-
                 return h.view(
                     'config/oauth/new',
                     {
@@ -2896,15 +3039,7 @@ return true;`
                         providerData,
                         defaultRedirectUrl,
 
-                        pubSubApps:
-                            pubSubApps &&
-                            pubSubApps.apps &&
-                            pubSubApps.apps.map(app => {
-                                if (app.id === request.payload.pubSubApp) {
-                                    app.selected = true;
-                                }
-                                return app;
-                            }),
+                        pubSubApps: await getPubSubAppsForSelect(request.payload.pubSubApp),
 
                         baseScopesApi: baseScopes === 'api',
                         baseScopesImap: baseScopes === 'imap' || !baseScopes,
@@ -2961,8 +3096,6 @@ return true;`
                         defaultRedirectUrl = defaultRedirectUrl.replace(/^http:\/\/127\.0\.0\.1\b/i, 'http://localhost');
                     }
 
-                    let pubSubApps = await oauth2Apps.list(0, 1000, { pubsub: true });
-
                     return h
                         .view(
                             'config/oauth/new',
@@ -2977,15 +3110,7 @@ return true;`
                                 providerData,
                                 defaultRedirectUrl,
 
-                                pubSubApps:
-                                    pubSubApps &&
-                                    pubSubApps.apps &&
-                                    pubSubApps.apps.map(app => {
-                                        if (app.id === request.payload.pubSubApp) {
-                                            app.selected = true;
-                                        }
-                                        return app;
-                                    }),
+                                pubSubApps: await getPubSubAppsForSelect(request.payload.pubSubApp),
 
                                 baseScopesApi: baseScopes === 'api',
                                 baseScopesImap: baseScopes === 'imap' || !baseScopes,
@@ -3041,8 +3166,6 @@ return true;`
                 tenant: appData.authority && !['common', 'organizations', 'consumers'].includes(appData.authority) ? appData.authority : ''
             });
 
-            let pubSubApps = await oauth2Apps.list(0, 1000, { pubsub: true });
-
             return h.view(
                 'config/oauth/edit',
                 {
@@ -3059,15 +3182,7 @@ return true;`
                     hasClientSecret: !!appData.clientSecret,
                     hasServiceKey: !!appData.serviceKey,
 
-                    pubSubApps:
-                        pubSubApps &&
-                        pubSubApps.apps &&
-                        pubSubApps.apps.map(app => {
-                            if (app.id === values.pubSubApp) {
-                                app.selected = true;
-                            }
-                            return app;
-                        }),
+                    pubSubApps: await getPubSubAppsForSelect(values.pubSubApp),
 
                     values,
 
@@ -3142,7 +3257,7 @@ return true;`
                     throw new Error('Unexpected result');
                 }
 
-                if (oauth2App && oauth2App.pubsubUpdates && oauth2App.pubsubUpdates.pubSubSubscription) {
+                if (oauth2App && oauth2App.pubsubUpdates && Object.keys(oauth2App.pubsubUpdates).length > 0) {
                     await call({ cmd: 'googlePubSub', app: oauth2App.id });
                 }
 
@@ -3160,8 +3275,6 @@ return true;`
                     defaultRedirectUrl = defaultRedirectUrl.replace(/^http:\/\/127\.0\.0\.1\b/i, 'http://localhost');
                 }
 
-                let pubSubApps = await oauth2Apps.list(0, 1000, { pubsub: true });
-
                 return h.view(
                     'config/oauth/edit',
                     {
@@ -3177,15 +3290,7 @@ return true;`
                         hasClientSecret: !!appData.clientSecret,
                         hasServiceKey: !!appData.serviceKey,
 
-                        pubSubApps:
-                            pubSubApps &&
-                            pubSubApps.apps &&
-                            pubSubApps.apps.map(app => {
-                                if (app.id === request.payload.pubSubApp) {
-                                    app.selected = true;
-                                }
-                                return app;
-                            }),
+                        pubSubApps: await getPubSubAppsForSelect(request.payload.pubSubApp),
 
                         baseScopesApi: request.payload.baseScopes === 'api',
                         baseScopesImap: request.payload.baseScopes === 'imap' || !request.payload.baseScopes,
@@ -3249,8 +3354,6 @@ return true;`
                         defaultRedirectUrl = defaultRedirectUrl.replace(/^http:\/\/127\.0\.0\.1\b/i, 'http://localhost');
                     }
 
-                    let pubSubApps = await oauth2Apps.list(0, 1000, { pubsub: true });
-
                     return h
                         .view(
                             'config/oauth/edit',
@@ -3268,15 +3371,7 @@ return true;`
                                 hasClientSecret: !!appData.clientSecret,
                                 hasServiceKey: !!appData.serviceKey,
 
-                                pubSubApps:
-                                    pubSubApps &&
-                                    pubSubApps.apps &&
-                                    pubSubApps.apps.map(app => {
-                                        if (app.id === request.payload.pubSubApp) {
-                                            app.selected = true;
-                                        }
-                                        return app;
-                                    }),
+                                pubSubApps: await getPubSubAppsForSelect(request.payload.pubSubApp),
 
                                 baseScopesApi: request.payload.baseScopes === 'api',
                                 baseScopesImap: request.payload.baseScopes === 'imap' || !request.payload.baseScopes,
@@ -5473,7 +5568,7 @@ ${Buffer.from(data.content, 'base64url').toString('base64')}
 
                 async failAction(request, h, err) {
                     await request.flash({ type: 'danger', message: `Couldn't delete account. Try again.` });
-                    request.logger.error({ msg: 'Failed to delete delete the account', err });
+                    request.logger.error({ msg: 'Failed to delete the account', err });
 
                     return h.redirect('/admin/accounts').takeover();
                 },

package/lib/schemas.js

@@ -450,6 +450,14 @@ const settingsSchema = {
     queueKeep: Joi.number().integer().empty('').min(0).description('Number of completed and failed queue entries to retain for debugging'),
     deliveryAttempts: Joi.number().integer().empty('').min(0).description('Maximum number of delivery attempts before marking a message as permanently failed'),
 
+    gmailSubscriptionTtl: Joi.number()
+        .integer()
+        .empty('')
+        .min(0)
+        .max(365)
+        .description('Gmail Pub/Sub subscription inactivity expiration in days. Empty for Google default (31 days), 0 for no expiration.')
+        .label('GmailSubscriptionTTL'),
+
     /* ──────────────  Templates  ────────────── */
 
     templateHeader: Joi.string()
@@ -1159,6 +1167,13 @@ const lastErrorSchema = Joi.object({
         .label('OAuthTokenRequestError')
 }).label('AccountErrorEntry');
 
+const pubSubErrorSchema = Joi.object({
+    message: Joi.string().example('Failed to process subscription loop').description('Error message'),
+    description: Joi.string().allow(null).example('Subscription not found').description('Error details')
+})
+    .description('Pub/Sub subscription error, if any')
+    .label('PubSubError');
+
 const templateSchemas = {
     subject: Joi.string()
         .allow('')
@@ -1350,6 +1365,7 @@ const googleProjectIdSchema = Joi.string()
     .trim()
     .allow('', false, null)
     .max(256)
+    .pattern(/^[a-z][a-z0-9-]{4,28}[a-z0-9]$/)
     .example('project-name-425411')
     .description('Google Cloud Project ID')
     .label('GoogleProjectId');
@@ -1880,7 +1896,8 @@ module.exports = {
     exportStatusSchema,
     exportListSchema,
     exportProgressSchema,
-    exportIdSchema
+    exportIdSchema,
+    pubSubErrorSchema
 };
 
 /*

package/lib/ui-routes/account-routes.js

@@ -1352,7 +1352,7 @@ function init(args) {
 
                 async failAction(request, h, err) {
                     await request.flash({ type: 'danger', message: `Couldn't delete account. Try again.` });
-                    request.logger.error({ msg: 'Failed to delete delete the account', err });
+                    request.logger.error({ msg: 'Failed to delete the account', err });
 
                     return h.redirect('/admin/accounts').takeover();
                 },

package/lib/ui-routes/admin-entities-routes.js

@@ -766,7 +766,7 @@ return payload;`)
 
                 async failAction(request, h, err) {
                     await request.flash({ type: 'danger', message: `Couldn't delete webhook. Try again.` });
-                    request.logger.error({ msg: 'Failed to delete delete Webhook Route', err });
+                    request.logger.error({ msg: 'Failed to delete Webhook Route', err });
 
                     return h.redirect('/admin/webhooks').takeover();
                 },
@@ -1415,7 +1415,7 @@ return payload;`)
 
                 async failAction(request, h, err) {
                     await request.flash({ type: 'danger', message: `Couldn't delete account. Try again.` });
-                    request.logger.error({ msg: 'Failed to delete delete the account', err });
+                    request.logger.error({ msg: 'Failed to delete the account', err });
 
                     return h.redirect('/admin/templates').takeover();
                 },
@@ -2122,7 +2122,7 @@ return payload;`)
 
                 async failAction(request, h, err) {
                     await request.flash({ type: 'danger', message: `Couldn't delete gateway. Try again.` });
-                    request.logger.error({ msg: 'Failed to delete delete the gateway', err });
+                    request.logger.error({ msg: 'Failed to delete the gateway', err });
 
                     return h.redirect('/admin/gateways').takeover();
                 },

package/lib/ui-routes/oauth-routes.js

@@ -402,6 +402,12 @@ function init({ server, call }) {
             try {
                 await oauth2Apps.del(request.payload.app);
 
+                try {
+                    await call({ cmd: 'googlePubSubRemove', app: request.payload.app });
+                } catch (err) {
+                    request.logger.error({ msg: 'Failed to notify workers about OAuth2 app deletion', err, app: request.payload.app });
+                }
+
                 await request.flash({ type: 'info', message: `OAuth2 app deleted` });
 
                 return h.redirect('/admin/config/oauth');
@@ -421,7 +427,7 @@ function init({ server, call }) {
 
                 async failAction(request, h, err) {
                     await request.flash({ type: 'danger', message: `Couldn't delete OAuth2 app. Try again.` });
-                    request.logger.error({ msg: 'Failed to delete delete the OAuth2 application', err });
+                    request.logger.error({ msg: 'Failed to delete the OAuth2 application', err });
 
                     return h.redirect('/admin/config/oauth').takeover();
                 },
@@ -538,7 +544,7 @@ function init({ server, call }) {
                     throw new Error('Unexpected result');
                 }
 
-                if (oauth2App && oauth2App.pubsubUpdates && oauth2App.pubsubUpdates.pubSubSubscription) {
+                if (oauth2App && oauth2App.pubsubUpdates && Object.keys(oauth2App.pubsubUpdates).length > 0) {
                     await call({ cmd: 'googlePubSub', app: oauth2App.id });
                 }
 
@@ -824,7 +830,7 @@ function init({ server, call }) {
                     throw new Error('Unexpected result');
                 }
 
-                if (oauth2App && oauth2App.pubsubUpdates && oauth2App.pubsubUpdates.pubSubSubscription) {
+                if (oauth2App && oauth2App.pubsubUpdates && Object.keys(oauth2App.pubsubUpdates).length > 0) {
                     await call({ cmd: 'googlePubSub', app: oauth2App.id });
                 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "emailengine-app",
-    "version": "2.63.4",
+    "version": "2.64.0",
     "private": false,
     "productTitle": "EmailEngine",
     "description": "Email Sync Engine",
@@ -44,8 +44,8 @@
     "homepage": "https://emailengine.app/",
     "dependencies": {
         "@bugsnag/js": "8.8.1",
-        "@bull-board/api": "6.20.3",
-        "@bull-board/hapi": "6.20.3",
+        "@bull-board/api": "6.20.5",
+        "@bull-board/hapi": "6.20.5",
         "@elastic/elasticsearch": "8.15.3",
         "@hapi/accept": "6.0.3",
         "@hapi/bell": "13.1.0",
@@ -56,11 +56,11 @@
         "@hapi/inert": "7.1.0",
         "@hapi/vision": "7.0.3",
         "@phc/pbkdf2": "1.1.14",
-        "@postalsys/bounce-classifier": "^2.0.0",
+        "@postalsys/bounce-classifier": "^2.1.0",
         "@postalsys/certs": "1.0.12",
         "@postalsys/ee-client": "1.3.0",
-        "@postalsys/email-ai-tools": "1.11.4",
-        "@postalsys/email-text-tools": "2.4.2",
+        "@postalsys/email-ai-tools": "1.12.0",
+        "@postalsys/email-text-tools": "2.4.3",
         "@postalsys/gettext": "4.1.1",
         "@postalsys/joi-messages": "1.0.5",
         "@postalsys/templates": "2.0.0",
@@ -68,7 +68,7 @@
         "@zone-eu/wild-config": "1.7.3",
         "ace-builds": "1.43.6",
         "base32.js": "0.1.0",
-        "bullmq": "5.70.4",
+        "bullmq": "5.71.0",
         "compare-versions": "6.1.1",
         "dotenv": "17.3.1",
         "encoding-japanese": "2.2.0",
@@ -82,7 +82,7 @@
         "html-to-text": "9.0.5",
         "ical.js": "1.5.0",
         "iconv-lite": "0.7.2",
-        "imapflow": "1.2.13",
+        "imapflow": "1.2.15",
         "ioredfour": "1.4.0",
         "ioredis": "5.10.0",
         "ipaddr.js": "2.3.0",
@@ -111,7 +111,7 @@
         "speakeasy": "2.0.0",
         "startbootstrap-sb-admin-2": "3.3.7",
         "timezones-list": "3.1.0",
-        "undici": "7.22.0",
+        "undici": "7.24.4",
         "xml2js": "0.6.2"
     },
     "devDependencies": {