emailengine-app 2.63.4 → 2.65.0

package/lib/outbox.js

@@ -11,7 +11,7 @@ async function list(options) {
 
     let jobCounts = await submitQueue.getJobCounts();
 
-    let jobStates = ['delayed', 'paused', 'wait', 'active'];
+    let jobStates = ['delayed', 'paused', 'wait', 'active', 'failed'];
 
     let totalJobs = jobStates.map(state => Number(jobCounts[state]) || 0).reduce((previousValue, currentValue) => previousValue + currentValue);
 

package/lib/routes-ui.js

@@ -37,7 +37,7 @@ const { Account } = require('./account');
 const { Gateway } = require('./gateway');
 const { redis, submitQueue, notifyQueue, documentsQueue } = require('./db');
 const psl = require('psl');
-const { oauth2Apps, LEGACY_KEYS, OAUTH_PROVIDERS, oauth2ProviderData } = require('./oauth2-apps');
+const { oauth2Apps, OAUTH_PROVIDERS, oauth2ProviderData, SERVICE_ACCOUNT_PROVIDERS } = require('./oauth2-apps');
 const { autodetectImapSettings } = require('./autodetect-imap-settings');
 const getSecret = require('./get-secret');
 const os = require('os');
@@ -45,10 +45,9 @@ const {
     ADDRESS_STRATEGIES,
     settingsSchema,
     oauthCreateSchema,
+    oauthUpdateSchema,
     accountIdSchema,
     defaultAccountTypeSchema,
-    googleProjectIdSchema,
-    googleWorkspaceAccountsSchema,
     exportIdSchema
 } = require('./schemas');
 const fs = require('fs');
@@ -58,7 +57,6 @@ const { Client: ElasticSearch } = require('@elastic/elasticsearch');
 const { llmPreProcess } = require('./llm-pre-process');
 const { locales } = require('./translations');
 const capa = require('./capa');
-const exampleWebhookPayloads = require('./payload-examples-webhooks.json');
 const exampleDocumentsPayloads = require('./payload-examples-documents.json');
 const { defaultMappings } = require('./es');
 const { getESClient } = require('../lib/document-store');
@@ -358,150 +356,6 @@ const OKTA_OAUTH2_CLIENT_ID = readEnvValue('OKTA_OAUTH2_CLIENT_ID');
 const OKTA_OAUTH2_CLIENT_SECRET = readEnvValue('OKTA_OAUTH2_CLIENT_SECRET');
 const USE_OKTA_AUTH = !!(OKTA_OAUTH2_ISSUER && OKTA_OAUTH2_CLIENT_ID && OKTA_OAUTH2_CLIENT_SECRET);
 
-const oauthUpdateSchema = {
-    app: Joi.string().empty('').max(255).example('gmail').label('Provider').required(),
-
-    provider: Joi.string()
-        .trim()
-        .empty('')
-        .max(256)
-        .valid(...Object.keys(OAUTH_PROVIDERS))
-        .example('gmail')
-        .required()
-        .description('OAuth2 provider'),
-
-    name: Joi.string()
-        .trim()
-        .empty('')
-        .max(256)
-        .example('My Gmail App')
-        .description('Application name')
-        .when('app', {
-            not: Joi.string().valid(...LEGACY_KEYS),
-            then: Joi.required(),
-            otherwise: Joi.optional().valid(false, null)
-        }),
-    description: Joi.string().trim().allow('').max(1024).example('My cool app').description('Application description'),
-
-    title: Joi.string().allow('').trim().max(256).example('App title').description('Title for the application button'),
-
-    enabled: Joi.boolean().truthy('Y', 'true', '1', 'on').falsy('N', 'false', 0, '').default(false).description('Enable this app'),
-
-    clientId: Joi.string()
-        .trim()
-        .allow('')
-        .max(256)
-        .when('provider', {
-            not: 'gmailService',
-            then: Joi.required(),
-            otherwise: Joi.optional().valid(false, null)
-        })
-        .description('OAuth2 Client ID'),
-
-    clientSecret: Joi.string()
-        .trim()
-        .empty('', false, null)
-        .max(256)
-        .when('provider', {
-            not: 'gmailService',
-            then: Joi.optional(),
-            otherwise: Joi.forbidden()
-        })
-        .description('OAuth2 Client Secret'),
-
-    pubSubApp: Joi.string()
-        .empty('')
-        .base64({ paddingRequired: false, urlSafe: true })
-        .max(512)
-        .example('AAAAAQAACnA')
-        .description('Cloud Pub/Sub app for Gmail API webhooks'),
-
-    extraScopes: Joi.string()
-        .allow('')
-        .trim()
-        .max(10 * 1024)
-        .description('OAuth2 Extra Scopes'),
-
-    skipScopes: Joi.string()
-        .allow('')
-        .trim()
-        .max(10 * 1024)
-        .description('OAuth2 scopes to skip from the base set'),
-
-    serviceClient: Joi.string()
-        .trim()
-        .allow('')
-        .max(256)
-        .when('provider', {
-            is: 'gmailService',
-            then: Joi.required(),
-            otherwise: Joi.optional().valid(false, null)
-        })
-        .description('OAuth2 Service Client ID'),
-
-    googleProjectId: googleProjectIdSchema,
-
-    googleWorkspaceAccounts: googleWorkspaceAccountsSchema.when('provider', {
-        is: 'gmail',
-        then: Joi.optional().default(false)
-    }),
-
-    serviceClientEmail: Joi.string()
-        .trim()
-        .allow('')
-        .email()
-        .when('provider', {
-            is: 'gmailService',
-            then: Joi.required(),
-            otherwise: Joi.optional().valid(false, null)
-        })
-        .example('name@project-123.iam.gserviceaccount.com')
-        .description('Service Client Email for 2-legged OAuth2 applications'),
-
-    serviceKey: Joi.string()
-        .trim()
-        .empty('', false, null)
-        .max(100 * 1024)
-        .when('provider', {
-            is: 'gmailService',
-            then: Joi.optional(),
-            otherwise: Joi.forbidden()
-        })
-        .description('OAuth2 Secret Service Key'),
-
-    authority: Joi.string()
-        .trim()
-        .empty('')
-        .max(1024)
-        .when('provider', {
-            is: 'outlook',
-            then: Joi.required(),
-            otherwise: Joi.optional().valid(false, null)
-        })
-        .example(false)
-        .label('SupportedAccountTypes'),
-
-    cloud: Joi.string()
-        .trim()
-        .empty('')
-        .valid('global', 'gcc-high', 'dod', 'china')
-        .example('global')
-        .description('Azure cloud type for Outlook OAuth2 applications')
-        .label('AzureCloud'),
-
-    tenant: Joi.string().trim().empty('').max(1024).example('f8cdef31-a31e-4b4a-93e4-5f571e91255a').label('Directorytenant'),
-
-    redirectUrl: Joi.string()
-        .allow('')
-        .uri({ scheme: ['http', 'https'], allowRelative: false })
-        .when('provider', {
-            not: 'gmailService',
-            then: Joi.required(),
-            otherwise: Joi.optional().valid(false, null)
-        })
-        .description('OAuth2 Callback URL')
-};
-
 function formatAccountData(account, gt) {
     account.type = {};
 
@@ -719,34 +573,6 @@ async function getOpenAiError(gt) {
     return openAiError;
 }
 
-async function getExampleWebhookPayloads() {
-    let serviceUrl = await settings.get('serviceUrl');
-    let date = new Date().toISOString();
-
-    let examplePayloads = structuredClone(exampleWebhookPayloads);
-
-    examplePayloads.forEach(payload => {
-        if (payload && payload.content) {
-            if (typeof payload.content.serviceUrl === 'string') {
-                payload.content.serviceUrl = serviceUrl;
-            }
-
-            if (typeof payload.content.date === 'string') {
-                payload.content.date = date;
-            }
-
-            if (payload.content.data && typeof payload.content.data.date === 'string') {
-                payload.content.data.date = date;
-            }
-
-            if (payload.content.data && typeof payload.content.data.created === 'string') {
-                payload.content.data.created = date;
-            }
-        }
-    });
-    return examplePayloads;
-}
-
 async function getExampleDocumentsPayloads() {
     let date = new Date().toISOString();
 
@@ -886,6 +712,21 @@ function applyRoutes(server, call) {
         throw error;
     }
 
+    /**
+     * Fetch the list of Pub/Sub apps and mark the one matching selectedId as selected.
+     * Returns the apps array ready for template rendering.
+     */
+    async function getPubSubAppsForSelect(selectedId) {
+        let result = await oauth2Apps.list(0, 1000, { pubsub: true });
+        let apps = (result && result.apps) || [];
+        for (let app of apps) {
+            if (app.id === selectedId) {
+                app.selected = true;
+            }
+        }
+        return apps;
+    }
+
     // List exports for account
     server.route({
         method: 'GET',
@@ -2568,6 +2409,7 @@ return true;`
                     pageTitle: 'OAuth2',
                     menuConfig: true,
                     menuConfigOauth: true,
+                    activeApplications: true,
 
                     newLink: newLink.pathname + newLink.search,
 
@@ -2614,6 +2456,160 @@ return true;`
         }
     });
 
+    // GET /admin/config/oauth/subscriptions - Gmail Pub/Sub subscriptions list
+    server.route({
+        method: 'GET',
+        path: '/admin/config/oauth/subscriptions',
+        async handler(request, h) {
+            try {
+                let data = await oauth2Apps.list(request.query.page - 1, request.query.pageSize, { pubsub: true });
+
+                let gmailSubscriptionTtl = await settings.get('gmailSubscriptionTtl');
+
+                // Compute human-readable expiration for each app
+                // meta.subscriptionExpiration is:
+                //   undefined - no data yet (app predates this feature or ensurePubsub hasn't run)
+                //   null      - indefinite (no TTL set, ensurePubsub confirmed this)
+                //   "Ns"      - TTL in seconds (e.g. "2678400s" for 31 days)
+                let gt = request.app.gt;
+                for (let app of data.apps) {
+                    if (!app.pubSubSubscription) {
+                        app.expirationLabel = '';
+                        continue;
+                    }
+
+                    let meta = app.meta || {};
+                    if (!('subscriptionExpiration' in meta)) {
+                        app.expirationLabel = gt.gettext('Unknown');
+                        continue;
+                    }
+
+                    let seconds = parseInt(meta.subscriptionExpiration, 10);
+                    if (seconds > 0) {
+                        let days = Math.round(seconds / 86400);
+                        app.expirationLabel = util.format(gt.ngettext('%d day', '%d days', days), days);
+                    } else {
+                        app.expirationLabel = gt.gettext('Indefinite');
+                    }
+                }
+
+                let nextPage = false;
+                let prevPage = false;
+
+                let getPagingUrl = page => {
+                    let url = new URL(`admin/config/oauth/subscriptions`, 'http://localhost');
+
+                    if (page) {
+                        url.searchParams.append('page', page);
+                    }
+
+                    if (request.query.pageSize !== DEFAULT_PAGE_SIZE) {
+                        url.searchParams.append('pageSize', request.query.pageSize);
+                    }
+
+                    return url.pathname + url.search;
+                };
+
+                if (data.pages > data.page + 1) {
+                    nextPage = getPagingUrl(data.page + 2);
+                }
+
+                if (data.page > 0) {
+                    prevPage = getPagingUrl(data.page);
+                }
+
+                return h.view(
+                    'config/oauth/subscriptions',
+                    {
+                        pageTitle: 'OAuth2',
+                        menuConfig: true,
+                        menuConfigOauth: true,
+                        activeSubscriptions: true,
+
+                        showPaging: data.pages > 1,
+                        nextPage,
+                        prevPage,
+                        firstPage: data.page === 0,
+                        pageLinks: new Array(data.pages || 1).fill(0).map((z, i) => ({
+                            url: getPagingUrl(i + 1),
+                            title: i + 1,
+                            active: i === data.page
+                        })),
+
+                        apps: data.apps,
+
+                        values: {
+                            gmailSubscriptionTtl: typeof gmailSubscriptionTtl === 'number' ? gmailSubscriptionTtl : ''
+                        }
+                    },
+                    {
+                        layout: 'app'
+                    }
+                );
+            } catch (err) {
+                request.logger.error({ msg: 'Failed to load subscriptions page', err });
+                throwAsBoom(err);
+            }
+        },
+
+        options: {
+            validate: {
+                options: {
+                    stripUnknown: true,
+                    abortEarly: false,
+                    convert: true
+                },
+
+                async failAction(request, h /*, err*/) {
+                    return h.redirect('/admin/config/oauth/subscriptions').takeover();
+                },
+
+                query: Joi.object({
+                    page: Joi.number().integer().min(1).max(1000000).default(1),
+                    pageSize: Joi.number().integer().min(1).max(250).default(DEFAULT_PAGE_SIZE)
+                })
+            }
+        }
+    });
+
+    server.route({
+        method: 'POST',
+        path: '/admin/config/oauth/subscriptions',
+        async handler(request, h) {
+            try {
+                // Joi .empty('').allow(null) ensures this is either null or a number
+                let ttl = request.payload.gmailSubscriptionTtl != null ? request.payload.gmailSubscriptionTtl : null;
+                await settings.set('gmailSubscriptionTtl', ttl);
+
+                await request.flash({ type: 'info', message: 'Configuration updated' });
+                return h.redirect('/admin/config/oauth/subscriptions');
+            } catch (err) {
+                await request.flash({ type: 'danger', message: 'Failed to save settings' });
+                request.logger.error({ msg: 'Failed to save subscription settings', err });
+                return h.redirect('/admin/config/oauth/subscriptions');
+            }
+        },
+        options: {
+            validate: {
+                options: {
+                    stripUnknown: true,
+                    abortEarly: false,
+                    convert: true
+                },
+
+                async failAction(request, h /*, err*/) {
+                    await request.flash({ type: 'danger', message: 'Invalid setting value' });
+                    return h.redirect('/admin/config/oauth/subscriptions').takeover();
+                },
+
+                payload: Joi.object({
+                    gmailSubscriptionTtl: Joi.number().integer().empty('').allow(null).min(0).max(365),
+                    crumb: Joi.string().max(256)
+                })
+            }
+        }
+    });
+
     server.route({
         method: 'GET',
         path: '/admin/config/oauth/app/{app}',
@@ -2724,6 +2720,12 @@ return true;`
             try {
                 await oauth2Apps.del(request.payload.app);
 
+                try {
+                    await call({ cmd: 'googlePubSubRemove', app: request.payload.app });
+                } catch (err) {
+                    request.logger.error({ msg: 'Failed to notify workers about OAuth2 app deletion', err, app: request.payload.app });
+                }
+
                 await request.flash({ type: 'info', message: `OAuth2 app deleted` });
 
                 return h.redirect('/admin/config/oauth');
@@ -2743,7 +2745,7 @@ return true;`
 
                 async failAction(request, h, err) {
                     await request.flash({ type: 'danger', message: `Couldn't delete OAuth2 app. Try again.` });
-                    request.logger.error({ msg: 'Failed to delete delete the OAuth2 application', err });
+                    request.logger.error({ msg: 'Failed to delete the OAuth2 application', err });
 
                     return h.redirect('/admin/config/oauth').takeover();
                 },
@@ -2768,8 +2770,6 @@ return true;`
                 defaultRedirectUrl = defaultRedirectUrl.replace(/^http:\/\/127\.0\.0\.1\b/i, 'http://localhost');
             }
 
-            let pubSubApps = await oauth2Apps.list(0, 1000, { pubsub: true });
-
             return h.view(
                 'config/oauth/new',
                 {
@@ -2787,7 +2787,7 @@ return true;`
                     baseScopesApi: false,
                     baseScopesPubsub: false,
 
-                    pubSubApps: pubSubApps && pubSubApps.apps,
+                    pubSubApps: await getPubSubAppsForSelect(null),
 
                     azureClouds: structuredClone(AZURE_CLOUDS).map(entry => {
                         if (entry.id === 'global') {
@@ -2858,7 +2858,7 @@ return true;`
                     throw new Error('Unexpected result');
                 }
 
-                if (oauth2App && oauth2App.pubsubUpdates && oauth2App.pubsubUpdates.pubSubSubscription) {
+                if (oauth2App && oauth2App.pubsubUpdates && Object.keys(oauth2App.pubsubUpdates).length > 0) {
                     await call({ cmd: 'googlePubSub', app: oauth2App.id });
                 }
 
@@ -2881,8 +2881,6 @@ return true;`
                     defaultRedirectUrl = defaultRedirectUrl.replace(/^http:\/\/127\.0\.0\.1\b/i, 'http://localhost');
                 }
 
-                let pubSubApps = await oauth2Apps.list(0, 1000, { pubsub: true });
-
                 return h.view(
                     'config/oauth/new',
                     {
@@ -2896,15 +2894,7 @@ return true;`
                         providerData,
                         defaultRedirectUrl,
 
-                        pubSubApps:
-                            pubSubApps &&
-                            pubSubApps.apps &&
-                            pubSubApps.apps.map(app => {
-                                if (app.id === request.payload.pubSubApp) {
-                                    app.selected = true;
-                                }
-                                return app;
-                            }),
+                        pubSubApps: await getPubSubAppsForSelect(request.payload.pubSubApp),
 
                         baseScopesApi: baseScopes === 'api',
                         baseScopesImap: baseScopes === 'imap' || !baseScopes,
@@ -2961,8 +2951,6 @@ return true;`
                         defaultRedirectUrl = defaultRedirectUrl.replace(/^http:\/\/127\.0\.0\.1\b/i, 'http://localhost');
                     }
 
-                    let pubSubApps = await oauth2Apps.list(0, 1000, { pubsub: true });
-
                     return h
                         .view(
                             'config/oauth/new',
@@ -2977,15 +2965,7 @@ return true;`
                                 providerData,
                                 defaultRedirectUrl,
 
-                                pubSubApps:
-                                    pubSubApps &&
-                                    pubSubApps.apps &&
-                                    pubSubApps.apps.map(app => {
-                                        if (app.id === request.payload.pubSubApp) {
-                                            app.selected = true;
-                                        }
-                                        return app;
-                                    }),
+                                pubSubApps: await getPubSubAppsForSelect(request.payload.pubSubApp),
 
                                 baseScopesApi: baseScopes === 'api',
                                 baseScopesImap: baseScopes === 'imap' || !baseScopes,
@@ -3041,8 +3021,6 @@ return true;`
                 tenant: appData.authority && !['common', 'organizations', 'consumers'].includes(appData.authority) ? appData.authority : ''
             });
 
-            let pubSubApps = await oauth2Apps.list(0, 1000, { pubsub: true });
-
             return h.view(
                 'config/oauth/edit',
                 {
@@ -3059,15 +3037,7 @@ return true;`
                     hasClientSecret: !!appData.clientSecret,
                     hasServiceKey: !!appData.serviceKey,
 
-                    pubSubApps:
-                        pubSubApps &&
-                        pubSubApps.apps &&
-                        pubSubApps.apps.map(app => {
-                            if (app.id === values.pubSubApp) {
-                                app.selected = true;
-                            }
-                            return app;
-                        }),
+                    pubSubApps: await getPubSubAppsForSelect(values.pubSubApp),
 
                     values,
 
@@ -3142,7 +3112,7 @@ return true;`
                     throw new Error('Unexpected result');
                 }
 
-                if (oauth2App && oauth2App.pubsubUpdates && oauth2App.pubsubUpdates.pubSubSubscription) {
+                if (oauth2App && oauth2App.pubsubUpdates && Object.keys(oauth2App.pubsubUpdates).length > 0) {
                     await call({ cmd: 'googlePubSub', app: oauth2App.id });
                 }
 
@@ -3160,8 +3130,6 @@ return true;`
                     defaultRedirectUrl = defaultRedirectUrl.replace(/^http:\/\/127\.0\.0\.1\b/i, 'http://localhost');
                 }
 
-                let pubSubApps = await oauth2Apps.list(0, 1000, { pubsub: true });
-
                 return h.view(
                     'config/oauth/edit',
                     {
@@ -3177,15 +3145,7 @@ return true;`
                         hasClientSecret: !!appData.clientSecret,
                         hasServiceKey: !!appData.serviceKey,
 
-                        pubSubApps:
-                            pubSubApps &&
-                            pubSubApps.apps &&
-                            pubSubApps.apps.map(app => {
-                                if (app.id === request.payload.pubSubApp) {
-                                    app.selected = true;
-                                }
-                                return app;
-                            }),
+                        pubSubApps: await getPubSubAppsForSelect(request.payload.pubSubApp),
 
                         baseScopesApi: request.payload.baseScopes === 'api',
                         baseScopesImap: request.payload.baseScopes === 'imap' || !request.payload.baseScopes,
@@ -3249,8 +3209,6 @@ return true;`
                         defaultRedirectUrl = defaultRedirectUrl.replace(/^http:\/\/127\.0\.0\.1\b/i, 'http://localhost');
                     }
 
-                    let pubSubApps = await oauth2Apps.list(0, 1000, { pubsub: true });
-
                     return h
                         .view(
                             'config/oauth/edit',
@@ -3268,15 +3226,7 @@ return true;`
                                 hasClientSecret: !!appData.clientSecret,
                                 hasServiceKey: !!appData.serviceKey,
 
-                                pubSubApps:
-                                    pubSubApps &&
-                                    pubSubApps.apps &&
-                                    pubSubApps.apps.map(app => {
-                                        if (app.id === request.payload.pubSubApp) {
-                                            app.selected = true;
-                                        }
-                                        return app;
-                                    }),
+                                pubSubApps: await getPubSubAppsForSelect(request.payload.pubSubApp),
 
                                 baseScopesApi: request.payload.baseScopes === 'api',
                                 baseScopesImap: request.payload.baseScopes === 'imap' || !request.payload.baseScopes,
@@ -4666,6 +4616,11 @@ ${Buffer.from(data.content, 'base64url').toString('base64')}
                 requestPayload.email = accountData.email;
             }
 
+            // Service providers use client_credentials - no interactive authorization
+            if (SERVICE_ACCOUNT_PROVIDERS.has(oAuth2Client.provider)) {
+                throw Boom.badRequest('Application-only OAuth providers do not support interactive authorization');
+            }
+
             let authorizeUrl = oAuth2Client.generateAuthUrl(requestPayload);
 
             return h.redirect(authorizeUrl);
@@ -5473,7 +5428,7 @@ ${Buffer.from(data.content, 'base64url').toString('base64')}
 
                 async failAction(request, h, err) {
                     await request.flash({ type: 'danger', message: `Couldn't delete account. Try again.` });
-                    request.logger.error({ msg: 'Failed to delete delete the account', err });
+                    request.logger.error({ msg: 'Failed to delete the account', err });
 
                     return h.redirect('/admin/accounts').takeover();
                 },