emailengine-app 2.67.2 → 2.68.0

package/lib/oauth2-apps.js

@@ -468,12 +468,16 @@ class OAuth2AppsHandler {
             }
 
             case 'gmailService': {
+                let storedAuthMethod = await settings.get(`${id}AuthMethod`);
+                let authMethod = storedAuthMethod === 'externalAccount' ? 'externalAccount' : 'serviceKey';
                 let appData = {
                     id: 'gmailService',
                     provider: 'gmailService',
                     legacy: true,
+                    authMethod,
                     serviceClient: await settings.get(`${id}Client`),
                     serviceKey: await settings.get(`${id}Key`),
+                    externalAccount: await settings.get(`${id}ExternalAccount`),
                     extraScopes,
                     skipScopes,
 
@@ -553,8 +557,17 @@ class OAuth2AppsHandler {
                 }
                 break;
 
-            case 'gmailService':
-                for (let key of ['serviceClient', 'serviceKey', 'extraScopes', 'skipScopes']) {
+            case 'gmailService': {
+                // The authentication method is fixed once a credential has been
+                // stored - never allow switching between serviceKey and externalAccount.
+                if (typeof updates.authMethod !== 'undefined') {
+                    let hasStoredCredential = (await settings.get(`${id}Key`)) || (await settings.get(`${id}ExternalAccount`));
+                    if (hasStoredCredential) {
+                        let storedAuthMethod = await settings.get(`${id}AuthMethod`);
+                        updates = Object.assign({}, updates, { authMethod: storedAuthMethod === 'externalAccount' ? 'externalAccount' : 'serviceKey' });
+                    }
+                }
+                for (let key of ['serviceClient', 'serviceKey', 'externalAccount', 'authMethod', 'extraScopes', 'skipScopes']) {
                     if (typeof updates[key] !== 'undefined') {
                         let dataKey;
                         switch (key) {
@@ -564,6 +577,12 @@ class OAuth2AppsHandler {
                             case 'skipScopes':
                                 dataKey = 'gmailServiceSkipScopes';
                                 break;
+                            case 'externalAccount':
+                                dataKey = 'gmailServiceExternalAccount';
+                                break;
+                            case 'authMethod':
+                                dataKey = 'gmailServiceAuthMethod';
+                                break;
                             default:
                                 dataKey = `gmail${key.replace(/^./, c => c.toUpperCase())}`;
                         }
@@ -571,6 +590,7 @@ class OAuth2AppsHandler {
                     }
                 }
                 break;
+            }
 
             case 'outlook':
                 for (let key of ['clientId', 'clientSecret', 'redirectUrl', 'authority', 'extraScopes', 'skipScopes']) {
@@ -607,7 +627,20 @@ class OAuth2AppsHandler {
 
     async delLegacyApp(id) {
         let pipeline = redis.multi();
-        for (let key of ['Enabled', 'RedirectUrl', 'Client', 'ClientId', 'ClientSecret', 'Authority', 'ExtraScopes', 'SkipScopes', 'Key', 'AuthFlag']) {
+        for (let key of [
+            'Enabled',
+            'RedirectUrl',
+            'Client',
+            'ClientId',
+            'ClientSecret',
+            'Authority',
+            'ExtraScopes',
+            'SkipScopes',
+            'Key',
+            'AuthFlag',
+            'AuthMethod',
+            'ExternalAccount'
+        ]) {
             pipeline = pipeline.hdel(`${REDIS_PREFIX}settings`, `${id}${key}`);
         }
         await pipeline.exec();
@@ -669,7 +702,7 @@ class OAuth2AppsHandler {
         const id = await this.generateId();
 
         let encryptedValues = {};
-        for (let key of ['clientSecret', 'serviceKey', 'accessToken']) {
+        for (let key of ['clientSecret', 'serviceKey', 'accessToken', 'externalAccount']) {
             if (data[key]) {
                 encryptedValues[key] = await this.encrypt(data[key]);
             }
@@ -729,10 +762,17 @@ class OAuth2AppsHandler {
 
         let existingData = msgpack.decode(existingDataBuf);
 
+        // The authentication method is fixed once an app has been configured with
+        // a credential - never allow switching between serviceKey and externalAccount
+        // (WIF). Pin the stored method, ignoring any incoming change.
+        if (existingData.provider === 'gmailService' && (existingData.serviceKey || existingData.externalAccount)) {
+            data = Object.assign({}, data || {}, { authMethod: existingData.authMethod === 'externalAccount' ? 'externalAccount' : 'serviceKey' });
+        }
+
         let oldPubSubApp = existingData.pubSubApp || null;
 
         let encryptedValues = {};
-        for (let key of ['clientSecret', 'serviceKey', 'accessToken']) {
+        for (let key of ['clientSecret', 'serviceKey', 'accessToken', 'externalAccount']) {
             if (data[key]) {
                 encryptedValues[key] = await this.encrypt(data[key]);
             }
@@ -1425,14 +1465,17 @@ class OAuth2AppsHandler {
                 let serviceClient = appData.serviceClient;
 
                 let serviceClientEmail = appData.serviceClientEmail;
+                let authMethod = appData.authMethod === 'externalAccount' ? 'externalAccount' : 'serviceKey';
                 let serviceKey = appData.serviceKey ? await this.decrypt(appData.serviceKey) : null;
+                let externalAccount = appData.externalAccount ? await this.decrypt(appData.externalAccount) : null;
 
                 let scopes = formatExtraScopes(appData.extraScopes, appData.baseScopes, GMAIL_SCOPES, appData.skipScopes);
 
                 let googleProjectId = appData.googleProjectId;
                 let workspaceAccounts = appData.googleWorkspaceAccounts;
 
-                if (!serviceClient || !serviceKey) {
+                let credentialsReady = serviceClient && (authMethod === 'externalAccount' ? !!externalAccount : !!serviceKey);
+                if (!credentialsReady) {
                     let error = Boom.boomify(new Error('OAuth2 credentials not set up for Gmail'), { statusCode: 400 });
                     throw error;
                 }
@@ -1441,7 +1484,9 @@ class OAuth2AppsHandler {
                     Object.assign(
                         {
                             serviceClient,
+                            authMethod,
                             serviceKey,
+                            externalAccount,
                             googleProjectId,
                             serviceClientEmail,
                             scopes,

package/lib/routes-ui.js

@@ -38,6 +38,7 @@ const { Gateway } = require('./gateway');
 const { redis, submitQueue, notifyQueue, documentsQueue } = require('./db');
 const psl = require('psl');
 const { oauth2Apps, OAUTH_PROVIDERS, oauth2ProviderData, SERVICE_ACCOUNT_PROVIDERS } = require('./oauth2-apps');
+const { verifyOAuth2App } = require('./oauth/verify-app');
 const { autodetectImapSettings } = require('./autodetect-imap-settings');
 const getSecret = require('./get-secret');
 const os = require('os');
@@ -719,6 +720,17 @@ function applyRoutes(server, call) {
         throw error;
     }
 
+    // Render-context booleans for the gmailService authMethod tab selector.
+    // When `locked` is set the selector is shown but cannot be switched - the
+    // authentication method is fixed once an app has been created.
+    function authMethodContext(authMethod, locked) {
+        return {
+            authMethodIsServiceKey: !authMethod || authMethod === 'serviceKey',
+            authMethodIsExternalAccount: authMethod === 'externalAccount',
+            authMethodLocked: !!locked
+        };
+    }
+
     /**
      * Fetch the list of Pub/Sub apps and mark the one matching selectedId as selected.
      * Returns the apps array ready for template rendering.
@@ -2675,6 +2687,11 @@ return true;`
                 app.cloudData = AZURE_CLOUDS.find(entry => entry.id === app.cloud);
             }
 
+            // Service account apps scoped for email access (IMAP/SMTP or Gmail/Graph API) can register
+            // accounts directly without an interactive consent flow. Pub/Sub-scoped service apps are for
+            // webhook notifications only, so they must not offer the direct add-account shortcut.
+            let canAddServiceAccount = SERVICE_ACCOUNT_PROVIDERS.has(app.provider) && app.enabled && app.baseScopes !== 'pubsub';
+
             return h.view(
                 'config/oauth/app',
                 {
@@ -2690,6 +2707,11 @@ return true;`
                     baseScopesImap: app.baseScopes === 'imap' || !app.baseScopes,
                     baseScopesPubsub: app.baseScopes === 'pubsub',
 
+                    appShowAuthMethod: app.provider === 'gmailService',
+                    authMethodIsExternalAccount: app.authMethod === 'externalAccount',
+
+                    canAddServiceAccount,
+
                     disabledScopes,
                     isSendOnlyGmail,
 
@@ -2764,6 +2786,119 @@ return true;`
         }
     });
 
+    server.route({
+        method: 'POST',
+        path: '/admin/config/oauth/app/{app}/add-account',
+        async handler(request, h) {
+            const appId = request.params.app;
+            try {
+                const app = await oauth2Apps.get(appId);
+                if (!app) {
+                    let error = Boom.boomify(new Error('Application was not found.'), { statusCode: 404 });
+                    throw error;
+                }
+
+                // Direct account registration is only valid for email-scoped service account apps. Interactive
+                // providers must use the hosted consent flow, and Pub/Sub-scoped apps grant no mailbox access.
+                if (!SERVICE_ACCOUNT_PROVIDERS.has(app.provider) || !app.enabled || app.baseScopes === 'pubsub') {
+                    let error = Boom.boomify(new Error('This application can not register accounts directly.'), { statusCode: 400 });
+                    throw error;
+                }
+
+                let accountData = {
+                    account: request.payload.account || null,
+                    email: request.payload.email,
+                    oauth2: {
+                        provider: app.id,
+                        auth: {
+                            user: request.payload.email
+                        }
+                    }
+                };
+
+                if (request.payload.name) {
+                    accountData.name = request.payload.name;
+                }
+
+                const accountObject = new Account({ redis, call, secret: await getSecret() });
+                const result = await accountObject.create(accountData);
+
+                await request.flash({ type: 'info', message: `Account ${result.state === 'existing' ? 'updated' : 'added'}` });
+
+                return h.redirect(`/admin/accounts/${result.account}`);
+            } catch (err) {
+                request.logger.error({ msg: 'Failed to register service account', err, app: appId, remoteAddress: request.app.ip });
+                await request.flash({ type: 'danger', message: `Failed to add account${err.message ? `: ${err.message}` : ''}` });
+                return h.redirect(`/admin/config/oauth/app/${appId}`);
+            }
+        },
+        options: {
+            validate: {
+                options: {
+                    stripUnknown: true,
+                    abortEarly: false,
+                    convert: true
+                },
+
+                async failAction(request, h, err) {
+                    request.logger.error({ msg: 'Failed to register service account', err, app: request.params.app });
+                    await request.flash({ type: 'danger', message: `Failed to add account. Provide a valid email address.` });
+                    return h.redirect(`/admin/config/oauth/app/${request.params.app}`).takeover();
+                },
+
+                params: Joi.object({
+                    app: Joi.string().empty('').max(255).example('gmail').label('Provider').required()
+                }),
+
+                payload: Joi.object({
+                    account: accountIdSchema.default(null),
+                    name: Joi.string().empty('').max(256).example('John Smith').description('Account Name'),
+                    email: Joi.string().email().required().example('user@example.com').label('Email').description('Mailbox email address')
+                })
+            }
+        }
+    });
+
+    server.route({
+        method: 'POST',
+        path: '/admin/config/oauth/verify/{app}',
+        async handler(request, h) {
+            try {
+                return await verifyOAuth2App(request.params.app, {
+                    account: request.payload.account,
+                    testConnection: request.payload.testConnection
+                });
+            } catch (err) {
+                request.logger.error({ msg: 'Failed to verify OAuth2 application', err, app: request.params.app });
+                return h.response({ error: err.message, code: err.code || null }).code(err.statusCode || 500);
+            }
+        },
+        options: {
+            validate: {
+                options: {
+                    stripUnknown: true,
+                    abortEarly: false,
+                    convert: true
+                },
+
+                async failAction(request, h, err) {
+                    request.logger.error({ msg: 'Invalid verify request', err, app: request.params.app });
+                    return h.response({ error: 'Invalid request' }).code(400).takeover();
+                },
+
+                params: Joi.object({
+                    app: Joi.string().empty('').max(255).required().label('Provider')
+                }),
+
+                payload: Joi.object({
+                    crumb: Joi.string().optional(),
+                    account: Joi.string().trim().empty('').max(256).optional(),
+                    testConnection: Joi.boolean().truthy('Y', 'true', '1', 'on').falsy('N', 'false', 0, '').default(true)
+                })
+            }
+        }
+    });
+
     server.route({
         method: 'GET',
         path: '/admin/config/oauth/new',
@@ -2794,6 +2929,8 @@ return true;`
                     baseScopesApi: false,
                     baseScopesPubsub: false,
 
+                    ...authMethodContext('serviceKey'),
+
                     pubSubApps: await getPubSubAppsForSelect(null),
 
                     azureClouds: structuredClone(AZURE_CLOUDS).map(entry => {
@@ -2805,7 +2942,8 @@ return true;`
 
                     values: {
                         provider,
-                        redirectUrl: defaultRedirectUrl
+                        redirectUrl: defaultRedirectUrl,
+                        authMethod: 'serviceKey'
                     },
 
                     authorityCommon: true
@@ -2907,6 +3045,8 @@ return true;`
                         baseScopesImap: baseScopes === 'imap' || !baseScopes,
                         baseScopesPubsub: baseScopes === 'pubsub',
 
+                        ...authMethodContext(request.payload.authMethod),
+
                         azureClouds: structuredClone(AZURE_CLOUDS).map(entry => {
                             entry.selected = request.payload.cloud === entry.id;
                             return entry;
@@ -2978,6 +3118,8 @@ return true;`
                                 baseScopesImap: baseScopes === 'imap' || !baseScopes,
                                 baseScopesPubsub: baseScopes === 'pubsub',
 
+                                ...authMethodContext(request.payload.authMethod),
+
                                 azureClouds: structuredClone(AZURE_CLOUDS).map(entry => {
                                     entry.selected = request.payload.cloud === entry.id;
                                     return entry;
@@ -3022,6 +3164,7 @@ return true;`
             let values = Object.assign({}, appData, {
                 clientSecret: '',
                 serviceKey: '',
+                externalAccount: '',
                 extraScopes: [].concat(appData.extraScopes || []).join('\n'),
                 skipScopes: [].concat(appData.skipScopes || []).join('\n'),
 
@@ -3043,6 +3186,9 @@ return true;`
 
                     hasClientSecret: !!appData.clientSecret,
                     hasServiceKey: !!appData.serviceKey,
+                    hasExternalAccount: !!appData.externalAccount,
+
+                    ...authMethodContext(appData.authMethod, !!appData.serviceKey || !!appData.externalAccount),
 
                     pubSubApps: await getPubSubAppsForSelect(values.pubSubApp),
 
@@ -3151,6 +3297,9 @@ return true;`
 
                         hasClientSecret: !!appData.clientSecret,
                         hasServiceKey: !!appData.serviceKey,
+                        hasExternalAccount: !!appData.externalAccount,
+
+                        ...authMethodContext(appData.authMethod, !!appData.serviceKey || !!appData.externalAccount),
 
                         pubSubApps: await getPubSubAppsForSelect(request.payload.pubSubApp),
 
@@ -3232,6 +3381,9 @@ return true;`
 
                                 hasClientSecret: !!appData.clientSecret,
                                 hasServiceKey: !!appData.serviceKey,
+                                hasExternalAccount: !!appData.externalAccount,
+
+                                ...authMethodContext(request.payload.authMethod),
 
                                 pubSubApps: await getPubSubAppsForSelect(request.payload.pubSubApp),
 

package/lib/schemas.js

@@ -5,6 +5,28 @@ const config = require('@zone-eu/wild-config');
 const { getByteSize } = require('./tools');
 const { locales } = require('./translations');
 const { LEGACY_KEYS, OAUTH_PROVIDERS } = require('./oauth2-apps');
+const { validateConfig: validateExternalAccountConfig } = require('./oauth/external-account-config');
+
+// Shared Joi custom validator for the gmailService external_account credential
+// JSON. Rejects malformed configurations at save time (with an inline field
+// error) instead of letting them fail opaquely on the first token refresh.
+const externalAccountCustomValidator = (value, helpers) => {
+    if (!value) {
+        return value;
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(value);
+    } catch (err) {
+        return helpers.message('externalAccount is not valid JSON: ' + err.message);
+    }
+    try {
+        validateExternalAccountConfig(parsed);
+    } catch (err) {
+        return helpers.message(err.message);
+    }
+    return value;
+};
 
 const RESYNC_DELAY = 15 * 60;
 
@@ -1512,12 +1534,43 @@ const oauthCreateSchema = {
         .max(100 * 1024)
         .when('provider', {
             is: 'gmailService',
-            then: Joi.required(),
+            then: Joi.when('authMethod', {
+                is: 'externalAccount',
+                then: Joi.optional().allow(''),
+                otherwise: Joi.required()
+            }),
             otherwise: Joi.optional().valid(false, null)
         })
         .example('-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgk...')
         .description('Service account private key in PEM format'),
 
+    authMethod: Joi.string()
+        .valid('serviceKey', 'externalAccount')
+        .when('provider', {
+            is: 'gmailService',
+            then: Joi.optional().default('serviceKey'),
+            otherwise: Joi.optional().valid(false, null)
+        })
+        .example('externalAccount')
+        .description('Authentication method for Gmail service account ("serviceKey" or "externalAccount")'),
+
+    externalAccount: Joi.string()
+        .trim()
+        .empty('')
+        .max(64 * 1024)
+        .when('provider', {
+            is: 'gmailService',
+            then: Joi.when('authMethod', {
+                is: 'externalAccount',
+                then: Joi.required(),
+                otherwise: Joi.optional().allow('')
+            }),
+            otherwise: Joi.optional().valid(false, null)
+        })
+        .custom(externalAccountCustomValidator, 'external account JSON shape')
+        .example('{"type":"external_account","audience":"//iam.googleapis.com/projects/.../providers/...",...}')
+        .description('Google external_account credential JSON for Workload Identity Federation'),
+
     authority: Joi.string()
         .trim()
         .empty('')
@@ -1689,6 +1742,27 @@ const oauthUpdateSchema = {
         })
         .description('OAuth2 Secret Service Key'),
 
+    authMethod: Joi.string()
+        .valid('serviceKey', 'externalAccount')
+        .when('provider', {
+            is: 'gmailService',
+            then: Joi.optional().default('serviceKey'),
+            otherwise: Joi.forbidden()
+        })
+        .description('Authentication method for Gmail service account ("serviceKey" or "externalAccount")'),
+
+    externalAccount: Joi.string()
+        .trim()
+        .empty('', false, null)
+        .max(64 * 1024)
+        .when('provider', {
+            is: 'gmailService',
+            then: Joi.optional(),
+            otherwise: Joi.forbidden()
+        })
+        .custom(externalAccountCustomValidator, 'external account JSON shape')
+        .description('Google external_account credential JSON for Workload Identity Federation'),
+
     authority: Joi.string()
         .trim()
         .empty('')
@@ -1851,7 +1925,6 @@ const messageReferenceSchema = Joi.object({
     message: Joi.string()
         .base64({ paddingRequired: false, urlSafe: true })
         .max(256)
-        .required()
         .example('AAAAAQAACnA')
         .description('EmailEngine message ID to reply to or forward'),
 
@@ -1894,8 +1967,13 @@ const messageReferenceSchema = Joi.object({
         .example('<test123@example.com>')
         .description('Verify the Message-ID of the referenced email matches this value before proceeding'),
 
+    threadId: threadIdSchema.description(
+        'Gmail thread ID to attach the outgoing message to. Used only by Gmail-API accounts; ignored for IMAP and Microsoft Graph backends, which derive threading from RFC In-Reply-To/References headers. When both "message" and "threadId" are provided, the caller-supplied "threadId" takes precedence over the value derived from the referenced message.'
+    ),
+
     documentStore: documentStoreSchema.default(false).meta({ swaggerHidden: true })
 })
+    .or('message', 'threadId')
     .description('Configuration for replying to or forwarding an existing message')
     .label('MessageReference');
 

package/lib/settings.js

@@ -17,6 +17,7 @@ const ENCRYPTED_KEYS = [
     'smtpServerPassword',
     'serviceSecret',
     'gmailServiceKey',
+    'gmailServiceExternalAccount',
     'documentStorePassword',
     'openAiAPIKey',
     'totpSeed'

package/lib/tools.js

@@ -1999,6 +1999,10 @@ s7YWddnO3q0D3Kqc
 BZuj6C0wk6AflB86
 Oa4rS6X5XHZenDBh
 K48CLaXb35Wyf8ud
+vlPaOaFr8rI0WLUc
+PeOFrimigEH1HdYk
+mQN7zFRsKgMoCQ5S
+pwJ9dhcxKxo7LWtz
 6JVHv+g7kUXqgLct
 fbc/2bFTnDtwN5Kc
 vVTQRmss6oa+o3Bj
@@ -2040,6 +2044,7 @@ Wp6z04Yho9uWIObC
 aPkKbjEuTbUCplG7
 8ORR3ntJhKaosv0/
 IbZu0b/MWBxfqGte
+MtZY/1P31tzM1aEv
 TYJ9FfWyzUlZbQaB
 bg5xk8Y5UaJ5MiNo
 V+7QViQcrIxNEjZO
@@ -2063,13 +2068,7 @@ GInvRxkFTnPN97g2
 zGGN/NlynQ4GvZ3e
 MQ87qSEBNQJip6GS
 Vqm95KM44kL4+7qd
-DBAhSvOXy5FuI/il
-C6il2vOLXSqXKzNL
-RHH2IL/f3lipzTFk
-5q3TdjzMplq0+yI7
-nL6uRkZxgug/JFyl
 uSY0M+yfff4Op/HP
-tpADMZ5Ir6bmwexH
 h7Squx/sKZNkDat0
 elb9jWFuO3UHnphx
 //m57pmYUuiv84pZ
@@ -2103,10 +2102,6 @@ r/CVn8qB2xdyVXCg
 iSeRSmpWBZGpkTZi
 ZZkXc6t+ZDyuhEoO
 vPHQ3BYeEr+9DSlW
-5ChBAv/YX3PWk/Wz
-YjuOzxqQEGdKLC0Q
-nmu7gC+IxPl4AgjV
-43Du3vhZaEGambkT
 vIwyKmyMuykMUPeA
 RTIy3POEF2gZADpU
 pqvSQnQ/jx8sYGi4
@@ -2129,7 +2124,9 @@ RSvpJCGXq7lbwx3S
 h/TUqvLGXjB8N8xu
 u61uYKx5DLO9eNR7
 vWuuhT9ely8AUX2F
+mF3GOI+Ev7mJODtG
 nQNwPlZ+tyx24XVo
+olObiSmdzVaMp7lH
 cyp1GKV4Cl+eC8G/
 Q1y/TzbtUQCqnotR
 59Q2qOkuyTLzQDhd
@@ -2153,6 +2150,9 @@ EsbWnuADOq9qe/EZ
 YfUSxQtq1+kpwW/W
 QodzxvoJ6NPGhCgW
 5/VK+O950efBio0t
+RA/lrkwxcTX80nxX
+b85BMAFRHn10uX8y
+vV2RgfFH2YFTYsly
 `
         .split(/\r?\n/)
         .map(l => l.trim())