emailengine-app 2.68.0 → 2.69.0

package/lib/schemas.js

@@ -1285,6 +1285,23 @@ const searchSchema = Joi.object({
 
     gmailRaw: Joi.string().max(1024).example('has:attachment in:unread').description('Gmail search syntax (only works with Gmail accounts)'),
 
+    labels: Joi.object({
+        has: Joi.array()
+            .items(Joi.string().max(128))
+            .single()
+            .description('Match messages that contain ALL of these labels/categories. Supported for Gmail and MS Graph (Outlook) accounts only.')
+            .example(['Horizon'])
+            .label('HasLabels'),
+        not: Joi.array()
+            .items(Joi.string().max(128))
+            .single()
+            .description('Match messages that contain NONE of these labels/categories. Supported for Gmail and MS Graph (Outlook) accounts only.')
+            .example(['Horizon'])
+            .label('NotLabels')
+    })
+        .description('Filter by Gmail labels or MS Graph (Outlook) categories. Not supported for generic IMAP accounts.')
+        .label('LabelFilter'),
+
     emailIds: Joi.array()
         .items(emailIdSchema)
         .single()
@@ -1855,6 +1872,13 @@ const ipSchema = Joi.string()
     })
     .example('127.0.0.1');
 
+const tokenIdSchema = Joi.string()
+    .length(64)
+    .hex()
+    .example('1bc12baf7f0d5e51fe0a4e0eda06e1be5b8d6cc2c66b95dc0fbe4d2e9f5d5e1a')
+    .description('Token identifier (SHA-256 hash of the token)')
+    .label('TokenId');
+
 const accountCountersSchema = Joi.object({
     events: Joi.object()
         .unknown()
@@ -2011,7 +2035,7 @@ const exportRequestSchema = Joi.object({
         .valid('plain', 'html', '*')
         .default('*')
         .example('*')
-        .description('Text content to include: "plain", "html", "*" (both), or omit for metadata only')
+        .description('Text content to include: "plain", "html", or "*" (both)')
         .label('ExportTextType'),
     maxBytes: Joi.number()
         .integer()
@@ -2117,6 +2141,7 @@ module.exports = {
     tokenRestrictionsSchema,
     accountIdSchema,
     ipSchema,
+    tokenIdSchema,
     accountCountersSchema,
     accountPathSchema,
     messageSpecialUseSchema,

package/lib/tools.js

@@ -145,6 +145,14 @@ async function reloadHttpProxyAgent() {
     }
 }
 
+// Reload the shared HTTP proxy agent, but only when proxy-related settings actually changed.
+// Centralizes the 'settings' message handling shared by the main thread and every worker.
+function maybeReloadHttpProxyAgent(data) {
+    if (data && ('httpProxyEnabled' in data || 'httpProxyUrl' in data)) {
+        reloadHttpProxyAgent().catch(err => logger.error({ msg: 'Failed to reload HTTP proxy agent', err }));
+    }
+}
+
 async function _doReloadHttpProxyAgent() {
     let enabled, proxyUrl;
     try {
@@ -1387,6 +1395,9 @@ MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEV3QUiYsp13nD9suD1/ZkEXnuMoSg
         let redisSoftware;
         let redisCluster = false;
 
+        // Consolidated list of Redis health issues, rendered as dashboard warning banners
+        let redisWarnings = [];
+
         try {
             let redisInfo = await module.exports.getRedisStats(redis);
 
@@ -1434,6 +1445,61 @@ MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEV3QUiYsp13nD9suD1/ZkEXnuMoSg
                 softwareDetails = `Amazon MemoryDB`;
                 redisSoftware = 'memorydb';
             }
+
+            // Incompatible backend: Amazon ElastiCache
+            if (redisSoftware === 'elasticache') {
+                redisWarnings.push({
+                    key: 'elasticache',
+                    color: 'danger',
+                    title: 'Redis compatibility warning',
+                    details: [
+                        'EmailEngine is incompatible with Amazon ElastiCache as the database backend.',
+                        'Please switch to a standard Redis instance. Using ElastiCache with EmailEngine can result in data loss and some features may not work correctly.'
+                    ]
+                });
+            }
+
+            // Incompatible backend: Redis Cluster
+            if (redisCluster) {
+                redisWarnings.push({
+                    key: 'cluster',
+                    color: 'danger',
+                    title: 'Redis compatibility warning',
+                    details: ['EmailEngine is incompatible with a Redis Cluster setup.', 'Please switch to a standard Redis primary instance.']
+                });
+            }
+
+            // EmailEngine uses Redis as a database, not a cache. Any eviction silently drops sync state,
+            // which makes already-synced messages look new and triggers duplicate webhooks.
+            let maxmemoryPolicy = typeof redisInfo.maxmemory_policy === 'string' ? redisInfo.maxmemory_policy : null;
+            let maxmemory = Number(redisInfo.maxmemory) || 0;
+            let evictedKeys = Number(redisInfo.evicted_keys) || 0;
+
+            if (evictedKeys > 0) {
+                // Definitive: Redis has already evicted keys
+                redisWarnings.push({
+                    key: 'evicted-keys',
+                    color: 'danger',
+                    title: 'Redis eviction detected',
+                    details: [
+                        `Redis has evicted ${evictedKeys.toLocaleString('en-US')} key${evictedKeys === 1 ? '' : 's'} after reaching its memory limit.`,
+                        'EmailEngine uses Redis as its primary database, not a cache. Evicted keys cause already-synced messages to be reprocessed - generating duplicate webhooks - and can lead to data loss.',
+                        'Set the Redis "maxmemory-policy" to "noeviction" and provision enough memory so Redis never reaches its limit.'
+                    ]
+                });
+            } else if (maxmemoryPolicy && maxmemoryPolicy !== 'noeviction') {
+                // Latent risk: an eviction policy is configured but nothing has been evicted yet
+                redisWarnings.push({
+                    key: 'maxmemory-policy',
+                    color: maxmemory > 0 ? 'danger' : 'warning',
+                    title: 'Unsafe Redis eviction policy',
+                    details: [
+                        `The Redis "maxmemory-policy" is set to "${maxmemoryPolicy}", but EmailEngine requires "noeviction".`,
+                        'With any other policy, Redis can delete EmailEngine sync state under memory pressure, causing already-synced messages to be reprocessed - generating duplicate webhooks - and possible data loss.',
+                        'Set "maxmemory-policy noeviction" and make sure Redis has enough memory for your workload.'
+                    ]
+                });
+            }
         } catch (err) {
             logger.error({ msg: 'Failed to get stats', err });
             redisVersion = err.message;
@@ -1477,6 +1543,7 @@ MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEV3QUiYsp13nD9suD1/ZkEXnuMoSg
                 redis: `${redisVersion}${softwareDetails ? ` (${softwareDetails})` : ''}`,
                 redisSoftware,
                 redisCluster,
+                redisWarnings,
                 imapflow: ImapFlow.version || 'please upgrade',
                 bullmq: bullmqPackage.version,
                 arch: process.arch,
@@ -1571,6 +1638,11 @@ MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEV3QUiYsp13nD9suD1/ZkEXnuMoSg
 
     mergeObjects(destination, source) {
         for (let propKey of Object.keys(source)) {
+            // Guard against prototype pollution from crafted keys in parsed JSON
+            if (propKey === '__proto__' || propKey === 'constructor' || propKey === 'prototype') {
+                continue;
+            }
+
             let sourceVal = source[propKey];
 
             if (typeof destination[propKey] === 'undefined') {
@@ -1958,6 +2030,7 @@ MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEV3QUiYsp13nD9suD1/ZkEXnuMoSg
 
     httpAgent,
     reloadHttpProxyAgent,
+    maybeReloadHttpProxyAgent,
     createSocksAgent,
 
     get fetchAgent() {

package/lib/ui-routes/account-routes.js

@@ -1,205 +1,45 @@
 'use strict';
 
-const Boom = require('@hapi/boom');
+// NB! This file is processed by the gettext parser (npm run gettext) and can not use newer syntax like ?.
+
+// Admin UI routes for account management: /admin/accounts (listing), /admin/accounts/new and
+// /accounts/new* (the add-account wizard incl. IMAP autoconfig/test/server steps), and
+// /admin/accounts/{account}* (view, edit, delete, reconnect, sync, logs, browse). Extracted
+// verbatim from lib/routes-ui.js. DISABLE_MESSAGE_BROWSER and the getMailboxListing helper move
+// with the routes (only these account pages use them).
+
 const Joi = require('joi');
 const crypto = require('crypto');
-const util = require('util');
-const psl = require('psl');
+const Boom = require('@hapi/boom');
 
 const settings = require('../settings');
+const consts = require('../consts');
 const tokens = require('../tokens');
 const { redis, documentsQueue } = require('../db');
+const getSecret = require('../get-secret');
+const capa = require('../capa');
+const { Account } = require('../account');
+const { Gateway } = require('../gateway');
+const { autodetectImapSettings } = require('../autodetect-imap-settings');
+const { oauth2Apps, oauth2ProviderData, SERVICE_ACCOUNT_PROVIDERS } = require('../oauth2-apps');
 const {
-    failAction,
-    verifyAccountInfo,
-    getLogs,
-    flattenObjectKeys,
-    getSignedFormData,
     getServiceHostname,
+    getSignedFormData,
     parseSignedFormData,
+    getLogs,
+    verifyAccountInfo,
+    flattenObjectKeys,
     getBoolean,
-    readEnvValue
+    readEnvValue,
+    failAction
 } = require('../tools');
-const { Account } = require('../account');
-const { Gateway } = require('../gateway');
-const { oauth2Apps, oauth2ProviderData, SERVICE_ACCOUNT_PROVIDERS } = require('../oauth2-apps');
-const { autodetectImapSettings } = require('../autodetect-imap-settings');
-const getSecret = require('../get-secret');
-const capa = require('../capa');
-const consts = require('../consts');
 const { settingsSchema, accountIdSchema, defaultAccountTypeSchema } = require('../schemas');
-const fs = require('fs');
-const pathlib = require('path');
+const { formatAccountData, cachedTemplates } = require('./route-helpers');
 
-const { DEFAULT_MAX_LOG_LINES, DEFAULT_PAGE_SIZE, REDIS_PREFIX, MAX_FORM_TTL, NONCE_BYTES } = consts;
+const { REDIS_PREFIX, DEFAULT_PAGE_SIZE, NONCE_BYTES, MAX_FORM_TTL, DEFAULT_MAX_LOG_LINES } = consts;
 
 const DISABLE_MESSAGE_BROWSER = getBoolean(readEnvValue('EENGINE_DISABLE_MESSAGE_BROWSER'));
 
-const cachedTemplates = {
-    testSend: fs.readFileSync(pathlib.join(__dirname, '..', '..', 'views', 'partials', 'test_send.hbs'), 'utf-8')
-};
-
-function formatAccountData(account, gt) {
-    account.type = {};
-
-    if (account.oauth2 && account.oauth2.app) {
-        let providerData = oauth2ProviderData(account.oauth2.app.provider);
-        account.type = providerData;
-    } else if (account.oauth2 && account.oauth2.provider) {
-        account.type = oauth2ProviderData(account.oauth2.provider);
-    } else if (account.imap && !account.imap.disabled) {
-        account.type.icon = 'fa fa-envelope-square';
-        account.type.name = 'IMAP';
-        account.type.comment = psl.get(account.imap.host) || account.imap.host;
-    } else if (account.smtp) {
-        account.type.icon = 'fa fa-paper-plane';
-        account.type.name = 'SMTP';
-        account.type.comment = psl.get(account.smtp.host) || account.smtp.host;
-    } else if (account.oauth2 && account.oauth2.auth && account.oauth2.auth.delegatedAccount) {
-        account.type.icon = 'fa fa-arrow-alt-circle-right';
-        account.type.name = gt.gettext('Delegated');
-        account.type.comment = util.format(gt.gettext('Using credentials from "%s"'), account.oauth2.auth.delegatedAccount);
-    } else {
-        account.type.name = 'N/A';
-    }
-
-    switch (account.state) {
-        case 'init':
-            account.stateLabel = {
-                type: 'info',
-                name: 'Initializing',
-                spinner: true
-            };
-            break;
-
-        case 'connecting':
-            account.stateLabel = {
-                type: 'info',
-                name: 'Connecting'
-            };
-            break;
-
-        case 'syncing':
-            account.stateLabel = {
-                type: 'info',
-                name: 'Syncing',
-                spinner: true
-            };
-            break;
-
-        case 'connected':
-            account.stateLabel = {
-                type: 'success',
-                name: 'Connected'
-            };
-            break;
-
-        case 'disabled':
-            account.stateLabel = {
-                type: 'secondary',
-                name: 'Disabled',
-                error: account.disabledReason
-            };
-            break;
-
-        case 'authenticationError':
-        case 'connectError': {
-            let errorMessage = account.lastErrorState ? account.lastErrorState.response : false;
-            if (account.lastErrorState) {
-                switch (account.lastErrorState.serverResponseCode) {
-                    case 'ETIMEDOUT':
-                        errorMessage = gt.gettext('Connection timed out. This usually occurs if you are behind a firewall or connecting to the wrong port.');
-                        break;
-                    case 'ClosedAfterConnectTLS':
-                        errorMessage = gt.gettext('The server unexpectedly closed the connection.');
-                        break;
-                    case 'ClosedAfterConnectText':
-                        errorMessage = gt.gettext(
-                            'The server unexpectedly closed the connection. This usually happens when attempting to connect to a TLS port without TLS enabled.'
-                        );
-                        break;
-                    case 'ECONNREFUSED':
-                        errorMessage = gt.gettext(
-                            'The server refused the connection. This typically occurs if the server is not running, is overloaded, or you are connecting to the wrong host or port.'
-                        );
-                        break;
-                }
-            }
-
-            account.stateLabel = {
-                type: 'danger',
-                name: 'Failed',
-                error: errorMessage
-            };
-            break;
-        }
-        case 'unset':
-            account.stateLabel = {
-                type: 'light',
-                name: 'Not syncing'
-            };
-            break;
-        case 'disconnected':
-            account.stateLabel = {
-                type: 'warning',
-                name: 'Disconnected'
-            };
-            break;
-        case 'paused':
-            account.stateLabel = {
-                type: 'secondary',
-                name: 'Paused'
-            };
-            break;
-        default:
-            account.stateLabel = {
-                type: 'secondary',
-                name: 'N/A'
-            };
-            break;
-    }
-
-    // Check if IMAP was disabled due to errors - override state label to show error
-    if (account.imap && account.imap.disabled && account.lastErrorState) {
-        account.stateLabel = {
-            type: 'danger',
-            name: 'Failed',
-            error: account.lastErrorState.description || account.lastErrorState.response
-        };
-    }
-
-    if (account.oauth2) {
-        account.oauth2.scopes = []
-            .concat(account.oauth2.scope || [])
-            .concat(account.oauth2.scopes || [])
-            .flatMap(entry => entry.split(/\s+/))
-            .map(entry => entry.trim())
-            .filter(entry => entry);
-
-        account.oauth2.expiresStr = account.oauth2.expires ? account.oauth2.expires.toISOString() : false;
-        account.oauth2.generatedStr = account.oauth2.generated ? account.oauth2.generated.toISOString() : false;
-
-        if (account.outlookSubscription) {
-            account.outlookSubscription.subscriptionExpiresStr = account.outlookSubscription.expirationDateTime
-                ? account.outlookSubscription.expirationDateTime.toISOString()
-                : false;
-
-            let state = account.outlookSubscription.state || {};
-
-            account.outlookSubscription.isValid =
-                state.state !== 'error' && account.outlookSubscription.expirationDateTime && account.outlookSubscription.expirationDateTime > new Date();
-
-            account.outlookSubscription.stateLabel = (state.state || '').replace(/^./, c => c.toUpperCase());
-
-            if ((state.state === 'created' && !account.outlookSubscription.expirationDateTime) || account.outlookSubscription.expirationDateTime < new Date()) {
-                account.outlookSubscription.stateLabel = 'Expired';
-            }
-        }
-    }
-
-    return account;
-}
-
 async function getMailboxListing(accountObject) {
     let mailboxes = [
         {
@@ -229,7 +69,7 @@ async function getMailboxListing(accountObject) {
             return a.path.toLowerCase().localeCompare(b.path.toLowerCase());
         });
     } catch (err) {
-        // ignore
+        // failed to get mailbox list
     }
 
     return mailboxes;
@@ -238,7 +78,6 @@ async function getMailboxListing(accountObject) {
 function init(args) {
     const { server, call } = args;
 
-    // Account listing route
     server.route({
         method: 'GET',
         path: '/admin/accounts',
@@ -256,14 +95,21 @@ function init(args) {
             }
 
             for (let account of accounts.accounts) {
-                let accountObj = new Account({ redis, account: account.account });
-                account.data = await accountObj.loadAccountData(null, null, runIndex);
+                let accountObject = new Account({ redis, account: account.account });
+                try {
+                    account.data = await accountObject.loadAccountData(null, null, runIndex);
 
-                if (account.data && account.data.oauth2 && account.data.oauth2.provider) {
-                    let oauth2App = await oauth2Apps.get(account.data.oauth2.provider);
-                    if (oauth2App) {
-                        account.data.oauth2.app = oauth2App;
+                    if (account.data && account.data.oauth2 && account.data.oauth2.provider) {
+                        let oauth2App = await oauth2Apps.get(account.data.oauth2.provider);
+                        if (oauth2App) {
+                            account.data.oauth2.app = oauth2App;
+                        }
                     }
+                } catch (err) {
+                    // Account has invalid config (e.g., broken delegation)
+                    account.data = {
+                        delegationError: err.message
+                    };
                 }
             }
 
@@ -365,7 +211,7 @@ function init(args) {
                     selectedState: stateOptions.find(entry => entry.state && entry.state === request.query.state),
 
                     searchTarget: '/admin/accounts',
-                    searchPlaceholder: 'Search for accounts...',
+                    searchPlaceholder: 'Search for accounts…',
 
                     showPaging: accounts.pages > 1,
                     nextPage,
@@ -415,7 +261,6 @@ function init(args) {
         }
     });
 
-    // New account POST handler
     server.route({
         method: 'POST',
         path: '/admin/accounts/new',
@@ -566,7 +411,6 @@ function init(args) {
         );
     }
 
-    // Public GET new account form
     server.route({
         method: 'GET',
         path: '/accounts/new',
@@ -628,7 +472,6 @@ function init(args) {
         }
     });
 
-    // Public POST new account form
     server.route({
         method: 'POST',
         path: '/accounts/new',
@@ -669,7 +512,6 @@ function init(args) {
         }
     });
 
-    // IMAP account setup form
     server.route({
         method: 'POST',
         path: '/accounts/new/imap',
@@ -772,7 +614,6 @@ function init(args) {
         }
     });
 
-    // Test IMAP settings
     server.route({
         method: 'POST',
         path: '/accounts/new/imap/test',
@@ -924,7 +765,6 @@ function init(args) {
         }
     });
 
-    // Submit IMAP server settings
     server.route({
         method: 'POST',
         path: '/accounts/new/imap/server',
@@ -1097,7 +937,6 @@ function init(args) {
         }
     });
 
-    // View account details
     server.route({
         method: 'GET',
         path: '/admin/accounts/{account}',
@@ -1327,7 +1166,6 @@ function init(args) {
         }
     });
 
-    // Delete account
     server.route({
         method: 'POST',
         path: '/admin/accounts/{account}/delete',
@@ -1369,7 +1207,6 @@ function init(args) {
         }
     });
 
-    // Reconnect account
     server.route({
         method: 'POST',
         path: '/admin/accounts/{account}/reconnect',
@@ -1408,7 +1245,6 @@ function init(args) {
         }
     });
 
-    // Sync account
     server.route({
         method: 'POST',
         path: '/admin/accounts/{account}/sync',
@@ -1447,7 +1283,6 @@ function init(args) {
         }
     });
 
-    // Toggle account logs
     server.route({
         method: 'POST',
         path: '/admin/accounts/{account}/logs',
@@ -1495,7 +1330,6 @@ function init(args) {
         }
     });
 
-    // Flush account logs
     server.route({
         method: 'POST',
         path: '/admin/accounts/{account}/logs-flush',
@@ -1532,7 +1366,6 @@ function init(args) {
         }
     });
 
-    // Get account logs as text
     server.route({
         method: 'GET',
         path: '/admin/accounts/{account}/logs.txt',
@@ -1556,7 +1389,6 @@ function init(args) {
         }
     });
 
-    // Browse account messages
     server.route({
         method: 'GET',
         path: '/admin/accounts/{account}/browse',
@@ -1573,7 +1405,7 @@ function init(args) {
                 if (request.cookieAuth) {
                     request.cookieAuth.clear();
                 }
-                await request.flash({ type: 'info', message: `Sign in again to continue` });
+                await request.flash({ type: 'info', message: `Sign in again to continue.` });
                 return h.redirect('/admin/login?next=' + encodeURIComponent('/admin/accounts/{account}/browse'));
             }
 
@@ -1635,7 +1467,6 @@ function init(args) {
         }
     });
 
-    // Edit account (GET)
     server.route({
         method: 'GET',
         path: '/admin/accounts/{account}/edit',
@@ -1715,7 +1546,6 @@ function init(args) {
         }
     });
 
-    // Edit account (POST)
     server.route({
         method: 'POST',
         path: '/admin/accounts/{account}/edit',