emailengine-app 2.69.0 → 2.70.0

package/lib/sentry.js

@@ -0,0 +1,139 @@
+'use strict';
+
+const packageData = require('../package.json');
+const logger = require('./logger');
+const { readEnvValue } = require('./tools');
+const { COMMUNITY_SENTRY_DSN } = require('./consts');
+
+const SENTRY_SETTINGS_CHECK_INTERVAL = 60 * 1000;
+
+let Sentry;
+let workerName;
+let activeDsn = false;
+
+function startSentry(dsn) {
+    // require lazily, the SDK loads several hundred modules in every worker thread,
+    // so only pay that cost when error tracking is actually enabled
+    if (!Sentry) {
+        Sentry = require('@sentry/node');
+    }
+
+    Sentry.init({
+        dsn,
+        release: packageData.version,
+        // Error capture only: skip the OpenTelemetry setup and the default
+        // integrations that patch http/fetch/console on hot paths. Sentry's own
+        // uncaughtException/unhandledRejection integrations are left out on
+        // purpose - they do not run in worker threads and do not guarantee a
+        // process exit, so lib/logger.js owns reporting and exiting instead.
+        skipOpenTelemetrySetup: true,
+        defaultIntegrations: false,
+        integrations: [
+            Sentry.eventFiltersIntegration(),
+            Sentry.functionToStringIntegration(),
+            Sentry.linkedErrorsIntegration(),
+            Sentry.contextLinesIntegration(),
+            Sentry.nodeContextIntegration(),
+            Sentry.modulesIntegration()
+        ],
+        initialScope: {
+            tags: { worker: workerName }
+        }
+    });
+
+    logger.notifyError = (err, opts) => {
+        let captureContext = {};
+        if (opts?.user) {
+            captureContext.user = { id: `${opts.user}` };
+        }
+        if (opts?.meta && Object.keys(opts.meta).length) {
+            captureContext.contexts = { ee: opts.meta };
+        }
+        Sentry.captureException(err, captureContext);
+    };
+
+    // the global exception handlers in lib/logger.js wait for this before exiting
+    logger.flushNotifications = () => Sentry.flush(2000);
+}
+
+// Tag all events with the installation identity, so reports from different
+// EmailEngine instances sharing the same DSN can be told apart. Read once per
+// SDK start - a license swapped at runtime is reflected after the next restart.
+async function applyIdentityTags() {
+    const settings = require('./settings');
+    let { serviceId, tract } = await settings.getMulti('serviceId', 'tract');
+    let tags = {};
+    if (serviceId) {
+        tags.instance = serviceId;
+    }
+    if (tract?.key) {
+        tags.license = tract.key;
+    }
+    Sentry.getGlobalScope().setTags(tags);
+}
+
+async function applySentryState(dsn) {
+    dsn = dsn || false;
+    if (dsn === activeDsn) {
+        return;
+    }
+
+    if (activeDsn) {
+        delete logger.notifyError;
+        delete logger.flushNotifications;
+        await Sentry.close(2000);
+        logger.info({ msg: 'Disabled Sentry error reporting', worker: workerName });
+    }
+
+    if (dsn) {
+        startSentry(dsn);
+        logger.info({ msg: 'Enabled Sentry error reporting', worker: workerName });
+
+        // identity lookup failure must not break error reporting, events just lack the tags
+        applyIdentityTags().catch(err => {
+            logger.error({ msg: 'Failed to apply Sentry identity tags', worker: workerName, err });
+        });
+    }
+
+    activeDsn = dsn;
+}
+
+async function checkSentrySettings() {
+    const settings = require('./settings');
+    let { sentryEnabled, sentryDsn } = await settings.getMulti('sentryEnabled', 'sentryDsn');
+    await applySentryState(sentryEnabled ? sentryDsn || COMMUNITY_SENTRY_DSN : false);
+}
+
+// Initialize Sentry error tracking. If the SENTRY_DSN environment variable is set,
+// it pins the configuration and runtime settings are ignored. Otherwise the
+// `sentryEnabled` and `sentryDsn` settings are applied at runtime and re-checked
+// periodically, so error reporting can be toggled from the admin UI without a
+// restart. While disabled, logger.notifyError stays undefined and the global
+// exception handlers in lib/logger.js exit without waiting for a delivery flush.
+function initSentry(worker) {
+    workerName = worker;
+
+    let envDsn = readEnvValue('SENTRY_DSN');
+    if (envDsn) {
+        applySentryState(envDsn).catch(err => {
+            logger.error({ msg: 'Failed to initialize Sentry', worker: workerName, err });
+        });
+        return;
+    }
+
+    // Settings changes are detected by polling instead of the {cmd: 'settings'}
+    // broadcast because that broadcast does not reach all the threads that
+    // initialize Sentry (smtp, imap-proxy, documents, and the main thread).
+    let checkSettings = async () => {
+        try {
+            await checkSentrySettings();
+        } catch (err) {
+            logger.error({ msg: 'Failed to apply Sentry settings', worker: workerName, err });
+        }
+        setTimeout(checkSettings, SENTRY_SETTINGS_CHECK_INTERVAL).unref();
+    };
+
+    setImmediate(checkSettings);
+}
+
+module.exports = { initSentry };

package/lib/settings.js

@@ -13,9 +13,12 @@ config.service = config.service || {};
 const ENCRYPTED_KEYS = [
     'gmailClientSecret',
     'outlookClientSecret',
+    'mailRuClientSecret',
     'cookiePassword',
     'smtpServerPassword',
+    'imapProxyServerPassword',
     'serviceSecret',
+    'serviceKey',
     'gmailServiceKey',
     'gmailServiceExternalAccount',
     'documentStorePassword',
@@ -100,6 +103,9 @@ module.exports = {
     formatSettingValue(key, value) {
         switch (key) {
             case 'serviceUrl': {
+                if (!value) {
+                    return value;
+                }
                 let urlObj = new URL(value);
                 return urlObj.origin;
             }
@@ -155,9 +161,9 @@ module.exports = {
             }
         }
 
-        if (key in ['generateEmailSummary', 'openAiGenerateEmbeddings'] && value) {
-            // make sure `notifyText` is enabled
-            return await redis.hset(`${REDIS_PREFIX}settings`, 'notifyText', 'true');
+        if (['generateEmailSummary', 'openAiGenerateEmbeddings'].includes(key) && formattedValue) {
+            // AI processing needs access to message text content, so make sure `notifyText` is enabled as well
+            await module.exports.set('notifyText', true);
         }
 
         if (/^openAi/.test(key) || key === 'generateEmailSummary') {

package/lib/stream-encrypt.js

@@ -162,7 +162,7 @@ class DecryptStream extends Transform {
             decrypted = Buffer.concat([decipher.update(encryptedData), decipher.final()]);
         } catch (err) {
             if (err.message.includes('auth')) {
-                throw new Error('Decryption failed: invalid secret or corrupted data');
+                throw new Error('Decryption failed: invalid secret or corrupted data', { cause: err });
             }
             throw err;
         }

package/lib/templates.js

@@ -53,7 +53,7 @@ class TemplateHandler {
                 let templateMeta = msgpack.decode(entry);
                 response.templates.push(templateMeta);
             } catch (err) {
-                logger.error({ msg: 'Failed to process template', entry: entry.toString('base64') });
+                logger.error({ msg: 'Failed to process template', entry: entry && entry.toString('base64'), err });
                 continue;
             }
         }

package/lib/tokens.js

@@ -325,16 +325,18 @@ module.exports = {
         let lastEntry = false;
         for (let i = 0; i < detailList.length; i++) {
             let entry = detailList[i];
+            // Each token occupies two consecutive multi results (data + access info)
+            let tokenHash = list[Math.floor(i / 2)];
             if (i % 2 === 0) {
                 lastEntry = false;
                 if (entry[1]) {
                     try {
                         let tokenData = msgpack.decode(entry[1]);
                         tokenData.created = new Date(tokenData.created);
-                        lastEntry = Object.assign({ id: list[response.tokens.length] }, tokenData);
+                        lastEntry = Object.assign({ id: tokenHash }, tokenData);
                         response.tokens.push(lastEntry);
                     } catch (err) {
-                        logger.error({ msg: 'Failed to process token data', hash: list[response.tokens.length], err });
+                        logger.error({ msg: 'Failed to process token data', hash: tokenHash, err });
                     }
                 }
             } else if (lastEntry && entry[1]) {
@@ -343,7 +345,7 @@ module.exports = {
                     accessData.time = accessData.time ? new Date(accessData.time) : null;
                     lastEntry.access = accessData;
                 } catch (err) {
-                    logger.error({ msg: 'Failed to process token data', hash: list[i], err });
+                    logger.error({ msg: 'Failed to process token data', hash: tokenHash, err });
                 }
             }
         }

package/lib/tools.js

@@ -1,7 +1,5 @@
 /* eslint no-bitwise: 0 */
 
-// NB! This file is processed by gettext parser and can not use newer syntax like ?.
-
 'use strict';
 
 const msgpack = require('msgpack5')();
@@ -1517,9 +1515,9 @@ MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEV3QUiYsp13nD9suD1/ZkEXnuMoSg
                     .hget(`${REDIS_PREFIX}bull:${queue}:meta`, 'paused')
                     .exec();
                 if (resActive[0] || resDelayed[0] || resWaiting[0] || resPaused[0] || resMeta[0]) {
-                    // counting failed
+                    // counting failed, skip this queue but keep the rest of the stats response
                     logger.error({ msg: 'Failed to count queue length', queue, active: resActive, delayed: resDelayed, waiting: resWaiting });
-                    return false;
+                    continue;
                 }
                 queues[queue] = {
                     active: Number(resActive[1]) || 0,

package/lib/ui-routes/account-routes.js

@@ -1,7 +1,5 @@
 'use strict';
 
-// NB! This file is processed by the gettext parser (npm run gettext) and can not use newer syntax like ?.
-
 // Admin UI routes for account management: /admin/accounts (listing), /admin/accounts/new and
 // /accounts/new* (the add-account wizard incl. IMAP autoconfig/test/server steps), and
 // /admin/accounts/{account}* (view, edit, delete, reconnect, sync, logs, browse). Extracted
@@ -33,7 +31,7 @@ const {
     readEnvValue,
     failAction
 } = require('../tools');
-const { settingsSchema, accountIdSchema, defaultAccountTypeSchema } = require('../schemas');
+const { settingsSchema, accountIdSchema, defaultAccountTypeSchema, ACCOUNT_DISPLAY_STATES } = require('../schemas');
 const { formatAccountData, cachedTemplates } = require('./route-helpers');
 
 const { REDIS_PREFIX, DEFAULT_PAGE_SIZE, NONCE_BYTES, MAX_FORM_TTL, DEFAULT_MAX_LOG_LINES } = consts;
@@ -179,6 +177,11 @@ function init(args) {
                     label: 'Disconnected'
                 },
 
+                {
+                    state: 'paused',
+                    label: 'Paused'
+                },
+
                 {
                     state: 'authenticationError',
                     label: 'Authentication failed'
@@ -252,7 +255,7 @@ function init(args) {
                     state: Joi.string()
                         .trim()
                         .empty('')
-                        .valid('init', 'syncing', 'connecting', 'connected', 'authenticationError', 'connectError', 'unset', 'disconnected')
+                        .valid(...ACCOUNT_DISPLAY_STATES)
                         .example('connected')
                         .description('Filter accounts by state')
                         .label('AccountState')

package/lib/ui-routes/admin-config-routes.js

@@ -1,7 +1,5 @@
 'use strict';
 
-// NB! This file is processed by the gettext parser (npm run gettext) and can not use newer syntax like ?.
-
 // Admin UI routes for the remaining /admin/config/* pages: webhooks, service URL/branding,
 // AI (OpenAI) settings, logging, and license. Extracted verbatim from lib/routes-ui.js.
 // The notificationTypes / configWebhooksSchema / configLoggingSchema consts, the
@@ -131,7 +129,9 @@ for (let type of notificationTypes) {
 
 const configLoggingSchema = {
     all: Joi.boolean().truthy('Y', 'true', '1', 'on').falsy('N', 'false', 0, '').default(false).description('Enable logs for all accounts'),
-    maxLogLines: Joi.number().integer().empty('').min(0).max(10000000).default(DEFAULT_MAX_LOG_LINES)
+    maxLogLines: Joi.number().integer().empty('').min(0).max(10000000).default(DEFAULT_MAX_LOG_LINES),
+    sentryEnabled: settingsSchema.sentryEnabled.default(false),
+    sentryDsn: settingsSchema.sentryDsn.default('')
 };
 
 async function getOpenAiError(gt) {
@@ -1120,6 +1120,10 @@ return true;`
 
             values.accounts = [].concat(values.accounts || []).join('\n');
 
+            let sentryValues = await settings.getMulti('sentryEnabled', 'sentryDsn');
+            values.sentryEnabled = !!sentryValues.sentryEnabled;
+            values.sentryDsn = sentryValues.sentryDsn || '';
+
             return h.view(
                 'config/logging',
                 {
@@ -1127,6 +1131,8 @@ return true;`
                     menuConfig: true,
                     menuConfigLogging: true,
 
+                    sentryEnvManaged: !!readEnvValue('SENTRY_DSN'),
+
                     values
                 },
                 {
@@ -1148,6 +1154,13 @@ return true;`
                     }
                 };
 
+                if (!readEnvValue('SENTRY_DSN')) {
+                    // the form renders these fields as disabled when the DSN is pinned by the
+                    // environment, so do not overwrite the stored values in that case
+                    data.sentryEnabled = !!request.payload.sentryEnabled;
+                    data.sentryDsn = request.payload.sentryDsn || '';
+                }
+
                 for (let key of Object.keys(data)) {
                     await settings.set(key, data[key]);
                 }

package/lib/ui-routes/oauth-config-routes.js

@@ -1,7 +1,5 @@
 'use strict';
 
-// NB! This file is processed by the gettext parser (npm run gettext) and can not use newer syntax like ?.
-
 // Admin UI routes for OAuth2 application config (/admin/config/oauth*): listing apps,
 // per-app view/edit/delete, creating apps, adding accounts, provider subscriptions, and
 // app verification. Extracted verbatim from lib/routes-ui.js. AZURE_CLOUDS and the

package/lib/ui-routes/route-helpers.js

@@ -1,7 +1,5 @@
 'use strict';
 
-// NB! This file is processed by the gettext parser (npm run gettext) and can not use newer syntax like ?.
-
 // Shared helpers used by more than one extracted UI route module - and still by
 // lib/routes-ui.js for the route groups not yet extracted. Lifting these here lets each
 // consumer import the single canonical copy instead of the monolith, so a route group can

package/lib/ui-routes/unsubscribe-routes.js

@@ -1,7 +1,5 @@
 'use strict';
 
-// NB! This file is processed by the gettext parser (npm run gettext) and can not use newer syntax like ?.
-
 // Public (unauthenticated) subscription-management routes. These render the unsubscribe
 // landing page reached from the List-Unsubscribe link in outgoing messages and process
 // the subscribe/unsubscribe form submission. Extracted verbatim from lib/routes-ui.js.

package/lib/webhooks.js

@@ -70,12 +70,12 @@ class WebhooksHandler {
 
             try {
                 let webhookMeta = msgpack.decode(entry);
-                if (webhookErrorFlag && typeof webhookErrorFlag === 'object' && !Object.keys(webhookErrorFlag)) {
+                if (webhookErrorFlag && typeof webhookErrorFlag === 'object' && !Object.keys(webhookErrorFlag).length) {
                     webhookErrorFlag = null;
                 }
                 response.webhooks.push(Object.assign(webhookMeta, { tcount, webhookErrorFlag }));
             } catch (err) {
-                logger.error({ msg: 'Failed to process webhook', entry: entry.toString('base64') });
+                logger.error({ msg: 'Failed to process webhook', entry: entry && entry.toString('base64'), err });
                 continue;
             }
         }
@@ -398,7 +398,6 @@ class WebhooksHandler {
         v = Number(v) || 0;
         if (v !== this.handlerCacheV) {
             // changes detected
-            v = this.handlerCacheV;
 
             let webhookIds = await this.redis.smembers(this.getWebhooksIndexKey());
             webhookIds = [].concat(webhookIds || []).sort((a, b) => -a.localeCompare(b));
@@ -418,7 +417,8 @@ class WebhooksHandler {
                     this.handlerCache.push(handler);
                 } else {
                     // compare existing
-                    let webhookV = await this.redis.hget(this.getWebhooksContentKey(), `${webhookId}:v`);
+                    // the per-route counter is stored as a string, existing.v is a number (see get())
+                    let webhookV = Number(await this.redis.hget(this.getWebhooksContentKey(), `${webhookId}:v`)) || 0;
                     if (existing.v !== webhookV) {
                         // update
                         for (let i = this.handlerCache.length - 1; i >= 0; i--) {
@@ -430,6 +430,10 @@ class WebhooksHandler {
                     }
                 }
             }
+
+            // mark the cache as current only after a successful refresh, so a failure above
+            // leaves the cache stale and the next call retries
+            this.handlerCacheV = v;
         }
 
         return this.handlerCache;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "emailengine-app",
-    "version": "2.69.0",
+    "version": "2.70.0",
     "private": false,
     "productTitle": "EmailEngine",
     "description": "Email Sync Engine",
@@ -11,13 +11,13 @@
         "single": "EE_OPENAPI_VERBOSE=true EENGINE_LOG_RAW=true EENGINE_SECRET=your-encryption-key EENGINE_WORKERS=1 node --inspect server --dbs.redis='redis://127.0.0.1:6379/6' --api.port=7003 --api.host=0.0.0.0 | tee $HOME/ee.log.single.txt | pino-pretty",
         "gmail": "EE_OPENAPI_VERBOSE=true EENGINE_LOG_RAW=true EENGINE_SECRET=your-encryption-key EENGINE_WORKERS=2 EENGINE_CORS_ORIGIN='*' node --inspect server --dbs.redis='redis://127.0.0.1:6379/11' --api.port=7003 --api.host=0.0.0.0 | tee $HOME/ee.log.gmail.txt | pino-pretty",
         "test": "NODE_ENV=test grunt",
-        "lint": "npx eslint lib/**/*.js workers/**/*.js test/**/*.js server.js Gruntfile.js",
+        "lint": "npx eslint 'lib/**/*.js' 'workers/**/*.js' 'test/**/*.js' server.js Gruntfile.js",
         "swagger": "./getswagger.sh",
         "build-source": "rm -rf node_modules && npm install && rm -rf node_modules && npm ci --omit=dev && rm -rf node_modules/ace-builds node_modules/@postalsys/ee-client && ./update-info.sh",
         "build-dist": "pkg --compress Brotli package.json && npm install && node winconf.js",
         "build-dist-fast": "pkg --debug package.json && npm install && node winconf.js",
         "licenses": "license-checker --excludePackages 'emailengine-app' --json | node list-generate.js > static/licenses.html",
-        "gettext": "find ./views -name \"*.hbs\" -print0 | xargs -0 xgettext-template -L Handlebars -o translations/messages.pot --force-po && jsxgettext --parser-options '{\"ecmaVersion\": 2018}' workers/api.js lib/tools.js lib/autodetect-imap-settings.js lib/ui-routes/admin-entities-routes.js lib/ui-routes/account-routes.js lib/ui-routes/oauth-config-routes.js lib/ui-routes/admin-config-routes.js lib/ui-routes/route-helpers.js lib/ui-routes/unsubscribe-routes.js -j -o translations/messages.pot",
+        "gettext": "find ./views -name \"*.hbs\" -print0 | xargs -0 xgettext-template -L Handlebars -o translations/messages.pot --force-po && node gettext-extract.js",
         "prepare-docker": "echo \"EE_DOCKER_LEGACY=$EE_DOCKER_LEGACY\" >> system.env && cat system.env",
         "update": "rm -rf node_modules package-lock.json && ncu -u && npm install && ./copy-static-files.sh && npm run licenses && npm run gettext",
         "test-gmail-api": "node lib/email-client/gmail-client.js --dbs.redis=redis://127.0.0.1/11",
@@ -43,9 +43,8 @@
     },
     "homepage": "https://emailengine.app/",
     "dependencies": {
-        "@bugsnag/js": "8.9.0",
-        "@bull-board/api": "7.2.1",
-        "@bull-board/hapi": "7.2.1",
+        "@bull-board/api": "8.0.0",
+        "@bull-board/hapi": "8.0.0",
         "@elastic/elasticsearch": "8.15.3",
         "@hapi/accept": "6.0.3",
         "@hapi/bell": "13.1.0",
@@ -64,6 +63,7 @@
         "@postalsys/gettext": "4.1.1",
         "@postalsys/joi-messages": "1.0.5",
         "@postalsys/templates": "2.0.1",
+        "@sentry/node": "10.57.0",
         "@simplewebauthn/browser": "13.3.0",
         "@simplewebauthn/server": "13.3.1",
         "@zone-eu/mailsplit": "5.4.12",
@@ -100,7 +100,7 @@
         "msgpack5": "6.0.2",
         "murmurhash": "2.0.1",
         "nanoid": "3.3.8",
-        "nodemailer": "8.0.10",
+        "nodemailer": "8.0.11",
         "pino": "10.3.1",
         "popper.js": "1.16.1",
         "prom-client": "15.1.3",
@@ -118,6 +118,8 @@
     },
     "devDependencies": {
         "@eslint/js": "10.0.1",
+        "acorn": "^8.16.0",
+        "acorn-walk": "^8.3.5",
         "chai": "4.3.10",
         "eerawlog": "1.5.3",
         "eslint": "10.4.1",
@@ -125,7 +127,6 @@
         "grunt-cli": "1.5.0",
         "grunt-shell-spawn": "0.5.0",
         "grunt-wait": "0.3.0",
-        "jsxgettext": "0.11.0",
         "pino-pretty": "13.0.0",
         "prettier": "3.8.4",
         "resedit": "3.0.2",