emailengine-app 2.61.5 → 2.62.0

package/lib/stream-encrypt.js

@@ -0,0 +1,263 @@
+'use strict';
+
+const crypto = require('crypto');
+const { Transform } = require('stream');
+
+const MAGIC = Buffer.from('EE01');
+const VERSION = 1;
+const CHUNK_SIZE = 64 * 1024;
+const IV_LENGTH = 12;
+const AUTH_TAG_LENGTH = 16;
+const SALT_LENGTH = 16;
+const HEADER_SIZE = MAGIC.length + 4 + 4 + SALT_LENGTH;
+const MAX_CHUNK_SIZE = CHUNK_SIZE * 4;
+
+function deriveKeyAsync(password, salt) {
+    return new Promise((resolve, reject) => {
+        crypto.scrypt(password, salt, 32, (err, key) => {
+            if (err) reject(err);
+            else resolve(key);
+        });
+    });
+}
+
+class EncryptStream extends Transform {
+    constructor({ key, salt }) {
+        super();
+        this.key = key;
+        this.salt = salt;
+        this.buffer = Buffer.alloc(0);
+        this.headerWritten = false;
+    }
+
+    _writeHeader() {
+        if (this.headerWritten) return;
+
+        const versionBuf = Buffer.alloc(4);
+        versionBuf.writeUInt32LE(VERSION);
+
+        const chunkSizeBuf = Buffer.alloc(4);
+        chunkSizeBuf.writeUInt32LE(CHUNK_SIZE);
+
+        this.push(Buffer.concat([MAGIC, versionBuf, chunkSizeBuf, this.salt]));
+        this.headerWritten = true;
+    }
+
+    _encryptChunk(chunk) {
+        const iv = crypto.randomBytes(IV_LENGTH);
+        const cipher = crypto.createCipheriv('aes-256-gcm', this.key, iv, { authTagLength: AUTH_TAG_LENGTH });
+
+        const encrypted = Buffer.concat([cipher.update(chunk), cipher.final()]);
+        const authTag = cipher.getAuthTag();
+
+        const chunkHeader = Buffer.alloc(IV_LENGTH + 4);
+        iv.copy(chunkHeader, 0);
+        chunkHeader.writeUInt32LE(encrypted.length, IV_LENGTH);
+
+        return Buffer.concat([chunkHeader, encrypted, authTag]);
+    }
+
+    _transform(data, encoding, callback) {
+        try {
+            this._writeHeader();
+
+            this.buffer = Buffer.concat([this.buffer, data]);
+
+            while (this.buffer.length >= CHUNK_SIZE) {
+                const chunk = this.buffer.subarray(0, CHUNK_SIZE);
+                this.buffer = this.buffer.subarray(CHUNK_SIZE);
+                this.push(this._encryptChunk(chunk));
+            }
+
+            callback();
+        } catch (err) {
+            callback(err);
+        }
+    }
+
+    _flush(callback) {
+        try {
+            this._writeHeader();
+
+            if (this.buffer.length > 0) {
+                this.push(this._encryptChunk(this.buffer));
+            }
+
+            this.key = null;
+            callback();
+        } catch (err) {
+            callback(err);
+        }
+    }
+}
+
+class DecryptStream extends Transform {
+    constructor(secret) {
+        super();
+        this.secret = secret;
+        this.buffer = Buffer.alloc(0);
+        this.headerParsed = false;
+        this.key = null;
+        this.chunkSize = CHUNK_SIZE;
+    }
+
+    _parseHeader() {
+        if (this.buffer.length < HEADER_SIZE) {
+            return false;
+        }
+
+        let offset = 0;
+
+        const magic = this.buffer.subarray(offset, offset + MAGIC.length);
+        if (!magic.equals(MAGIC)) {
+            throw new Error('Invalid encrypted file format: bad magic bytes');
+        }
+        offset += MAGIC.length;
+
+        const version = this.buffer.readUInt32LE(offset);
+        if (version !== VERSION) {
+            throw new Error(`Unsupported encryption version: ${version}`);
+        }
+        offset += 4;
+
+        this.chunkSize = this.buffer.readUInt32LE(offset);
+        if (this.chunkSize > MAX_CHUNK_SIZE) {
+            throw new Error('Invalid chunk size in header');
+        }
+        offset += 4;
+
+        this.salt = this.buffer.subarray(offset, offset + SALT_LENGTH);
+
+        this.buffer = this.buffer.subarray(HEADER_SIZE);
+        this.headerParsed = true;
+
+        return true;
+    }
+
+    _decryptChunk() {
+        const minChunkHeader = IV_LENGTH + 4;
+        if (this.buffer.length < minChunkHeader) {
+            return null;
+        }
+
+        const iv = this.buffer.subarray(0, IV_LENGTH);
+        const encryptedLength = this.buffer.readUInt32LE(IV_LENGTH);
+        if (encryptedLength > this.chunkSize + 256) {
+            throw new Error('Invalid encrypted chunk length');
+        }
+        const totalChunkSize = minChunkHeader + encryptedLength + AUTH_TAG_LENGTH;
+
+        if (this.buffer.length < totalChunkSize) {
+            return null;
+        }
+
+        const encryptedData = this.buffer.subarray(minChunkHeader, minChunkHeader + encryptedLength);
+        const authTag = this.buffer.subarray(minChunkHeader + encryptedLength, totalChunkSize);
+
+        const decipher = crypto.createDecipheriv('aes-256-gcm', this.key, iv, { authTagLength: AUTH_TAG_LENGTH });
+        decipher.setAuthTag(authTag);
+
+        let decrypted;
+        try {
+            decrypted = Buffer.concat([decipher.update(encryptedData), decipher.final()]);
+        } catch (err) {
+            if (err.message.includes('auth')) {
+                throw new Error('Decryption failed: invalid secret or corrupted data');
+            }
+            throw err;
+        }
+
+        this.buffer = this.buffer.subarray(totalChunkSize);
+
+        return decrypted;
+    }
+
+    _processBuffer(callback) {
+        try {
+            let decrypted;
+            while ((decrypted = this._decryptChunk()) !== null) {
+                this.push(decrypted);
+            }
+            callback();
+        } catch (err) {
+            callback(err);
+        }
+    }
+
+    _transform(data, encoding, callback) {
+        this.buffer = Buffer.concat([this.buffer, data]);
+
+        if (!this.headerParsed) {
+            try {
+                if (!this._parseHeader()) {
+                    callback();
+                    return;
+                }
+            } catch (err) {
+                callback(err);
+                return;
+            }
+            crypto.scrypt(this.secret, this.salt, 32, (err, key) => {
+                if (err) {
+                    callback(err);
+                    return;
+                }
+                this.key = key;
+                this.secret = null;
+                this._processBuffer(callback);
+            });
+            return;
+        }
+
+        this._processBuffer(callback);
+    }
+
+    _flush(callback) {
+        try {
+            if (this.buffer.length > 0) {
+                if (!this.headerParsed) {
+                    throw new Error('Invalid encrypted file: incomplete header');
+                }
+
+                const decrypted = this._decryptChunk();
+                if (decrypted !== null) {
+                    this.push(decrypted);
+                }
+
+                if (this.buffer.length > 0) {
+                    throw new Error('Invalid encrypted file: incomplete final chunk');
+                }
+            }
+
+            this.key = null;
+            callback();
+        } catch (err) {
+            callback(err);
+        }
+    }
+}
+
+async function createEncryptStream(secret) {
+    if (!secret) {
+        throw new Error('Encryption secret is required');
+    }
+    const salt = crypto.randomBytes(SALT_LENGTH);
+    const key = await deriveKeyAsync(secret, salt);
+    return new EncryptStream({ key, salt });
+}
+
+async function createDecryptStream(secret) {
+    if (!secret) {
+        throw new Error('Decryption secret is required');
+    }
+    return new DecryptStream(secret);
+}
+
+module.exports = {
+    createEncryptStream,
+    createDecryptStream,
+    MAGIC,
+    VERSION,
+    CHUNK_SIZE,
+    HEADER_SIZE
+};

package/lib/tools.js

@@ -77,6 +77,22 @@ class LRUCache extends Map {
 
 const regexCache = new LRUCache(1000);
 
+function formatTokenError(provider, tokenRequest) {
+    let parts = [`Token request failed for ${provider}`];
+    if (tokenRequest) {
+        let detail = `${tokenRequest.grant || 'unknown'}, HTTP ${tokenRequest.status || '?'}`;
+        parts[0] += ` (${detail})`;
+        if (tokenRequest.response) {
+            let resp = tokenRequest.response;
+            let errorParts = [resp.error, resp.error_description].filter(Boolean);
+            if (errorParts.length) {
+                parts.push(errorParts.join(' - '));
+            }
+        }
+    }
+    return parts.join(': ');
+}
+
 module.exports = {
     /**
      * Helper function to set specific bit in a buffer
@@ -1382,8 +1398,12 @@ MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEV3QUiYsp13nD9suD1/ZkEXnuMoSg
     },
 
     validUidValidity(value) {
-        if (typeof value === 'bigint' || typeof value === 'number') {
-            return true;
+        if (typeof value === 'bigint') {
+            return value > 0n;
+        }
+
+        if (typeof value === 'number') {
+            return Number.isFinite(value) && value > 0;
         }
 
         if (isNaN(value)) {
@@ -1762,7 +1782,10 @@ MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEV3QUiYsp13nD9suD1/ZkEXnuMoSg
 
     prepareUrl(url, baseUrl, queryParams) {
         let { pathname: baseUrlPathname } = new URL(baseUrl);
-        baseUrlPathname = baseUrlPathname.replace(/\/?/, '/');
+        // Ensure pathname ends with trailing slash for proper path concatenation
+        if (!baseUrlPathname.endsWith('/')) {
+            baseUrlPathname += '/';
+        }
 
         const urlObj = new URL(baseUrlPathname + url.replace(/^\//, ''), baseUrl);
 
@@ -1777,11 +1800,14 @@ MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEV3QUiYsp13nD9suD1/ZkEXnuMoSg
         return urlObj.href;
     },
 
+    fetchAgent,
     retryAgent,
 
     LRUCache,
 
-    normalizeHashKeys
+    normalizeHashKeys,
+
+    formatTokenError
 };
 
 function msgpackDecode(buf) {

package/lib/ui-routes/account-routes.js

@@ -520,6 +520,11 @@ function init(args) {
 
             const nonce = data.n || crypto.randomBytes(NONCE_BYTES).toString('base64url');
 
+            // Validate nonce format (base64url, 21-22 chars; also accept base64 for backward compatibility)
+            if (!/^[A-Za-z0-9_\-+/]{21,22}={0,2}$/.test(nonce)) {
+                throw Boom.badRequest('Invalid nonce format. Please generate a new authentication URL.');
+            }
+
             // store account data with atomic SET + EX
             await redis.set(`${REDIS_PREFIX}account:add:${nonce}`, JSON.stringify(accountData), 'EX', Math.floor(MAX_FORM_TTL / 1000));
 
@@ -814,11 +819,29 @@ function init(args) {
                             case 'EAUTH':
                                 verifyResult.smtp.error = request.app.gt.gettext('Invalid username or password');
                                 break;
+                            case 'ENOAUTH':
+                                verifyResult.smtp.error = request.app.gt.gettext('Authentication credentials were not provided');
+                                break;
+                            case 'EOAUTH2':
+                                verifyResult.smtp.error = request.app.gt.gettext('OAuth2 authentication failed');
+                                break;
+                            case 'ETLS':
+                                verifyResult.smtp.error = request.app.gt.gettext('TLS protocol error');
+                                break;
                             case 'ESOCKET':
                                 if (/openssl/.test(verifyResult.smtp.error)) {
                                     verifyResult.smtp.error = request.app.gt.gettext('TLS protocol error');
                                 }
                                 break;
+                            case 'ETIMEDOUT':
+                                verifyResult.smtp.error = request.app.gt.gettext('Connection timed out');
+                                break;
+                            case 'ECONNECTION':
+                                verifyResult.smtp.error = request.app.gt.gettext('Could not connect to server');
+                                break;
+                            case 'EPROTOCOL':
+                                verifyResult.smtp.error = request.app.gt.gettext('Unexpected server response');
+                                break;
                         }
                     }
                 }

package/lib/ui-routes/admin-config-routes.js

@@ -19,7 +19,8 @@ const { failAction, getByteSize, formatByteSize, getDuration, readEnvValue, hasE
 
 const { settingsSchema } = require('../schemas');
 
-const { DEFAULT_MAX_LOG_LINES, DEFAULT_DELIVERY_ATTEMPTS, REDIS_PREFIX, NONCE_BYTES } = consts;
+const { DEFAULT_MAX_LOG_LINES, DEFAULT_DELIVERY_ATTEMPTS, REDIS_PREFIX, NONCE_BYTES, DEFAULT_GMAIL_EXPORT_BATCH_SIZE, DEFAULT_OUTLOOK_EXPORT_BATCH_SIZE } =
+    consts;
 
 const { fetch: fetchCmd } = require('undici');
 
@@ -359,7 +360,9 @@ function init(args) {
                 enableOAuthTokensApi: (await settings.get('enableOAuthTokensApi')) || false,
                 ignoreMailCertErrors: (await settings.get('ignoreMailCertErrors')) || false,
                 locale: (await settings.get('locale')) || false,
-                timezone: (await settings.get('timezone')) || false
+                timezone: (await settings.get('timezone')) || false,
+                gmailExportBatchSize: (await settings.get('gmailExportBatchSize')) || DEFAULT_GMAIL_EXPORT_BATCH_SIZE,
+                outlookExportBatchSize: (await settings.get('outlookExportBatchSize')) || DEFAULT_OUTLOOK_EXPORT_BATCH_SIZE
             };
 
             if (typeof values.trackClicks !== 'boolean') {
@@ -423,7 +426,9 @@ function init(args) {
                     locale: request.payload.locale,
                     timezone: request.payload.timezone,
                     deliveryAttempts: request.payload.deliveryAttempts,
-                    imapIndexer: request.payload.imapIndexer
+                    imapIndexer: request.payload.imapIndexer,
+                    gmailExportBatchSize: request.payload.gmailExportBatchSize,
+                    outlookExportBatchSize: request.payload.outlookExportBatchSize
                 };
 
                 if (request.payload.serviceUrl) {
@@ -530,7 +535,9 @@ function init(args) {
                         .empty('')
                         .valid(...locales.map(locale => locale.locale))
                         .default('en'),
-                    timezone: settingsSchema.timezone.empty('')
+                    timezone: settingsSchema.timezone.empty(''),
+                    gmailExportBatchSize: settingsSchema.gmailExportBatchSize.default(DEFAULT_GMAIL_EXPORT_BATCH_SIZE),
+                    outlookExportBatchSize: settingsSchema.outlookExportBatchSize.default(DEFAULT_OUTLOOK_EXPORT_BATCH_SIZE)
                 })
             }
         }

package/lib/ui-routes/admin-entities-routes.js

@@ -2022,11 +2022,29 @@ return payload;`)
                             case 'EAUTH':
                                 verifyResult.smtp.error = request.app.gt.gettext('Invalid username or password');
                                 break;
+                            case 'ENOAUTH':
+                                verifyResult.smtp.error = request.app.gt.gettext('Authentication credentials were not provided');
+                                break;
+                            case 'EOAUTH2':
+                                verifyResult.smtp.error = request.app.gt.gettext('OAuth2 authentication failed');
+                                break;
+                            case 'ETLS':
+                                verifyResult.smtp.error = request.app.gt.gettext('TLS protocol error');
+                                break;
                             case 'ESOCKET':
                                 if (/openssl/.test(verifyResult.smtp.error)) {
                                     verifyResult.smtp.error = request.app.gt.gettext('TLS protocol error');
                                 }
                                 break;
+                            case 'ETIMEDOUT':
+                                verifyResult.smtp.error = request.app.gt.gettext('Connection timed out');
+                                break;
+                            case 'ECONNECTION':
+                                verifyResult.smtp.error = request.app.gt.gettext('Could not connect to server');
+                                break;
+                            case 'EPROTOCOL':
+                                verifyResult.smtp.error = request.app.gt.gettext('Unexpected server response');
+                                break;
                         }
                     }
                 }

package/lib/webhooks.js

@@ -512,9 +512,21 @@ class WebhooksHandler {
     }
 
     async pushToQueue(event, originalPayload, opts = {}) {
-        // custom webhoom routes
+        // custom webhook routes
         let webhookRoutes = await this.getWebhookHandlers();
-        let queueKeep = (await settings.get('queueKeep')) || true;
+        let queueKeep = (await settings.get('queueKeep')) ?? true;
+
+        let removePolicy = typeof queueKeep === 'number' ? { age: 24 * 3600, count: queueKeep } : queueKeep;
+        let jobOpts = {
+            removeOnComplete: removePolicy,
+            removeOnFail: removePolicy,
+            attempts: 10,
+            backoff: {
+                type: 'exponential',
+                delay: 5000,
+                jitter: 0.2 // 20% randomization to prevent thundering herd
+            }
+        };
 
         for (let route of webhookRoutes) {
             if (route.enabled && route.targetUrl && typeof route.filterFn === 'function') {
@@ -560,15 +572,7 @@ class WebhooksHandler {
                             webhook: route.id
                         });
                     } else {
-                        let job = await notifyQueue.add(event, payload, {
-                            removeOnComplete: queueKeep,
-                            removeOnFail: queueKeep,
-                            attempts: 10,
-                            backoff: {
-                                type: 'exponential',
-                                delay: 5000
-                            }
-                        });
+                        let job = await notifyQueue.add(event, payload, jobOpts);
 
                         logger.trace({
                             msg: 'Triggered custom webhook route',
@@ -589,15 +593,7 @@ class WebhooksHandler {
 
         if (!opts.routesOnly) {
             // MAIN webhook
-            await notifyQueue.add(event, originalPayload, {
-                removeOnComplete: queueKeep,
-                removeOnFail: queueKeep,
-                attempts: 10,
-                backoff: {
-                    type: 'exponential',
-                    delay: 5000
-                }
-            });
+            await notifyQueue.add(event, originalPayload, jobOpts);
         }
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "emailengine-app",
-    "version": "2.61.5",
+    "version": "2.62.0",
     "private": false,
     "productTitle": "EmailEngine",
     "description": "Email Sync Engine",
@@ -43,9 +43,9 @@
     },
     "homepage": "https://emailengine.app/",
     "dependencies": {
-        "@bugsnag/js": "8.6.0",
-        "@bull-board/api": "6.16.2",
-        "@bull-board/hapi": "6.16.2",
+        "@bugsnag/js": "8.8.1",
+        "@bull-board/api": "6.16.4",
+        "@bull-board/hapi": "6.16.4",
         "@elastic/elasticsearch": "8.15.3",
         "@hapi/accept": "6.0.3",
         "@hapi/bell": "13.1.0",
@@ -59,16 +59,16 @@
         "@postalsys/bounce-classifier": "^2.0.0",
         "@postalsys/certs": "1.0.12",
         "@postalsys/ee-client": "1.3.0",
-        "@postalsys/email-ai-tools": "1.11.2",
+        "@postalsys/email-ai-tools": "1.11.3",
         "@postalsys/email-text-tools": "2.4.1",
         "@postalsys/gettext": "4.1.1",
         "@postalsys/joi-messages": "1.0.5",
         "@postalsys/templates": "2.0.0",
         "@zone-eu/mailsplit": "5.4.8",
         "@zone-eu/wild-config": "1.7.3",
-        "ace-builds": "1.43.5",
+        "ace-builds": "1.43.6",
         "base32.js": "0.1.0",
-        "bullmq": "5.66.5",
+        "bullmq": "5.67.2",
         "compare-versions": "6.1.1",
         "dotenv": "17.2.3",
         "encoding-japanese": "2.2.0",
@@ -82,9 +82,9 @@
         "html-to-text": "9.0.5",
         "ical.js": "1.5.0",
         "iconv-lite": "0.7.2",
-        "imapflow": "1.2.6",
+        "imapflow": "1.2.8",
         "ioredfour": "1.3.0-ioredis-07",
-        "ioredis": "5.9.1",
+        "ioredis": "5.9.2",
         "ipaddr.js": "2.3.0",
         "joi": "17.13.3",
         "jquery": "3.7.1",
@@ -92,26 +92,26 @@
         "libmime": "5.3.7",
         "libqp": "2.1.1",
         "license-checker": "25.0.1",
-        "mailparser": "3.9.1",
+        "mailparser": "3.9.3",
         "marked": "9.1.6",
         "minimist": "1.2.8",
         "msgpack5": "6.0.2",
         "murmurhash": "2.0.1",
         "nanoid": "3.3.8",
-        "nodemailer": "7.0.12",
-        "pino": "10.2.0",
+        "nodemailer": "8.0.0",
+        "pino": "10.3.0",
         "popper.js": "1.16.1",
         "prom-client": "15.1.3",
         "psl": "1.15.0",
-        "pubface": "1.0.17",
+        "pubface": "1.0.18",
         "punycode.js": "2.3.1",
         "qrcode": "1.5.4",
-        "smtp-server": "3.18.0",
+        "smtp-server": "3.18.1",
         "socks": "2.8.7",
         "speakeasy": "2.0.0",
         "startbootstrap-sb-admin-2": "3.3.7",
         "timezones-list": "3.1.0",
-        "undici": "7.18.2",
+        "undici": "7.20.0",
         "xml2js": "0.6.2"
     },
     "devDependencies": {
@@ -125,7 +125,7 @@
         "grunt-wait": "0.3.0",
         "jsxgettext": "0.11.0",
         "pino-pretty": "13.0.0",
-        "prettier": "3.8.0",
+        "prettier": "3.8.1",
         "resedit": "3.0.1",
         "spdx-satisfies": "6.0.0",
         "supertest": "7.2.2",