emailengine-app 2.61.5 → 2.62.1

package/lib/oauth/mail-ru.js

@@ -2,7 +2,7 @@
 
 const packageData = require('../../package.json');
 const { fetch: fetchCmd } = require('undici');
-const { formatPartialSecretKey, structuredClone, retryAgent } = require('../tools');
+const { formatPartialSecretKey, structuredClone, httpAgent, formatTokenError } = require('../tools');
 
 const MAIL_RU_SCOPES = ['userinfo', 'mail.imap'];
 
@@ -18,11 +18,16 @@ const checkForFlags = err => {
 };
 
 const formatFetchBody = (searchParams, logRaw) => {
-    let data = Object.fromEntries(searchParams);
+    let entries = typeof searchParams === 'string' ? new URLSearchParams(searchParams) : searchParams;
+    let data = Object.fromEntries(entries);
+
+    if (logRaw) {
+        return data;
+    }
 
     for (let key of ['refresh_token', 'client_secret']) {
         if (data[key]) {
-            data[key] = logRaw ? data[key] : formatPartialSecretKey(data[key]);
+            data[key] = formatPartialSecretKey(data[key]);
         }
     }
 
@@ -92,7 +97,7 @@ class MailRuOauth {
 
         return {
             url: url.origin + url.pathname,
-            body: url.searchParams
+            body: url.searchParams.toString()
         };
     }
 
@@ -112,7 +117,7 @@ class MailRuOauth {
                 Authorization: `Basic ${Buffer.from(`${this.clientId}:${this.clientSecret}`).toString('base64')}`
             },
             body: tokenRequest.body,
-            dispatcher: retryAgent
+            dispatcher: httpAgent.retry
         });
 
         let responseJson;
@@ -154,7 +159,7 @@ class MailRuOauth {
                 code
             };
             try {
-                err.tokenRequest.response = responseJson;
+                err.tokenRequest.response = responseJson || { error: 'Failed to parse response' };
 
                 let flag = checkForFlags(err.tokenRequest.response);
                 if (flag) {
@@ -164,6 +169,7 @@ class MailRuOauth {
             } catch (e) {
                 // ignore
             }
+            err.message = formatTokenError(this.provider, err.tokenRequest);
             throw err;
         }
 
@@ -188,14 +194,15 @@ class MailRuOauth {
         let requestUrl = url.origin + url.pathname;
         let method = 'post';
 
+        const bodyString = url.searchParams.toString();
         let res = await fetchCmd(requestUrl, {
-            method: 'post',
+            method,
             headers: {
                 'Content-Type': 'application/x-www-form-urlencoded',
                 'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`
             },
-            body: url.searchParams,
-            dispatcher: retryAgent
+            body: bodyString,
+            dispatcher: httpAgent.retry
         });
 
         let responseJson;
@@ -236,7 +243,7 @@ class MailRuOauth {
                 scopes: this.scopes
             };
             try {
-                err.tokenRequest.response = responseJson;
+                err.tokenRequest.response = responseJson || { error: 'Failed to parse response' };
 
                 let flag = checkForFlags(err.tokenRequest.response);
 
@@ -247,6 +254,7 @@ class MailRuOauth {
             } catch (e) {
                 // ignore
             }
+            err.message = formatTokenError(this.provider, err.tokenRequest);
             throw err;
         }
 
@@ -263,18 +271,23 @@ class MailRuOauth {
             headers: {
                 'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`
             },
-            dispatcher: retryAgent
+            dispatcher: httpAgent.retry
         };
 
         if (payload) {
             if (!Buffer.isBuffer(payload)) {
                 reqData.headers.Accept = 'application/json';
                 reqData.headers['Content-Type'] = options?.contentType || 'application/json';
-                payload = Buffer.from(JSON.stringify(payload));
+                // Use string body instead of Buffer to avoid ArrayBuffer detachment on retry
+                reqData.body = JSON.stringify(payload);
             } else {
                 reqData.headers['Content-Type'] = options?.contentType || 'application/x-www-form-urlencoded';
+                reqData.body = payload;
+                if (payload.length > 0) {
+                    // Non-empty buffers use non-retry dispatcher to prevent ArrayBuffer detachment
+                    reqData.dispatcher = httpAgent.fetch;
+                }
             }
-            reqData.body = payload;
         }
 
         const requestUrl = new URL(url);

package/lib/oauth/outlook.js

@@ -1,7 +1,7 @@
 'use strict';
 
 const packageData = require('../../package.json');
-const { formatPartialSecretKey, structuredClone, retryAgent } = require('../tools');
+const { formatPartialSecretKey, structuredClone, httpAgent, formatTokenError } = require('../tools');
 
 const { fetch: fetchCmd } = require('undici');
 
@@ -127,11 +127,16 @@ const checkForUserFlags = err => {
 };
 
 const formatFetchBody = (searchParams, logRaw) => {
-    let data = Object.fromEntries(searchParams);
+    let entries = typeof searchParams === 'string' ? new URLSearchParams(searchParams) : searchParams;
+    let data = Object.fromEntries(entries);
+
+    if (logRaw) {
+        return data;
+    }
 
     for (let key of ['refresh_token', 'client_secret']) {
         if (data[key]) {
-            data[key] = logRaw ? data[key] : formatPartialSecretKey(data[key]);
+            data[key] = formatPartialSecretKey(data[key]);
         }
     }
 
@@ -242,7 +247,7 @@ class OutlookOauth {
 
         return {
             url: url.origin + url.pathname,
-            body: url.searchParams
+            body: url.searchParams.toString()
         };
     }
 
@@ -261,7 +266,7 @@ class OutlookOauth {
                 'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`
             },
             body: tokenRequest.body,
-            dispatcher: retryAgent
+            dispatcher: httpAgent.retry
         });
 
         let responseJson;
@@ -304,9 +309,9 @@ class OutlookOauth {
                 code
             };
             try {
-                err.tokenRequest.response = responseJson;
+                err.tokenRequest.response = responseJson || { error: 'Failed to parse response' };
 
-                if (EXPOSE_PARTIAL_SECRET_KEY_REGEX.test(err.tokenRequest.response.error_description)) {
+                if (EXPOSE_PARTIAL_SECRET_KEY_REGEX.test(err.tokenRequest.response?.error_description)) {
                     // key might have been invalidated or renewed
                     err.tokenRequest.clientSecret = formatPartialSecretKey(this.clientSecret);
                 }
@@ -319,6 +324,7 @@ class OutlookOauth {
             } catch (e) {
                 // ignore
             }
+            err.message = formatTokenError(this.provider, err.tokenRequest);
             throw err;
         }
 
@@ -349,14 +355,15 @@ class OutlookOauth {
         let requestUrl = url.origin + url.pathname;
         let method = 'post';
 
+        const bodyString = url.searchParams.toString();
         let res = await fetchCmd(requestUrl, {
-            method: 'post',
+            method,
             headers: {
                 'Content-Type': 'application/x-www-form-urlencoded',
                 'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`
             },
-            body: url.searchParams,
-            dispatcher: retryAgent
+            body: bodyString,
+            dispatcher: httpAgent.retry
         });
 
         let responseJson;
@@ -372,7 +379,7 @@ class OutlookOauth {
             this.logger.info({
                 msg: 'OAuth2 authentication request',
                 action: 'oauth2Fetch',
-                fn: 'getToken',
+                fn: 'refreshToken',
                 method,
                 url: requestUrl,
                 success: !!res.ok,
@@ -398,9 +405,9 @@ class OutlookOauth {
                 scopes: this.scopes
             };
             try {
-                err.tokenRequest.response = responseJson;
+                err.tokenRequest.response = responseJson || { error: 'Failed to parse response' };
 
-                if (EXPOSE_PARTIAL_SECRET_KEY_REGEX.test(err.tokenRequest.response.error_description)) {
+                if (EXPOSE_PARTIAL_SECRET_KEY_REGEX.test(err.tokenRequest.response?.error_description)) {
                     // key might have been invalidated or renewed
                     err.tokenRequest.clientSecret = formatPartialSecretKey(this.clientSecret);
                 }
@@ -418,6 +425,7 @@ class OutlookOauth {
             } catch (e) {
                 // ignore
             }
+            err.message = formatTokenError(this.provider, err.tokenRequest);
             throw err;
         }
 
@@ -438,7 +446,7 @@ class OutlookOauth {
                 Authorization: `Bearer ${accessToken}`,
                 'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`
             },
-            dispatcher: retryAgent
+            dispatcher: httpAgent.retry
         };
 
         if (options.headers) {
@@ -449,11 +457,16 @@ class OutlookOauth {
             if (!Buffer.isBuffer(payload)) {
                 reqData.headers.Accept = 'application/json';
                 reqData.headers['Content-Type'] = options?.contentType || 'application/json';
-                payload = Buffer.from(JSON.stringify(payload));
+                // Use string body instead of Buffer to avoid ArrayBuffer detachment on retry
+                reqData.body = JSON.stringify(payload);
             } else {
                 reqData.headers['Content-Type'] = options?.contentType || 'application/x-www-form-urlencoded';
+                reqData.body = payload;
+                if (payload.length > 0) {
+                    // Non-empty buffers use non-retry dispatcher to prevent ArrayBuffer detachment
+                    reqData.dispatcher = httpAgent.fetch;
+                }
             }
-            reqData.body = payload;
         } else if (payload && method === 'get') {
             let parsedUrl = new URL(url);
             for (let key of Object.keys(payload)) {
@@ -465,8 +478,6 @@ class OutlookOauth {
             url = parsedUrl.href;
         }
 
-        let retryCount = 0;
-
         let startTime = Date.now();
         let res = await fetchCmd(url, reqData);
         let reqTime = Date.now() - startTime;
@@ -481,7 +492,6 @@ class OutlookOauth {
                 status: res.status,
                 clientId: this.clientId,
                 scopes: this.scopes,
-                retryCount,
                 reqTime
             };
 

package/lib/oauth/pubsub/google.js

@@ -202,6 +202,11 @@ class PubSubInstance {
 
             await oauth2Apps.setMeta(this.app, { pubSubFlag: null });
         } catch (err) {
+            // Transient network errors are expected for long-polling connections
+            if (['ETIMEDOUT', 'ECONNRESET', 'ECONNREFUSED', 'UND_ERR_SOCKET', 'UND_ERR_CONNECT_TIMEOUT'].includes(err.code)) {
+                logger.warn({ msg: 'Transient error pulling subscription messages', app: this.app, code: err.code });
+                return;
+            }
             logger.error({ msg: 'Failed to pull subscription messages', app: this.app, err });
             throw err;
         }

package/lib/routes-ui.js

@@ -24,8 +24,10 @@ const {
     getDuration,
     parseSignedFormData,
     hasEnvValue,
-    retryAgent
+    httpAgent,
+    reloadHttpProxyAgent
 } = require('./tools');
+const { parentPort } = require('worker_threads');
 const { updatePublicInterfaces } = require('./utils/network');
 const packageData = require('../package.json');
 const he = require('he');
@@ -46,7 +48,8 @@ const {
     accountIdSchema,
     defaultAccountTypeSchema,
     googleProjectIdSchema,
-    googleWorkspaceAccountsSchema
+    googleWorkspaceAccountsSchema,
+    exportIdSchema
 } = require('./schemas');
 const fs = require('fs');
 const pathlib = require('path');
@@ -67,6 +70,7 @@ const { simpleParser } = require('mailparser');
 const libmime = require('libmime');
 
 const adminEntitiesRoutes = require('./ui-routes/admin-entities-routes');
+const { Export } = require('./export');
 
 const {
     DEFAULT_MAX_LOG_LINES,
@@ -869,6 +873,215 @@ function applyRoutes(server, call) {
     // Initialize admin entity routes (webhooks, templates, gateways, tokens)
     adminEntitiesRoutes({ server, call });
 
+    // Export routes for session-authenticated UI
+
+    function throwAsBoom(err) {
+        if (Boom.isBoom(err)) {
+            throw err;
+        }
+        let error = Boom.boomify(err, { statusCode: err.statusCode || 500 });
+        if (err.code) {
+            error.output.payload.code = err.code;
+        }
+        throw error;
+    }
+
+    // List exports for account
+    server.route({
+        method: 'GET',
+        path: '/admin/accounts/{account}/exports',
+        async handler(request) {
+            try {
+                return await Export.list(request.params.account, {
+                    page: request.query.page,
+                    pageSize: request.query.pageSize
+                });
+            } catch (err) {
+                request.logger.error({ msg: 'Failed to list exports', err, account: request.params.account });
+                throwAsBoom(err);
+            }
+        },
+        options: {
+            validate: {
+                options: {
+                    stripUnknown: true,
+                    abortEarly: false,
+                    convert: true
+                },
+                failAction,
+                params: Joi.object({
+                    account: Joi.string().max(256).required()
+                }),
+                query: Joi.object({
+                    page: Joi.number().integer().min(0).default(0),
+                    pageSize: Joi.number().integer().min(1).max(100).default(20)
+                })
+            }
+        }
+    });
+
+    // Get export status
+    server.route({
+        method: 'GET',
+        path: '/admin/accounts/{account}/export/{exportId}',
+        async handler(request) {
+            try {
+                const result = await Export.get(request.params.account, request.params.exportId);
+                if (!result) {
+                    throw Boom.notFound('Export not found');
+                }
+                return result;
+            } catch (err) {
+                request.logger.error({ msg: 'Failed to get export', err, account: request.params.account, exportId: request.params.exportId });
+                throwAsBoom(err);
+            }
+        },
+        options: {
+            validate: {
+                options: {
+                    stripUnknown: true,
+                    abortEarly: false,
+                    convert: true
+                },
+                failAction,
+                params: Joi.object({
+                    account: Joi.string().max(256).required(),
+                    exportId: exportIdSchema
+                })
+            }
+        }
+    });
+
+    // Create export
+    server.route({
+        method: 'POST',
+        path: '/admin/accounts/{account}/export',
+        async handler(request) {
+            try {
+                return await Export.create(request.params.account, {
+                    startDate: request.payload.startDate,
+                    endDate: request.payload.endDate,
+                    includeAttachments: request.payload.includeAttachments,
+                    folders: []
+                });
+            } catch (err) {
+                request.logger.error({ msg: 'Failed to create export', err, account: request.params.account });
+                throwAsBoom(err);
+            }
+        },
+        options: {
+            validate: {
+                options: {
+                    stripUnknown: true,
+                    abortEarly: false,
+                    convert: true
+                },
+                failAction,
+                params: Joi.object({
+                    account: Joi.string().max(256).required()
+                }),
+                payload: Joi.object({
+                    startDate: Joi.date().iso().required(),
+                    endDate: Joi.date().iso().required(),
+                    includeAttachments: Joi.boolean().default(false)
+                })
+            }
+        }
+    });
+
+    // Delete export
+    server.route({
+        method: 'DELETE',
+        path: '/admin/accounts/{account}/export/{exportId}',
+        async handler(request) {
+            try {
+                const deleted = await Export.delete(request.params.account, request.params.exportId);
+                if (!deleted) {
+                    throw Boom.notFound('Export not found');
+                }
+                return { deleted: true };
+            } catch (err) {
+                request.logger.error({ msg: 'Failed to delete export', err, account: request.params.account, exportId: request.params.exportId });
+                throwAsBoom(err);
+            }
+        },
+        options: {
+            validate: {
+                options: {
+                    stripUnknown: true,
+                    abortEarly: false,
+                    convert: true
+                },
+                failAction,
+                params: Joi.object({
+                    account: Joi.string().max(256).required(),
+                    exportId: exportIdSchema
+                })
+            }
+        }
+    });
+
+    // Download export file
+    server.route({
+        method: 'GET',
+        path: '/admin/accounts/{account}/export/{exportId}/download',
+        async handler(request, h) {
+            try {
+                const { account, exportId } = request.params;
+                const fileInfo = await Export.getFile(account, exportId);
+                if (!fileInfo) {
+                    throw Boom.notFound('Export not found');
+                }
+
+                const fileReadStream = fs.createReadStream(fileInfo.filePath);
+                let stream = fileReadStream;
+
+                stream.on('error', err => {
+                    request.logger.error({ msg: 'Export download stream error', exportId, err });
+                });
+
+                // Decrypt file if encrypted
+                if (fileInfo.isEncrypted) {
+                    const secret = await getSecret();
+                    if (!secret) {
+                        fileReadStream.destroy();
+                        throw Boom.serverUnavailable('Encryption secret not available for decryption');
+                    }
+                    const { createDecryptStream } = require('./stream-encrypt');
+                    const decryptStream = await createDecryptStream(secret);
+                    decryptStream.on('error', err => {
+                        request.logger.error({ msg: 'Export decryption error', exportId, err });
+                        fileReadStream.destroy();
+                    });
+                    stream = fileReadStream.pipe(decryptStream);
+                }
+
+                return h
+                    .response(stream)
+                    .type('application/gzip')
+                    .header('Content-Disposition', `attachment; filename="${fileInfo.filename}"`)
+                    .header('Content-Encoding', 'identity');
+            } catch (err) {
+                request.logger.error({ msg: 'Failed to download export', err, account: request.params.account, exportId: request.params.exportId });
+                throwAsBoom(err);
+            }
+        },
+        options: {
+            validate: {
+                options: {
+                    stripUnknown: true,
+                    abortEarly: false,
+                    convert: true
+                },
+                failAction,
+                params: Joi.object({
+                    account: Joi.string().max(256).required(),
+                    exportId: exportIdSchema
+                })
+            }
+        }
+    });
+
     const getDefaultPrompt = async () =>
         await call({
             cmd: 'openAiDefaultPrompt'
@@ -2230,7 +2443,7 @@ return true;`
                                 }
                             }),
                         headers,
-                        dispatcher: retryAgent
+                        dispatcher: httpAgent.retry
                     });
                     duration = Date.now() - start;
                 } catch (err) {
@@ -3293,7 +3506,7 @@ return true;`
                         url: (await settings.get('serviceUrl')) || ''
                     }),
                     headers,
-                    dispatcher: retryAgent
+                    dispatcher: httpAgent.retry
                 });
 
                 if (!res.ok) {
@@ -4435,6 +4648,11 @@ ${Buffer.from(data.content, 'base64url').toString('base64')}
 
             const nonce = data.n || crypto.randomBytes(NONCE_BYTES).toString('base64url');
 
+            // Validate nonce format (base64url, 21-22 chars; also accept base64 for backward compatibility)
+            if (!/^[A-Za-z0-9_\-+/]{21,22}={0,2}$/.test(nonce)) {
+                throw Boom.badRequest('Invalid nonce format. Please generate a new authentication URL.');
+            }
+
             // store account data with atomic SET + EX
             await redis.set(`${REDIS_PREFIX}account:add:${nonce}`, JSON.stringify(accountData), 'EX', Math.floor(MAX_FORM_TTL / 1000));
 
@@ -4725,11 +4943,29 @@ ${Buffer.from(data.content, 'base64url').toString('base64')}
                             case 'EAUTH':
                                 verifyResult.smtp.error = request.app.gt.gettext('Invalid username or password');
                                 break;
+                            case 'ENOAUTH':
+                                verifyResult.smtp.error = request.app.gt.gettext('Authentication credentials were not provided');
+                                break;
+                            case 'EOAUTH2':
+                                verifyResult.smtp.error = request.app.gt.gettext('OAuth2 authentication failed');
+                                break;
+                            case 'ETLS':
+                                verifyResult.smtp.error = request.app.gt.gettext('TLS protocol error');
+                                break;
                             case 'ESOCKET':
                                 if (/openssl/.test(verifyResult.smtp.error)) {
                                     verifyResult.smtp.error = request.app.gt.gettext('TLS protocol error');
                                 }
                                 break;
+                            case 'ETIMEDOUT':
+                                verifyResult.smtp.error = request.app.gt.gettext('Connection timed out');
+                                break;
+                            case 'ECONNECTION':
+                                verifyResult.smtp.error = request.app.gt.gettext('Could not connect to server');
+                                break;
+                            case 'EPROTOCOL':
+                                verifyResult.smtp.error = request.app.gt.gettext('Unexpected server response');
+                                break;
                         }
                     }
                 }
@@ -6547,6 +6783,8 @@ return payload;`
             let proxyEnabled = await settings.get('proxyEnabled');
             let proxyUrl = await settings.get('proxyUrl');
             let smtpEhloName = await settings.get('smtpEhloName');
+            let httpProxyEnabled = await settings.get('httpProxyEnabled');
+            let httpProxyUrl = await settings.get('httpProxyUrl');
 
             let localAddresses = [].concat((await settings.get('localAddresses')) || []);
 
@@ -6566,7 +6804,9 @@ return payload;`
                     values: {
                         proxyEnabled,
                         proxyUrl,
-                        smtpEhloName
+                        smtpEhloName,
+                        httpProxyEnabled,
+                        httpProxyUrl
                     },
 
                     addresses: await listPublicInterfaces(localAddresses),
@@ -6619,10 +6859,26 @@ return payload;`
         path: '/admin/config/network',
         async handler(request, h) {
             try {
-                for (let key of ['smtpStrategy', 'imapStrategy', 'localAddresses', 'proxyUrl', 'smtpEhloName', 'proxyEnabled']) {
+                for (let key of [
+                    'smtpStrategy',
+                    'imapStrategy',
+                    'localAddresses',
+                    'proxyUrl',
+                    'smtpEhloName',
+                    'proxyEnabled',
+                    'httpProxyEnabled',
+                    'httpProxyUrl'
+                ]) {
                     await settings.set(key, request.payload[key]);
                 }
 
+                await reloadHttpProxyAgent();
+
+                // Notify other workers about settings change
+                if (parentPort) {
+                    parentPort.postMessage({ cmd: 'settings', data: request.payload });
+                }
+
                 await request.flash({ type: 'info', message: `Configuration updated` });
 
                 return h.redirect('/admin/config/network');
@@ -6706,7 +6962,10 @@ return payload;`
 
                     proxyUrl: settingsSchema.proxyUrl,
                     smtpEhloName: settingsSchema.smtpEhloName,
-                    proxyEnabled: settingsSchema.proxyEnabled
+                    proxyEnabled: settingsSchema.proxyEnabled,
+
+                    httpProxyEnabled: settingsSchema.httpProxyEnabled,
+                    httpProxyUrl: settingsSchema.httpProxyUrl
                 })
             }
         }
@@ -7241,7 +7500,7 @@ Token: ${JSON.stringify(request.params.token)}`
                         requestor: '@postalsys/emailengine-app'
                     }),
                     headers,
-                    dispatcher: retryAgent
+                    dispatcher: httpAgent.retry
                 });
 
                 if (!res.ok) {
@@ -7352,7 +7611,7 @@ ${now}`,
                 let res = await fetchCmd(`${SMTP_TEST_HOST}/test-address/${user}`, {
                     method: 'get',
                     headers,
-                    dispatcher: retryAgent
+                    dispatcher: httpAgent.retry
                 });
 
                 if (!res.ok) {