emailengine-app 2.61.5 → 2.62.1

package/workers/api.js

@@ -39,7 +39,8 @@ const {
     hasEnvValue,
     getBoolean,
     loadTlsConfig,
-    retryAgent
+    httpAgent,
+    reloadHttpProxyAgent
 } = require('../lib/tools');
 const { matchIp, detectAutomatedRequest } = require('../lib/utils/network');
 
@@ -132,6 +133,7 @@ const chatRoutes = require('../lib/api-routes/chat-routes');
 const bullBoardRoutes = require('../lib/api-routes/bull-board-routes');
 const accountRoutes = require('../lib/api-routes/account-routes');
 const messageRoutes = require('../lib/api-routes/message-routes');
+const exportRoutes = require('../lib/api-routes/export-routes');
 
 const {
     settingsSchema,
@@ -174,7 +176,8 @@ const AccountTypeSchema = Joi.string()
     .valid(...['imap'].concat(Object.keys(OAUTH_PROVIDERS)).concat('oauth2'))
     .example('outlook')
     .description('Account type')
-    .required();
+    .required()
+    .label('AccountType');
 
 const SUPPORTED_LOCALES = locales.map(locale => locale.locale);
 
@@ -289,7 +292,7 @@ class ResponseStream extends Transform {
         this._finalized = false;
 
         // Ensure cleanup on all stream end scenarios
-        this.once('error', () => this.finalize());
+        this.on('error', () => this.finalize());
         this.once('close', () => this.finalize());
         this.once('end', () => this.finalize());
     }
@@ -297,6 +300,7 @@ class ResponseStream extends Transform {
     updateTimer() {
         clearTimeout(this.periodicKeepAliveTimer);
         this.periodicKeepAliveTimer = setTimeout(() => {
+            if (this._finalized || this.destroyed) return;
             this.write(': still here\n\n');
             if (this._compressor) {
                 this._compressor.flush();
@@ -311,6 +315,7 @@ class ResponseStream extends Transform {
     }
 
     sendMessage(payload) {
+        if (this._finalized || this.destroyed) return;
         let sendData = JSON.stringify(payload);
         this.write('event: message\ndata:' + sendData + '\n\n');
         if (this._compressor) {
@@ -325,6 +330,10 @@ class ResponseStream extends Transform {
 
         clearTimeout(this.periodicKeepAliveTimer);
         registeredPublishers.delete(this);
+
+        if (!this.destroyed) {
+            this.destroy();
+        }
     }
 
     _transform(data, encoding, done) {
@@ -914,6 +923,8 @@ Include your token in requests using one of these methods:
 
         security: [{ bearerAuth: [] }],
 
+        definitionPrefix: 'useLabel',
+
         cors: !!CORS_CONFIG,
         cache: {
             expiresIn: 7 * 24 * 60 * 60 * 1000
@@ -1008,6 +1019,11 @@ Include your token in requests using one of these methods:
             {
                 name: 'Multi Message Actions',
                 description: 'Perform bulk operations on multiple messages simultaneously, such as marking as read, moving, or deleting'
+            },
+            {
+                name: 'Export (Beta)',
+                description:
+                    'Bulk export messages from email accounts. This feature is in beta and the API may change in future releases. Export files are encrypted at rest when a service secret is configured.'
             }
         ],
 
@@ -2027,10 +2043,10 @@ Include your token in requests using one of these methods:
             }
 
             // Validate nonce format: 16 bytes base64url encoded = 21-22 characters
+            // Also accept base64 encoding (+, /, =) for backward compatibility with old cached nonces
             const stateNonce = request.query.state.slice('account:add:'.length);
-            if (!/^[A-Za-z0-9_-]{21,22}$/.test(stateNonce)) {
-                let error = Boom.boomify(new Error(`Oauth failed: invalid state format`), { statusCode: 400 });
-                throw error;
+            if (!/^[A-Za-z0-9_\-+/]{21,22}={0,2}$/.test(stateNonce)) {
+                throw Boom.badRequest('Oauth failed: invalid state format');
             }
 
             let [[, accountData]] = await redis.multi().get(`${REDIS_PREFIX}${request.query.state}`).del(`${REDIS_PREFIX}${request.query.state}`).exec();
@@ -3314,7 +3330,7 @@ Include your token in requests using one of these methods:
 
                     envelope: Joi.object({
                         from: Joi.string().email().allow('').example('sender@example.com'),
-                        to: Joi.array().items(Joi.string().email().required().example('recipient@example.com')).single()
+                        to: Joi.array().items(Joi.string().email().required().example('recipient@example.com')).single().label('SmtpEnvelopeTo')
                     })
                         .description(
                             "An optional object specifying the SMTP envelope used during email transmission. If not provided, the envelope is automatically derived from the email's message headers. This is useful when you need the envelope addresses to differ from those in the email headers."
@@ -3386,7 +3402,11 @@ Include your token in requests using one of these methods:
                     template: Joi.string().max(256).example('example').description('Stored template ID to load the email content from'),
 
                     render: Joi.object({
-                        format: Joi.string().valid('html', 'markdown').default('html').description('Markup language for HTML ("html" or "markdown")'),
+                        format: Joi.string()
+                            .valid('html', 'markdown')
+                            .default('html')
+                            .description('Markup language for HTML ("html" or "markdown")')
+                            .label('RenderFormat'),
                         params: Joi.object().label('RenderValues').description('An object of variables for the template renderer')
                     })
                         .allow(false)
@@ -3432,9 +3452,9 @@ Include your token in requests using one of these methods:
                                     }),
 
                                 contentType: Joi.string().lowercase().max(256).example('image/gif'),
-                                contentDisposition: Joi.string().lowercase().valid('inline', 'attachment'),
+                                contentDisposition: Joi.string().lowercase().valid('inline', 'attachment').label('AttachmentContentDisposition'),
                                 cid: Joi.string().max(256).example('unique-image-id@localhost').description('Content-ID value for embedded images'),
-                                encoding: Joi.string().valid('base64').default('base64'),
+                                encoding: Joi.string().valid('base64').default('base64').label('AttachmentEncoding'),
 
                                 reference: Joi.string()
                                     .base64({ paddingRequired: false, urlSafe: true })
@@ -3444,6 +3464,7 @@ Include your token in requests using one of these methods:
                                     .description(
                                         'References an existing attachment by its ID instead of providing new attachment content. If this field is set, the `content` field must not be included. If not set, the `content` field is required.'
                                     )
+                                    .label('AttachmentReference')
                             }).label('UploadAttachment')
                         )
                         .description('List of attachments')
@@ -3475,7 +3496,7 @@ Include your token in requests using one of these methods:
                         .example('Sent Mail')
                         .description("Upload sent message to this folder. By default the account's Sent Mail folder is used."),
 
-                    locale: Joi.string().empty('').max(100).example('fr').description('Optional locale'),
+                    locale: Joi.string().empty('').max(100).example('fr').description('Optional locale').label('MessageLocale'),
                     tz: Joi.string().empty('').max(100).example('Europe/Tallinn').description('Optional timezone'),
 
                     sendAt: Joi.date().iso().example('2021-07-08T07:06:34.336Z').description('Send message at specified time'),
@@ -3483,7 +3504,7 @@ Include your token in requests using one of these methods:
                         .integer()
                         .example(10)
                         .description('How many delivery attempts to make until message is considered as failed'),
-                    gateway: Joi.string().max(256).example('example').description('Optional SMTP gateway ID for message routing'),
+                    gateway: Joi.string().max(256).example('example').description('Optional SMTP gateway ID for message routing').label('MessageGateway'),
 
                     listId: Joi.string()
                         .hostname()
@@ -3505,11 +3526,13 @@ Include your token in requests using one of these methods:
                             .empty('')
                             .valid('headers', 'full')
                             .required()
-                            .description('Specifies if only headers or the entire body of the message should be included in the response (RET)'),
+                            .description('Specifies if only headers or the entire body of the message should be included in the response (RET)')
+                            .label('DsnReturn'),
                         notify: Joi.array()
                             .single()
                             .items(Joi.string().valid('never', 'success', 'failure', 'delay').label('NotifyEntry'))
-                            .description('Defines the conditions under which a DSN response should be sent'),
+                            .description('Defines the conditions under which a DSN response should be sent')
+                            .label('DsnNotify'),
                         recipient: Joi.string().trim().empty('').email().description('The email address the DSN should be sent (ORCPT)')
                     })
                         .description('Request DSN notifications')
@@ -3636,7 +3659,9 @@ Include your token in requests using one of these methods:
                                 skipped: Joi.object({
                                     reason: Joi.string().example('unsubscribe').description('Why this message was skipped'),
                                     listId: Joi.string().example('test-list')
-                                }).description('Info about skipped message. If this value is set, then the message was not sent')
+                                })
+                                    .description('Info about skipped message. If this value is set, then the message was not sent')
+                                    .label('SkippedMessageInfo')
                             })
                                 .label('BulkResponseEntry')
                                 .example({
@@ -3752,6 +3777,9 @@ Include your token in requests using one of these methods:
             }
 
             notify('settings', request.payload);
+            if ('httpProxyEnabled' in request.payload || 'httpProxyUrl' in request.payload) {
+                reloadHttpProxyAgent().catch(err => logger.error({ msg: 'Failed to reload HTTP proxy agent', err }));
+            }
             return { updated };
         },
         options: {
@@ -3856,13 +3884,27 @@ Include your token in requests using one of these methods:
                 failAction,
 
                 params: Joi.object({
-                    queue: Joi.string().empty('').trim().valid('notify', 'submit', 'documents').required().example('notify').description('Queue ID')
+                    queue: Joi.string()
+                        .empty('')
+                        .trim()
+                        .valid('notify', 'submit', 'documents')
+                        .required()
+                        .example('notify')
+                        .description('Queue ID')
+                        .label('QueueId')
                 })
             },
 
             response: {
                 schema: Joi.object({
-                    queue: Joi.string().empty('').trim().valid('notify', 'submit', 'documents').required().example('notify').description('Queue ID'),
+                    queue: Joi.string()
+                        .empty('')
+                        .trim()
+                        .valid('notify', 'submit', 'documents')
+                        .required()
+                        .example('notify')
+                        .description('Queue ID')
+                        .label('QueueIdResponse'),
                     jobs: Joi.object({
                         active: Joi.number().integer().example(123).description('Jobs that are currently being processed'),
                         delayed: Joi.number().integer().example(123).description('Jobs that are processed in the future'),
@@ -3946,7 +3988,14 @@ Include your token in requests using one of these methods:
                 failAction,
 
                 params: Joi.object({
-                    queue: Joi.string().empty('').trim().valid('notify', 'submit', 'documents').required().example('notify').description('Queue ID')
+                    queue: Joi.string()
+                        .empty('')
+                        .trim()
+                        .valid('notify', 'submit', 'documents')
+                        .required()
+                        .example('notify')
+                        .description('Queue ID')
+                        .label('QueueIdParam')
                 }),
 
                 payload: Joi.object({
@@ -3956,7 +4005,14 @@ Include your token in requests using one of these methods:
 
             response: {
                 schema: Joi.object({
-                    queue: Joi.string().empty('').trim().valid('notify', 'submit', 'documents').required().example('notify').description('Queue ID'),
+                    queue: Joi.string()
+                        .empty('')
+                        .trim()
+                        .valid('notify', 'submit', 'documents')
+                        .required()
+                        .example('notify')
+                        .description('Queue ID')
+                        .label('QueueIdPutResponse'),
                     paused: Joi.boolean().example(false).description('Is the queue paused or not')
                 }).label('SettingsPutQueueResponse'),
                 failAction: 'log'
@@ -4108,7 +4164,7 @@ Include your token in requests using one of these methods:
                 failAction,
 
                 payload: Joi.object({
-                    mailboxes: Joi.boolean().example(false).description('Include mailbox listing in response').default(false),
+                    mailboxes: Joi.boolean().example(false).description('Include mailbox listing in response').default(false).label('IncludeMailboxes'),
                     imap: Joi.object(imapSchema).allow(false).description('IMAP configuration').label('ImapConfiguration'),
                     smtp: Joi.object(smtpSchema).allow(false).description('SMTP configuration').label('SmtpConfiguration'),
                     proxy: settingsSchema.proxyUrl,
@@ -4127,7 +4183,7 @@ Include your token in requests using one of these methods:
                             .example('ERR_SSL_WRONG_VERSION_NUMBER')
                             .description('Error code. Only present if success=false')
                             .label('VerifyImapCode')
-                    }),
+                    }).label('VerifyImapResult'),
                     smtp: Joi.object({
                         success: Joi.boolean().example(true).description('Was SMTP account verified').label('VerifySmtpSuccess'),
                         error: Joi.string()
@@ -4138,7 +4194,7 @@ Include your token in requests using one of these methods:
                             .example('ERR_SSL_WRONG_VERSION_NUMBER')
                             .description('Error code. Only present if success=false')
                             .label('VerifySmtpCode')
-                    }),
+                    }).label('VerifySmtpResult'),
                     mailboxes: shortMailboxesSchema
                 }).label('VerifyAccountResponse'),
                 failAction: 'log'
@@ -4597,6 +4653,12 @@ Include your token in requests using one of these methods:
         MAX_PAYLOAD_TIMEOUT
     });
 
+    // setup export routes
+    await exportRoutes({
+        server,
+        CORS_CONFIG
+    });
+
     server.route({
         method: 'GET',
         path: '/v1/webhookRoutes',
@@ -4867,7 +4929,8 @@ Include your token in requests using one of these methods:
 
                                 clientId: Joi.string()
                                     .example('4f05f488-d858-4f2c-bd12-1039062612fe')
-                                    .description('Client or Application ID for 3-legged OAuth2 applications'),
+                                    .description('Client or Application ID for 3-legged OAuth2 applications')
+                                    .label('OAuth2AppListClientId'),
                                 clientSecret: Joi.string()
                                     .example('******')
                                     .description('Client secret for 3-legged OAuth2 applications. Actual value is not revealed.'),
@@ -4878,9 +4941,13 @@ Include your token in requests using one of these methods:
                                         allowRelative: false
                                     })
                                     .example('https://myservice.com/oauth')
-                                    .description('Redirect URL for 3-legged OAuth2 applications'),
+                                    .description('Redirect URL for 3-legged OAuth2 applications')
+                                    .label('OAuth2AppListRedirectUrl'),
 
-                                serviceClient: Joi.string().example('9103965568215821627203').description('Service client ID for 2-legged OAuth2 applications'),
+                                serviceClient: Joi.string()
+                                    .example('9103965568215821627203')
+                                    .description('Service client ID for 2-legged OAuth2 applications')
+                                    .label('OAuth2AppListServiceClient'),
 
                                 googleProjectId: googleProjectIdSchema,
                                 googleWorkspaceAccounts: googleWorkspaceAccountsSchema,
@@ -5002,7 +5069,8 @@ Include your token in requests using one of these methods:
 
                     clientId: Joi.string()
                         .example('4f05f488-d858-4f2c-bd12-1039062612fe')
-                        .description('Client or Application ID for 3-legged OAuth2 applications'),
+                        .description('Client or Application ID for 3-legged OAuth2 applications')
+                        .label('OAuth2AppGetClientId'),
                     clientSecret: Joi.string().example('******').description('Client secret for 3-legged OAuth2 applications. Actual value is not revealed.'),
                     authority: Joi.string().example('common').description('Authorization tenant value for Outlook OAuth2 applications'),
                     redirectUrl: Joi.string()
@@ -5011,7 +5079,8 @@ Include your token in requests using one of these methods:
                             allowRelative: false
                         })
                         .example('https://myservice.com/oauth')
-                        .description('Redirect URL for 3-legged OAuth2 applications'),
+                        .description('Redirect URL for 3-legged OAuth2 applications')
+                        .label('OAuth2AppGetRedirectUrl'),
 
                     googleProjectId: googleProjectIdSchema,
                     googleWorkspaceAccounts: googleWorkspaceAccountsSchema,
@@ -5023,7 +5092,10 @@ Include your token in requests using one of these methods:
                         .example('name@project-123.iam.gserviceaccount.com')
                         .description('Service Client Email for 2-legged OAuth2 applications'),
 
-                    serviceClient: Joi.string().example('9103965568215821627203').description('Service client ID for 2-legged OAuth2 applications'),
+                    serviceClient: Joi.string()
+                        .example('9103965568215821627203')
+                        .description('Service client ID for 2-legged OAuth2 applications')
+                        .label('OAuth2AppGetServiceClient'),
 
                     serviceKey: Joi.string()
                         .example('******')
@@ -5163,7 +5235,8 @@ Include your token in requests using one of these methods:
                         .allow('', null, false)
                         .max(256)
                         .example('52422112755-3uov8bjwlrullq122rdm6l8ui25ho7qf.apps.googleusercontent.com')
-                        .description('Client or Application ID for 3-legged OAuth2 applications'),
+                        .description('Client or Application ID for 3-legged OAuth2 applications')
+                        .label('UpdateOAuth2ClientId'),
 
                     clientSecret: Joi.string()
                         .trim()
@@ -5177,18 +5250,26 @@ Include your token in requests using one of these methods:
                         .base64({ paddingRequired: false, urlSafe: true })
                         .max(512)
                         .example('AAAAAQAACnA')
-                        .description('Cloud Pub/Sub app for Gmail API webhooks'),
+                        .description('Cloud Pub/Sub app for Gmail API webhooks')
+                        .label('UpdatePubSubAppId'),
 
-                    extraScopes: Joi.array().items(Joi.string().trim().max(255).example('User.Read')).description('OAuth2 Extra Scopes'),
+                    extraScopes: Joi.array()
+                        .items(Joi.string().trim().max(255).example('User.Read').label('UpdateExtraScopeEntry'))
+                        .description('OAuth2 Extra Scopes')
+                        .label('UpdateOAuth2ExtraScopes'),
 
-                    skipScopes: Joi.array().items(Joi.string().trim().max(255).example('SMTP.Send')).description('OAuth2 scopes to skip from the base set'),
+                    skipScopes: Joi.array()
+                        .items(Joi.string().trim().max(255).example('SMTP.Send').label('UpdateSkipScopeEntry'))
+                        .description('OAuth2 scopes to skip from the base set')
+                        .label('UpdateOAuth2SkipScopes'),
 
                     serviceClient: Joi.string()
                         .trim()
                         .allow('', null, false)
                         .max(256)
                         .example('7103296518315821565203')
-                        .description('Service client ID for 2-legged OAuth2 applications'),
+                        .description('Service client ID for 2-legged OAuth2 applications')
+                        .label('UpdateServiceClient'),
 
                     googleProjectId: googleProjectIdSchema,
                     googleWorkspaceAccounts: googleWorkspaceAccountsSchema,
@@ -5230,6 +5311,7 @@ Include your token in requests using one of these methods:
                         .uri({ scheme: ['http', 'https'], allowRelative: false })
                         .example('https://myservice.com/oauth')
                         .description('Redirect URL for 3-legged OAuth2 applications')
+                        .label('UpdateOAuth2RedirectUrl')
                 }).label('UpdateOAuthApp')
             },
 
@@ -5465,7 +5547,7 @@ Include your token in requests using one of these methods:
                         .default(false)
                         .example(true)
                         .description('Should connection use TLS. Usually true for port 465')
-                        .label('TLS'),
+                        .label('GatewayTlsOptions'),
 
                     lastError: lastErrorSchema.allow(null)
                 }).label('GatewayResponse'),
@@ -5542,7 +5624,7 @@ Include your token in requests using one of these methods:
                         .default(false)
                         .example(true)
                         .description('Should connection use TLS. Usually true for port 465')
-                        .label('TLS')
+                        .label('GatewayCreateTlsOptions')
                 }).label('CreateGateway')
             },
 
@@ -5628,7 +5710,7 @@ Include your token in requests using one of these methods:
                         .falsy('N', 'false', 0, '')
                         .example(true)
                         .description('Should connection use TLS. Usually true for port 465')
-                        .label('TLS')
+                        .label('GatewayUpdateTlsOptions')
                 }).label('UpdateGateway')
             },
 
@@ -5696,7 +5778,7 @@ Include your token in requests using one of these methods:
                 schema: Joi.object({
                     gateway: Joi.string().max(256).required().example('example').description('Gateway ID'),
                     deleted: Joi.boolean().truthy('Y', 'true', '1').falsy('N', 'false', 0).default(true).description('Was the gateway deleted')
-                }).label('DeleteRequestResponse'),
+                }).label('DeleteGatewayResponse'),
                 failAction: 'log'
             }
         }
@@ -5778,7 +5860,7 @@ Include your token in requests using one of these methods:
                 schema: Joi.object({
                     account: accountIdSchema.required(),
                     user: Joi.string().max(256).required().example('user@example.com').description('Username'),
-                    accessToken: Joi.string().max(256).required().example('aGVsbG8gd29ybGQ=').description('Access Token'),
+                    accessToken: Joi.string().max(256).required().example('aGVsbG8gd29ybGQ=').description('Access Token').label('OAuthAccessToken'),
                     provider: OAuth2ProviderSchema
                 }).label('AccountTokenResponse'),
                 failAction: 'log'
@@ -5848,7 +5930,7 @@ Include your token in requests using one of these methods:
                             }).label('SignatureResponseItem')
                         )
                         .label('SignatureEntries')
-                }).label('AccountTokenResponse'),
+                }).label('AccountSignaturesResponse'),
                 failAction: 'log'
             }
         }
@@ -5884,7 +5966,7 @@ Include your token in requests using one of these methods:
                         requestor: '@postalsys/emailengine-app'
                     }),
                     headers,
-                    dispatcher: retryAgent
+                    dispatcher: httpAgent.retry
                 });
 
                 if (!res.ok) {
@@ -5995,7 +6077,7 @@ ${now}`,
                 }),
 
                 payload: Joi.object({
-                    gateway: Joi.string().allow(false, null).empty('').max(256).example(false).description('Optional gateway ID')
+                    gateway: Joi.string().allow(false, null).empty('').max(256).example(false).description('Optional gateway ID').label('DeliveryTestGateway')
                 }).label('DeliveryStartRequest')
             },
 
@@ -6037,7 +6119,7 @@ ${now}`,
                 let res = await fetchCmd(`${SMTP_TEST_HOST}/test-address/${request.params.deliveryTest}`, {
                     method: 'get',
                     headers,
-                    dispatcher: retryAgent
+                    dispatcher: httpAgent.retry
                 });
 
                 if (!res.ok) {
@@ -6141,14 +6223,15 @@ ${now}`,
             response: {
                 schema: Joi.object({
                     success: Joi.boolean().example(true).description('Was the test completed').label('ResponseDeliveryCheckSuccess'),
-                    dkim: Joi.object().unknown().description('DKIM results'),
-                    spf: Joi.object().unknown().description('SPF results'),
-                    dmarc: Joi.object().unknown().description('DMARC results'),
-                    bimi: Joi.object().unknown().description('BIMI results'),
-                    arc: Joi.object().unknown().description('ARC results'),
+                    dkim: Joi.object().unknown().description('DKIM results').label('DkimResults'),
+                    spf: Joi.object().unknown().description('SPF results').label('SpfResults'),
+                    dmarc: Joi.object().unknown().description('DMARC results').label('DmarcResults'),
+                    bimi: Joi.object().unknown().description('BIMI results').label('BimiResults'),
+                    arc: Joi.object().unknown().description('ARC results').label('ArcResults'),
                     mainSig: Joi.object()
                         .unknown()
                         .description('Primary DKIM signature. `status.aligned` should be set, otherwise DKIM check should not be considered as passed.')
+                        .label('MainSignature')
                 }).label('DeliveryCheckResponse'),
                 failAction: 'log'
             }

package/workers/documents.js

@@ -241,7 +241,7 @@ const documentsWorker = new Worker(
                         return;
                     }
 
-                    let deleteResult = null;
+                    let deleteResult;
 
                     let filterQuery = {
                         bool: {
@@ -612,7 +612,7 @@ const documentsWorker = new Worker(
                         return;
                     }
 
-                    let deleteResult = null;
+                    let deleteResult;
 
                     try {
                         let messageIdHeader;
@@ -912,6 +912,7 @@ if( ctx._source.bounces != null) {
     Object.assign(
         {
             concurrency: 1,
+            lockDuration: 2 * 60 * 1000, // 2 minutes for ES operations
             maxStalledCount: 5,
             stalledInterval: 60 * 1000
         },