emailengine-app 2.61.5 → 2.62.1

package/lib/api-routes/message-routes.js

@@ -147,7 +147,8 @@ async function init(args) {
                         .lowercase()
                         .valid('html', 'plain', '*')
                         .example('*')
-                        .description('Which text content to return, use * for all. By default text content is not returned.'),
+                        .description('Which text content to return, use * for all. By default text content is not returned.')
+                        .label('MessageTextType'),
 
                     webSafeHtml: Joi.boolean()
                         .truthy('Y', 'true', '1')
@@ -310,9 +311,9 @@ async function init(args) {
                                     }),
 
                                 contentType: Joi.string().lowercase().max(256).example('image/gif'),
-                                contentDisposition: Joi.string().lowercase().valid('inline', 'attachment'),
+                                contentDisposition: Joi.string().lowercase().valid('inline', 'attachment').label('MsgUploadContentDisposition'),
                                 cid: Joi.string().max(256).example('unique-image-id@localhost').description('Content-ID value for embedded images'),
-                                encoding: Joi.string().valid('base64').default('base64'),
+                                encoding: Joi.string().valid('base64').default('base64').label('MsgUploadEncoding'),
 
                                 reference: Joi.string()
                                     .base64({ paddingRequired: false, urlSafe: true })
@@ -322,6 +323,7 @@ async function init(args) {
                                     .description(
                                         'References an existing attachment by its ID instead of providing new attachment content. If this field is set, the `content` field must not be included. If not set, the `content` field is required.'
                                     )
+                                    .label('MsgUploadReference')
                             }).label('UploadAttachment')
                         )
                         .description('List of attachments')
@@ -430,15 +432,15 @@ async function init(args) {
             response: {
                 schema: Joi.object({
                     flags: Joi.object({
-                        add: Joi.array().items(Joi.string()).example(['\\Seen', '\\Flagged']),
-                        delete: Joi.array().items(Joi.string()).example(['\\Draft']),
-                        set: Joi.array().items(Joi.string()).example(['\\Seen'])
-                    }).label('FlagResponse'),
+                        add: Joi.array().items(Joi.string().label('FlagEntry')).example(['\\Seen', '\\Flagged']).label('FlagAddList'),
+                        delete: Joi.array().items(Joi.string().label('FlagEntry')).example(['\\Draft']).label('FlagDeleteList'),
+                        set: Joi.array().items(Joi.string().label('FlagEntry')).example(['\\Seen']).label('FlagSetList')
+                    }).label('FlagUpdateResponse'),
                     labels: Joi.object({
-                        add: Joi.array().items(Joi.string()).example(['Label1', 'Label2']),
-                        delete: Joi.array().items(Joi.string()).example(['Label3']),
-                        set: Joi.array().items(Joi.string()).example(['Label1'])
-                    }).label('FlagResponse')
+                        add: Joi.array().items(Joi.string().label('LabelEntry')).example(['Label1', 'Label2']).label('LabelAddList'),
+                        delete: Joi.array().items(Joi.string().label('LabelEntry')).example(['Label3']).label('LabelDeleteList'),
+                        set: Joi.array().items(Joi.string().label('LabelEntry')).example(['Label1']).label('LabelSetList')
+                    }).label('LabelUpdateResponse')
                 }).label('MessageUpdateResponse'),
                 failAction: 'log'
             }
@@ -510,16 +512,16 @@ async function init(args) {
             response: {
                 schema: Joi.object({
                     flags: Joi.object({
-                        add: Joi.array().items(Joi.string()).example(['\\Seen', '\\Flagged']),
-                        delete: Joi.array().items(Joi.string()).example(['\\Draft']),
-                        set: Joi.array().items(Joi.string()).example(['\\Seen'])
-                    }).label('FlagResponse'),
+                        add: Joi.array().items(Joi.string().label('BulkFlagEntry')).example(['\\Seen', '\\Flagged']).label('BulkFlagAddList'),
+                        delete: Joi.array().items(Joi.string().label('BulkFlagEntry')).example(['\\Draft']).label('BulkFlagDeleteList'),
+                        set: Joi.array().items(Joi.string().label('BulkFlagEntry')).example(['\\Seen']).label('BulkFlagSetList')
+                    }).label('BulkFlagUpdateResponse'),
                     labels: Joi.object({
-                        add: Joi.array().items(Joi.string()).example(['Label1', 'Label2']),
-                        delete: Joi.array().items(Joi.string()).example(['Label3']),
-                        set: Joi.array().items(Joi.string()).example(['Label1'])
-                    }).label('FlagResponse')
-                }).label('MessageUpdateResponse'),
+                        add: Joi.array().items(Joi.string().label('BulkLabelEntry')).example(['Label1', 'Label2']).label('BulkLabelAddList'),
+                        delete: Joi.array().items(Joi.string().label('BulkLabelEntry')).example(['Label3']).label('BulkLabelDeleteList'),
+                        set: Joi.array().items(Joi.string().label('BulkLabelEntry')).example(['Label1']).label('BulkLabelSetList')
+                    }).label('BulkLabelUpdateResponse')
+                }).label('BulkMessageUpdateResponse'),
                 failAction: 'log'
             }
         }
@@ -759,7 +761,9 @@ async function init(args) {
                     moved: Joi.object({
                         destination: Joi.string().required().example('Trash').description('Trash folder path').label('TrashPath'),
                         message: Joi.string().required().example('AAAAAwAAAWg').description('Message ID in Trash').label('TrashMessageId')
-                    }).description('Present if message was moved to Trash')
+                    })
+                        .description('Present if message was moved to Trash')
+                        .label('MessageMovedToTrash')
                 }).label('MessageDeleteResponse'),
                 failAction: 'log'
             }
@@ -1083,7 +1087,7 @@ async function init(args) {
                         .unknown()
                         .meta({ swaggerHidden: true })
                 })
-                    .label('SearchQuery')
+                    .label('MessageSearchPayload')
                     .example({
                         search: {
                             unseen: true,
@@ -1281,7 +1285,8 @@ async function init(args) {
                         .valid('html', 'plain', '*')
                         .default('*')
                         .example('*')
-                        .description('Which text content to return, use * for all. By default all contents are returned.'),
+                        .description('Which text content to return, use * for all. By default all contents are returned.')
+                        .label('TextSearchTextType'),
                     documentStore: documentStoreSchema.default(false)
                 }),
 

package/lib/api-routes/template-routes.js

@@ -93,7 +93,11 @@ async function init(args) {
                         .example('Something about the template')
                         .description('Optional description of the template')
                         .label('TemplateDescription'),
-                    format: Joi.string().valid('html', 'markdown').default('html').description('Markup language for HTML ("html" or "markdown")'),
+                    format: Joi.string()
+                        .valid('html', 'markdown')
+                        .default('html')
+                        .description('Markup language for HTML ("html" or "markdown")')
+                        .label('TemplateFormat'),
                     content: Joi.object({
                         subject: templateSchemas.subject,
                         text: templateSchemas.text,
@@ -176,7 +180,12 @@ async function init(args) {
                         .example('Something about the template')
                         .description('Optional description of the template')
                         .label('TemplateDescription'),
-                    format: Joi.string().empty('').valid('html', 'markdown').default('html').description('Markup language for HTML ("html" or "markdown")'),
+                    format: Joi.string()
+                        .empty('')
+                        .valid('html', 'markdown')
+                        .default('html')
+                        .description('Markup language for HTML ("html" or "markdown")')
+                        .label('TemplateUpdateFormat'),
                     content: Joi.object({
                         subject: templateSchemas.subject,
                         text: templateSchemas.text,
@@ -277,7 +286,11 @@ async function init(args) {
                                     .example('Something about the template')
                                     .description('Optional description of the template')
                                     .label('TemplateDescription'),
-                                format: Joi.string().valid('html', 'markdown').default('html').description('Markup language for HTML ("html" or "markdown")'),
+                                format: Joi.string()
+                                    .valid('html', 'markdown')
+                                    .default('html')
+                                    .description('Markup language for HTML ("html" or "markdown")')
+                                    .label('TemplateListFormat'),
                                 created: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this template was created'),
                                 updated: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this template was last updated')
                             }).label('AccountTemplate')
@@ -345,7 +358,11 @@ async function init(args) {
                         .example('Something about the template')
                         .description('Optional description of the template')
                         .label('TemplateDescription'),
-                    format: Joi.string().valid('html', 'markdown').default('html').description('Markup language for HTML ("html" or "markdown")'),
+                    format: Joi.string()
+                        .valid('html', 'markdown')
+                        .default('html')
+                        .description('Markup language for HTML ("html" or "markdown")')
+                        .label('TemplateResponseFormat'),
                     created: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this template was created'),
                     updated: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this template was last updated'),
                     content: Joi.object({
@@ -353,7 +370,11 @@ async function init(args) {
                         text: templateSchemas.text,
                         html: templateSchemas.html,
                         previewText: templateSchemas.previewText,
-                        format: Joi.string().valid('html', 'markdown').default('html').description('Markup language for HTML ("html" or "markdown")')
+                        format: Joi.string()
+                            .valid('html', 'markdown')
+                            .default('html')
+                            .description('Markup language for HTML ("html" or "markdown")')
+                            .label('TemplateContentFormat')
                     }).label('RequestTemplateContent')
                 }).label('AccountTemplateResponse'),
                 failAction: 'log'
@@ -411,7 +432,7 @@ async function init(args) {
                     deleted: Joi.boolean().truthy('Y', 'true', '1').falsy('N', 'false', 0).default(true).description('Was the template deleted'),
                     account: accountIdSchema.required(),
                     id: Joi.string().max(256).required().example('AAABgS-UcAYAAAABAA').description('Template ID')
-                }).label('DeleteTemplateRequestResponse'),
+                }).label('DeleteTemplateResponse'),
                 failAction: 'log'
             }
         }
@@ -480,7 +501,7 @@ async function init(args) {
                 schema: Joi.object({
                     deleted: Joi.boolean().truthy('Y', 'true', '1').falsy('N', 'false', 0).default(true).description('Were the templates flushed'),
                     account: accountIdSchema.required()
-                }).label('DeleteTemplateRequestResponse'),
+                }).label('FlushTemplatesResponse'),
                 failAction: 'log'
             }
         }

package/lib/arf-detect.js

@@ -27,7 +27,7 @@ const arfDetect = async messageInfo => {
 
     let returnPath;
 
-    for (let attachment of messageInfo.attachments) {
+    for (let attachment of messageInfo.attachments || []) {
         switch (attachment.contentType.toLowerCase()) {
             case 'message/feedback-report': {
                 // found a feedback report

package/lib/autodetect-imap-settings.js

@@ -20,7 +20,7 @@ const parseXml = util.promisify((xml, cb) => {
 });
 
 const { fetch: fetchCmd } = require('undici');
-const { retryAgent } = require('./tools');
+const { httpAgent } = require('./tools');
 
 const RESOLV_TIMEOUT = 5 * 1000;
 
@@ -191,7 +191,7 @@ async function resolveUsingMozillaDirectory(email, domain, source) {
         headers: {
             'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`
         },
-        dispatcher: retryAgent
+        dispatcher: httpAgent.retry
     });
 
     if (!res.ok) {
@@ -216,7 +216,7 @@ async function resolveUsingAutoconfig(email, domain, source) {
         headers: {
             'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`
         },
-        dispatcher: retryAgent
+        dispatcher: httpAgent.retry
     });
 
     if (!res.ok) {
@@ -241,7 +241,7 @@ async function resolveUsingWellKnown(email, domain, source) {
         headers: {
             'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage})`
         },
-        dispatcher: retryAgent
+        dispatcher: httpAgent.retry
     });
 
     if (!res.ok) {
@@ -640,7 +640,7 @@ async function resolveUsingAutodiscovery(email, domain, source) {
             'Content-type': 'application/xml'
         },
         body,
-        dispatcher: retryAgent
+        dispatcher: httpAgent.retry
     });
 
     if (!res.ok) {

package/lib/consts.js

@@ -87,6 +87,22 @@ module.exports = {
     LIST_SUBSCRIBE_NOTIFY: 'listSubscribe',
     LIST_SUBSCRIBE_DESCRIPTION: 'Recipient Subscribed - A recipient re-subscribed to a list',
 
+    EXPORT_COMPLETED_NOTIFY: 'exportCompleted',
+    EXPORT_COMPLETED_DESCRIPTION: 'Export Completed - A bulk message export has completed successfully',
+
+    EXPORT_FAILED_NOTIFY: 'exportFailed',
+    EXPORT_FAILED_DESCRIPTION: 'Export Failed - A bulk message export has failed',
+
+    // Export defaults
+    DEFAULT_EXPORT_MAX_AGE: 24 * 60 * 60 * 1000, // 24 hours
+    DEFAULT_EXPORT_MAX_SIZE: 10 * 1024 * 1024 * 1024, // 10GB
+    DEFAULT_EXPORT_MAX_MESSAGES: 500000,
+    DEFAULT_EXPORT_MAX_CONCURRENT: 2,
+    DEFAULT_EXPORT_MAX_GLOBAL_CONCURRENT: 8,
+    DEFAULT_EXPORT_MAX_MESSAGE_SIZE: 50 * 1024 * 1024, // 50MB
+    DEFAULT_GMAIL_EXPORT_BATCH_SIZE: 10,
+    DEFAULT_OUTLOOK_EXPORT_BATCH_SIZE: 20,
+
     MAX_DAYS_STATS: 7,
 
     DEFAULT_MAX_LOG_LINES: 10000,

package/lib/db.js

@@ -107,12 +107,14 @@ const reqisQueue = new Redis(REDIS_CONF);
 
 module.exports.queueConf = {
     connection: reqisQueue,
+    sharedConnection: true, // Prevent BullMQ from closing shared connection
     prefix: `${REDIS_PREFIX}bull`
 };
 
 const notifyQueue = new Queue('notify', module.exports.queueConf);
 const submitQueue = new Queue('submit', module.exports.queueConf);
 const documentsQueue = new Queue('documents', module.exports.queueConf);
+const exportQueue = new Queue('export', module.exports.queueConf);
 
 const zExpungeScript = fs.readFileSync(pathlib.join(__dirname, '/lua/z-expunge.lua'), 'utf-8');
 const zSetScript = fs.readFileSync(pathlib.join(__dirname, 'lua/z-set.lua'), 'utf-8');
@@ -215,6 +217,7 @@ module.exports.redis = redis;
 module.exports.notifyQueue = notifyQueue;
 module.exports.submitQueue = submitQueue;
 module.exports.documentsQueue = documentsQueue;
+module.exports.exportQueue = exportQueue;
 
 // do not set up the flow producer by default
 module.exports.getFlowProducer = () => new FlowProducer(module.exports.queueConf /*, Redis*/);

package/lib/email-client/base-client.js

@@ -1984,7 +1984,7 @@ class BaseClient {
         // Store message in Redis
         await this.redis.hsetBuffer(`${REDIS_PREFIX}iaq:${this.account}`, queueId, msgEntry);
 
-        let queueKeep = (await settings.get('queueKeep')) || true;
+        let queueKeep = (await settings.get('queueKeep')) ?? true;
 
         // Configure delivery retry settings
         let defaultDeliveryAttempts = await settings.get('deliveryAttempts');
@@ -2007,14 +2007,16 @@ class BaseClient {
 
         // Configure queue job options
         let queueName = 'queued';
+        let retention = typeof queueKeep === 'number' ? { age: 24 * 3600, count: queueKeep } : queueKeep;
         let jobOpts = {
             jobId: queueId,
-            removeOnComplete: queueKeep,
-            removeOnFail: queueKeep,
+            removeOnComplete: retention,
+            removeOnFail: retention,
             attempts: typeof deliveryAttempts === 'number' ? deliveryAttempts : defaultDeliveryAttempts,
             backoff: {
                 type: 'exponential',
-                delay: 5000
+                delay: 5000,
+                jitter: 0.2 // 20% randomization to prevent thundering herd
             }
         };