emailengine-app 2.68.1 → 2.70.0

package/lib/api-routes/webhook-route-routes.js

@@ -0,0 +1,181 @@
+'use strict';
+
+const Joi = require('joi');
+const { webhooks: Webhooks } = require('../webhooks');
+const { failAction } = require('../tools');
+const { handleError, throwNotFound } = require('./route-helpers');
+const { settingsSchema, errorResponses } = require('../schemas');
+
+const webhookErrorFlagSchema = Joi.object({
+    message: Joi.string().example('Request failed with status 500').description('Error message from the last failed delivery')
+})
+    .unknown()
+    .allow(null)
+    .description('Information about the last webhook delivery error. Null if no errors have been registered')
+    .label('WebhookRouteErrorFlag');
+
+const webhookCustomHeadersSchema = settingsSchema.webhooksCustomHeaders
+    .description('Custom HTTP headers added to webhook requests for this route')
+    .label('WebhookRouteCustomHeaders');
+
+async function init(args) {
+    const { server, CORS_CONFIG } = args;
+
+    server.route({
+        method: 'GET',
+        path: '/v1/webhookRoutes',
+
+        async handler(request) {
+            try {
+                return await Webhooks.list(request.query.page, request.query.pageSize);
+            } catch (err) {
+                handleError(request, err);
+            }
+        },
+
+        options: {
+            description: 'List webhook routes',
+            notes: 'List custom webhook routes',
+            tags: ['api', 'Webhooks'],
+
+            plugins: {
+                'hapi-swagger': {
+                    responses: errorResponses(400, 401, 403, 429, 500)
+                }
+            },
+
+            auth: {
+                strategy: 'api-token',
+                mode: 'required'
+            },
+            cors: CORS_CONFIG,
+
+            validate: {
+                options: {
+                    stripUnknown: false,
+                    abortEarly: false,
+                    convert: true
+                },
+                failAction,
+
+                query: Joi.object({
+                    page: Joi.number()
+                        .integer()
+                        .min(0)
+                        .max(1024 * 1024)
+                        .default(0)
+                        .example(0)
+                        .description('Page number (zero indexed, so use 0 for first page)')
+                        .label('PageNumber'),
+                    pageSize: Joi.number().integer().min(1).max(1000).default(20).example(20).description('How many entries per page').label('PageSize')
+                }).label('WebhookRoutesListRequest')
+            },
+
+            response: {
+                schema: Joi.object({
+                    total: Joi.number().integer().example(120).description('How many matching entries').label('TotalNumber'),
+                    page: Joi.number().integer().example(0).description('Current page (0-based index)').label('PageNumber'),
+                    pages: Joi.number().integer().example(24).description('Total page count').label('PagesNumber'),
+
+                    webhooks: Joi.array()
+                        .items(
+                            Joi.object({
+                                id: Joi.string().max(256).required().example('AAABgS-UcAYAAAABAA').description('Webhook ID'),
+                                name: Joi.string().max(256).example('Send to Slack').description('Name of the route').label('WebhookRouteName').required(),
+                                description: Joi.string()
+                                    .allow('')
+                                    .max(1024)
+                                    .example('Something about the route')
+                                    .description('Optional description of the webhook route')
+                                    .label('WebhookRouteDescription'),
+                                created: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this route was created'),
+                                updated: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this route was last updated'),
+                                enabled: Joi.boolean().example(true).description('Is the route enabled').label('WebhookRouteEnabled'),
+                                targetUrl: settingsSchema.webhooks,
+                                tcount: Joi.number().integer().example(123).description('How many times this route has been applied'),
+                                webhookErrorFlag: webhookErrorFlagSchema,
+                                customHeaders: webhookCustomHeadersSchema
+                            }).label('WebhookRoutesListEntry')
+                        )
+                        .label('WebhookRoutesList')
+                }).label('WebhookRoutesListResponse'),
+                failAction: 'log'
+            }
+        }
+    });
+
+    server.route({
+        method: 'GET',
+        path: '/v1/webhookRoutes/webhookRoute/{webhookRoute}',
+
+        async handler(request) {
+            try {
+                let webhookRouteData = await Webhooks.get(request.params.webhookRoute);
+                if (!webhookRouteData) {
+                    throwNotFound();
+                }
+                return webhookRouteData;
+            } catch (err) {
+                handleError(request, err);
+            }
+        },
+
+        options: {
+            description: 'Get webhook route information',
+            notes: 'Retrieve webhook route content and information',
+            tags: ['api', 'Webhooks'],
+
+            plugins: {
+                'hapi-swagger': {
+                    responses: errorResponses(400, 401, 403, 404, 429, 500)
+                }
+            },
+
+            auth: {
+                strategy: 'api-token',
+                mode: 'required'
+            },
+            cors: CORS_CONFIG,
+
+            validate: {
+                options: {
+                    stripUnknown: false,
+                    abortEarly: false,
+                    convert: true
+                },
+                failAction,
+                params: Joi.object({
+                    webhookRoute: Joi.string().max(256).required().example('example').description('Webhook ID')
+                }).label('GetWebhookRouteRequest')
+            },
+
+            response: {
+                schema: Joi.object({
+                    id: Joi.string().max(256).required().example('AAABgS-UcAYAAAABAA').description('Webhook ID'),
+                    name: Joi.string().max(256).example('Send to Slack').description('Name of the route').label('WebhookRouteName').required(),
+                    description: Joi.string()
+                        .allow('')
+                        .max(1024)
+                        .example('Something about the route')
+                        .description('Optional description of the webhook route')
+                        .label('WebhookRouteDescription'),
+                    created: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this route was created'),
+                    updated: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this route was last updated'),
+                    enabled: Joi.boolean().example(true).description('Is the route enabled').label('WebhookRouteEnabled'),
+                    targetUrl: settingsSchema.webhooks,
+                    tcount: Joi.number().integer().example(123).description('How many times this route has been applied'),
+                    v: Joi.number().integer().example(1).description('Internal version counter, increased on every update'),
+                    webhookErrorFlag: webhookErrorFlagSchema,
+                    customHeaders: webhookCustomHeadersSchema,
+                    content: Joi.object({
+                        fn: Joi.string().allow(null).example('return true;').description('Filter function. Null if not set'),
+                        map: Joi.string().allow(null).example('payload.ts = Date.now(); return payload;').description('Mapping function. Null if not set')
+                    }).label('WebhookRouteContent')
+                }).label('WebhookRouteResponse'),
+                failAction: 'log'
+            }
+        }
+    });
+}
+
+module.exports = init;

package/lib/autodetect-imap-settings.js

@@ -709,8 +709,6 @@ async function resolveUsingAutodiscovery(email, domain, source) {
         });
     });
 
-    resp = resp.filter(entry => entry.account);
-
     return { imap, smtp, _source: source || 'autodiscover' };
 }
 

package/lib/consts.js

@@ -107,6 +107,11 @@ module.exports = {
 
     DEFAULT_MAX_LOG_LINES: 10000,
 
+    // Shared Sentry instance run by the EmailEngine developers. Used as the error
+    // reporting target when error reporting is enabled without a custom DSN. A DSN
+    // is a write-only credential, it can only be used to submit events.
+    COMMUNITY_SENTRY_DSN: 'https://bdd958f3e813a488904b0f254e0bb8a8@sentry.emailengine.dev/3',
+
     PDKDF2_ITERATIONS: 600000,
     PDKDF2_SALT_SIZE: 16,
     PDKDF2_DIGEST: 'sha256', // 'sha512', 'sha256' or 'sha1'

package/lib/email-client/base-client.js

@@ -240,6 +240,33 @@ class BaseClient {
         return rid;
     }
 
+    /**
+     * Normalizes an IMAP-style flag update request (add/delete/set) into plain add/delete
+     * flag lists. If `set` is present then it replaces the state of the flags the backend
+     * supports and add/delete are ignored, matching the IMAP backend behavior.
+     * @param {Object} [flags] - Flag update request ({ add, delete, set })
+     * @param {string[]} supportedFlags - Flags the backend can represent
+     * @returns {Object} Normalized flag lists ({ add: string[], delete: string[] })
+     */
+    normalizeFlagUpdates(flags, supportedFlags) {
+        if (flags?.set) {
+            // If set exists then ignore add/delete calls
+            let setFlags = [].concat(flags.set);
+            let addFlags = [];
+            let deleteFlags = [];
+            for (let flag of supportedFlags) {
+                if (setFlags.includes(flag)) {
+                    addFlags.push(flag);
+                } else {
+                    deleteFlags.push(flag);
+                }
+            }
+            return { add: addFlags, delete: deleteFlags };
+        }
+
+        return { add: [].concat(flags?.add || []), delete: [].concat(flags?.delete || []) };
+    }
+
     // Redis key generators for different data types
 
     getAccountKey() {
@@ -721,7 +748,7 @@ class BaseClient {
         }
 
         // use existing response
-        switch (idempotencyData.status) {
+        switch (idempotencyData?.status) {
             case 'completed':
                 // Return cached result
                 idempotencyData.returnValue = Object.assign({}, idempotencyData.result, {
@@ -3194,11 +3221,6 @@ class BaseClient {
     async handleSubmitError(err, context) {
         const { smtpSettings, networkRouting, gatewayData, gatewayObject, data, jobData, queueId, envelope } = context;
 
-        // Handle permanent failures
-        if (err.responseCode >= 500 && jobData.opts?.attempts <= jobData.attemptsMade) {
-            jobData.nextAttempt = false;
-        }
-
         // Build SMTP status from error
         const smtpStatus = SmtpErrorBuilder.buildStatus(err, smtpSettings, networkRouting);
 

package/lib/email-client/gmail-client.js

@@ -46,7 +46,7 @@ const SYSTEM_LABELS = {
 
 // User-friendly names for system labels
 const SYSTEM_NAMES = {
-    SENT: 'Sent ',
+    SENT: 'Sent',
     INBOX: 'Inbox',
     TRASH: 'Trash',
     DRAFT: 'Drafts',
@@ -63,6 +63,12 @@ for (let label of Object.keys(SYSTEM_LABELS)) {
     SYSTEM_LABELS_REV[SYSTEM_LABELS[label]] = label;
 }
 
+// Convert an IMAP special-use label (e.g. '\Important') to its Gmail system label ID, pass every
+// other value through as-is
+function toGmailLabelId(label) {
+    return SYSTEM_LABELS_REV.hasOwnProperty(label) ? SYSTEM_LABELS_REV[label] : label;
+}
+
 // Timing constants for Gmail Pub/Sub watch
 const RENEW_WATCH_TTL = 60 * 60 * 1000; // 1h - how often to check if watch needs renewal
 const MIN_WATCH_TTL = 24 * 3600 * 1000; // 1day - minimum time before renewing watch
@@ -753,7 +759,7 @@ class GmailClient extends BaseClient {
             // NB! Might throw if using unsupported search terms
             const preparedQuery = this.prepareQuery(query.search);
             if (preparedQuery) {
-                requestQuery.q = this.prepareQuery(query.search);
+                requestQuery.q = preparedQuery;
             }
         }
 
@@ -875,16 +881,13 @@ class GmailClient extends BaseClient {
 
         let sourceLabel = path && path !== '\\All' ? await this.getLabel(path) : null;
         if (path && path !== '\\All' && !sourceLabel) {
-            let error = new Error('Unknown path');
-            error.info = {
-                response: `Mailbox doesn't exist: ${path}`
-            };
-            error.code = 'NotFound';
-            error.statusCode = 404;
-            throw error;
+            throw this.unknownPathError(path);
         }
 
-        // Add TRASH label and remove source label
+        // Add TRASH label and remove source label. When deleting from the Trash folder itself both
+        // labels are TRASH - updateMessages resolves the add/remove conflict in favor of the add,
+        // keeping the request valid (Gmail API accounts only hold the gmail.modify scope, so a
+        // permanent batchDelete is not possible)
         let labelsUpdate = { add: 'TRASH' };
         if (sourceLabel) {
             labelsUpdate.delete = sourceLabel.id;
@@ -911,51 +914,28 @@ class GmailClient extends BaseClient {
         await this.prepare();
         updates = updates || {};
 
-        let addLabelIds = new Set();
-        let removeLabelIds = new Set();
-
         // Convert IMAP flags to Gmail labels
-        if (updates.flags) {
-            let labelUpdates = [];
-
-            for (let flag of [].concat(updates.flags.add || [])) {
-                labelUpdates.push(this.flagToLabel(flag));
-            }
-
-            for (let flag of [].concat(updates.flags.delete || [])) {
-                labelUpdates.push(this.flagToLabel(flag, true));
-            }
-
-            labelUpdates
-                .filter(label => label)
-                .forEach(label => {
-                    if (label.add) {
-                        addLabelIds.add(label.add);
-                    }
-                    if (label.remove) {
-                        removeLabelIds.add(label.remove);
-                    }
-                });
-        }
+        let { addLabelIds, removeLabelIds } = this.flagsToLabelIds(updates.flags);
 
         // Process direct label updates
         if (updates.labels) {
+            this.assertLabelSetSupported(updates.labels);
+
             for (let label of [].concat(updates.labels.add || [])) {
-                // Convert IMAP special-use to Gmail label
-                if (SYSTEM_LABELS_REV.hasOwnProperty(label)) {
-                    label = SYSTEM_LABELS_REV[label];
-                }
-                addLabelIds.add(label);
+                addLabelIds.add(toGmailLabelId(label));
             }
 
             for (let label of [].concat(updates.labels.delete || [])) {
-                if (SYSTEM_LABELS_REV.hasOwnProperty(label)) {
-                    label = SYSTEM_LABELS_REV[label];
-                }
-                removeLabelIds.add(label);
+                removeLabelIds.add(toGmailLabelId(label));
             }
         }
 
+        // Gmail rejects modify calls where the same label is both added and removed (deleting or
+        // moving a message within its current folder re-adds the source label) - the add wins
+        for (let label of addLabelIds) {
+            removeLabelIds.delete(label);
+        }
+
         if (!addLabelIds.size && !removeLabelIds.size) {
             return updates;
         }
@@ -1024,6 +1004,17 @@ class GmailClient extends BaseClient {
         await this.prepare();
         updates = updates || {};
 
+        // Reject unsupported operations before any API calls are made
+        if (updates.labels) {
+            this.assertLabelSetSupported(updates.labels);
+        }
+
+        path = [].concat(path || []).join('/');
+
+        if (path && path !== '\\All' && !(await this.getLabel(path))) {
+            throw this.unknownPathError(path);
+        }
+
         // Step 1. Resolve matching messages
         let messages = [];
         let cursor;
@@ -1043,8 +1034,8 @@ class GmailClient extends BaseClient {
                 { metadataOnly: true }
             );
 
-            if (messageListResult?.messages) {
-                messages = messages.concat(messageListResult?.messages);
+            if (messageListResult.messages) {
+                messages = messages.concat(messageListResult.messages);
                 if (messages.length >= maxMessages) {
                     messages = messages.slice(0, maxMessages);
                     break;
@@ -1064,43 +1055,25 @@ class GmailClient extends BaseClient {
             return updates;
         }
 
-        let addLabelIds = new Set();
-        let removeLabelIds = new Set();
-
         // Convert flags to label operations
-        if (updates.flags) {
-            let labelUpdates = [];
-
-            for (let flag of [].concat(updates.flags.add || [])) {
-                labelUpdates.push(this.flagToLabel(flag));
-            }
-
-            for (let flag of [].concat(updates.flags.delete || [])) {
-                labelUpdates.push(this.flagToLabel(flag, true));
-            }
-
-            labelUpdates
-                .filter(label => label)
-                .forEach(label => {
-                    if (label.add) {
-                        addLabelIds.add(label.add);
-                    }
-                    if (label.remove) {
-                        removeLabelIds.add(label.remove);
-                    }
-                });
-        }
+        let { addLabelIds, removeLabelIds } = this.flagsToLabelIds(updates.flags);
 
         if (updates.labels) {
             for (let label of [].concat(updates.labels.add || [])) {
-                addLabelIds.add(label);
+                addLabelIds.add(toGmailLabelId(label));
             }
 
             for (let label of [].concat(updates.labels.delete || [])) {
-                removeLabelIds.add(label);
+                removeLabelIds.add(toGmailLabelId(label));
             }
         }
 
+        // Gmail rejects batchModify calls where the same label is both added and removed (deleting
+        // or moving messages within their current folder re-adds the source label) - the add wins
+        for (let label of addLabelIds) {
+            removeLabelIds.delete(label);
+        }
+
         if (!addLabelIds.size && !removeLabelIds.size) {
             return { flags: {}, labels: {} };
         }
@@ -1141,13 +1114,7 @@ class GmailClient extends BaseClient {
 
         let label = await this.getLabel(path);
         if (!label) {
-            let error = new Error('Unknown path');
-            error.info = {
-                response: `Mailbox doesn't exist: ${path}`
-            };
-            error.code = 'NotFound';
-            error.statusCode = 404;
-            throw error;
+            throw this.unknownPathError(path);
         }
         let labelsUpdate = { add: [label.id] };
 
@@ -1155,13 +1122,7 @@ class GmailClient extends BaseClient {
 
         let sourceLabel = sourcePath ? await this.getLabel(sourcePath) : null;
         if (sourcePath && !sourceLabel) {
-            let error = new Error('Unknown path');
-            error.info = {
-                response: `Mailbox doesn't exist: ${sourcePath}`
-            };
-            error.code = 'NotFound';
-            error.statusCode = 404;
-            throw error;
+            throw this.unknownPathError(sourcePath);
         }
 
         if (sourceLabel) {
@@ -1190,24 +1151,12 @@ class GmailClient extends BaseClient {
 
         let targetLabel = await this.getLabel(path);
         if (!targetLabel) {
-            let error = new Error('Unknown path');
-            error.info = {
-                response: `Mailbox doesn't exist: ${path}`
-            };
-            error.code = 'NotFound';
-            error.statusCode = 404;
-            throw error;
+            throw this.unknownPathError(path);
         }
 
         let sourceLabel = source ? await this.getLabel(source) : null;
         if (source && !sourceLabel) {
-            let error = new Error('Unknown path');
-            error.info = {
-                response: `Mailbox doesn't exist: ${source}`
-            };
-            error.code = 'NotFound';
-            error.statusCode = 404;
-            throw error;
+            throw this.unknownPathError(source);
         }
 
         let labelsUpdate = { add: targetLabel.id };
@@ -1251,7 +1200,7 @@ class GmailClient extends BaseClient {
 
         const contentResponse = {
             headers: {
-                'content-type': attachmentData.mimeType || 'application/octet-stream',
+                'content-type': attachmentData.contentType || 'application/octet-stream',
                 'content-disposition': 'attachment' + filenameParam
             },
             contentType: attachmentData.contentType,
@@ -1421,7 +1370,7 @@ class GmailClient extends BaseClient {
 
         // Map part IDs to content types
         textParts[0].forEach(p => {
-            bodyParts.set(p, 'text');
+            bodyParts.set(p, 'plain');
         });
 
         textParts[1].forEach(p => {
@@ -1481,13 +1430,7 @@ class GmailClient extends BaseClient {
 
         let targetLabel = await this.getLabel(path);
         if (!targetLabel) {
-            let error = new Error('Unknown path');
-            error.info = {
-                response: `Mailbox doesn't exist: ${path}`
-            };
-            error.code = 'NotFound';
-            error.statusCode = 404;
-            throw error;
+            throw this.unknownPathError(path);
         }
 
         // Generate raw message
@@ -2400,7 +2343,7 @@ class GmailClient extends BaseClient {
             flags.push('\\Flagged');
         }
 
-        if (messageData.labelIds?.includes('DRAFTS')) {
+        if (messageData.labelIds?.includes('DRAFT')) {
             flags.push('\\Draft');
         }
 
@@ -2609,6 +2552,21 @@ class GmailClient extends BaseClient {
         return term;
     }
 
+    /**
+     * Builds the canonical 404 error for a mailbox path that does not match any Gmail label
+     * @param {string} path - Requested mailbox path
+     * @returns {Error} Error object with code and statusCode set
+     */
+    unknownPathError(path) {
+        let error = new Error('Unknown path');
+        error.info = {
+            response: `Mailbox doesn't exist: ${path}`
+        };
+        error.code = 'NotFound';
+        error.statusCode = 404;
+        return error;
+    }
+
     /**
      * Converts IMAP flags to Gmail label operations
      * @param {string} flag - IMAP flag
@@ -2625,6 +2583,55 @@ class GmailClient extends BaseClient {
         }
     }
 
+    /**
+     * Converts an IMAP-style flag update request (add/delete/set) into Gmail label ID sets.
+     * Set precedence is handled by BaseClient#normalizeFlagUpdates.
+     * @param {Object} [flagUpdates] - Flag update request ({ add, delete, set })
+     * @returns {Object} Label ID sets ({ addLabelIds: Set, removeLabelIds: Set })
+     */
+    flagsToLabelIds(flagUpdates) {
+        let addLabelIds = new Set();
+        let removeLabelIds = new Set();
+
+        let normalized = this.normalizeFlagUpdates(flagUpdates, ['\\Seen', '\\Flagged']);
+
+        let labelUpdates = [];
+
+        for (let flag of normalized.add) {
+            labelUpdates.push(this.flagToLabel(flag));
+        }
+
+        for (let flag of normalized.delete) {
+            labelUpdates.push(this.flagToLabel(flag, true));
+        }
+
+        labelUpdates
+            .filter(label => label)
+            .forEach(label => {
+                if (label.add) {
+                    addLabelIds.add(label.add);
+                }
+                if (label.remove) {
+                    removeLabelIds.add(label.remove);
+                }
+            });
+
+        return { addLabelIds, removeLabelIds };
+    }
+
+    /**
+     * Throws if the update request uses `labels.set`, which Gmail API accounts do not support
+     * @param {Object} labels - Label update request
+     */
+    assertLabelSetSupported(labels) {
+        if (labels.set) {
+            let error = new Error('Replacing the full label set is not supported for Gmail API accounts');
+            error.code = 'UnsupportedOperation';
+            error.statusCode = 400;
+            throw error;
+        }
+    }
+
     /**
      * Converts IMAP SEARCH query to Gmail API query
      * @param {Object} search - IMAP search object
@@ -2700,6 +2707,20 @@ class GmailClient extends BaseClient {
             queryParts.push(search.gmailRaw);
         }
 
+        // Label filters - "has" matches messages with the label, "not" excludes them
+        if (search.labels && typeof search.labels === 'object') {
+            for (let label of [].concat(search.labels.has || [])) {
+                if (label) {
+                    queryParts.push(`label:${this.formatSearchTerm(label)}`);
+                }
+            }
+            for (let label of [].concat(search.labels.not || [])) {
+                if (label) {
+                    queryParts.push(`-label:${this.formatSearchTerm(label)}`);
+                }
+            }
+        }
+
         // body search
         if (search.body && typeof search.body === 'string') {
             queryParts.push(`${this.formatSearchTerm(search.body)}`);
@@ -2791,7 +2812,7 @@ class GmailClient extends BaseClient {
                         case 'STARRED':
                             changes.flags[addedProp].push('\\Flagged');
                             break;
-                        case 'DRAFTS':
+                        case 'DRAFT':
                             changes.flags[addedProp].push('\\Draft');
                             break;
                         default: