emailengine-app 2.69.0 → 2.71.0

package/lib/account.js

@@ -20,6 +20,7 @@ const { deepStrictEqual, strictEqual } = require('assert');
 const { encrypt, decrypt } = require('./encrypt');
 const { oauth2Apps, LEGACY_KEYS, isApiBasedApp } = require('./oauth2-apps');
 const settings = require('./settings');
+const { isDocumentStoreEnabled, documentStoreFeatureEnabled } = require('./document-store');
 const redisScanDelete = require('./redis-scan-delete');
 const { customAlphabet } = require('nanoid');
 const Lock = require('ioredfour');
@@ -945,7 +946,7 @@ class Account {
             throw error;
         }
 
-        let state = false;
+        let state;
         if (result[0][1] && result[0][1].account) {
             // existing user
             state = 'existing';
@@ -1048,28 +1049,33 @@ class Account {
             };
         }
 
-        try {
-            let queueKeep = (await settings.get('queueKeep')) || true;
-            let serviceUrl = (await settings.get('serviceUrl')) || null;
+        // Only notify the documents queue when the deprecated Document Store feature is enabled.
+        // When it is off the documents worker is not running, so an enqueued job would never be
+        // consumed and would pile up in Redis.
+        if (documentStoreFeatureEnabled) {
+            try {
+                let queueKeep = (await settings.get('queueKeep')) || true;
+                let serviceUrl = (await settings.get('serviceUrl')) || null;
 
-            let payload = {
-                serviceUrl,
-                account: this.account,
-                date: new Date().toISOString(),
-                event: ACCOUNT_DELETED_NOTIFY
-            };
+                let payload = {
+                    serviceUrl,
+                    account: this.account,
+                    date: new Date().toISOString(),
+                    event: ACCOUNT_DELETED_NOTIFY
+                };
 
-            await this.documentsQueue.add(ACCOUNT_DELETED_NOTIFY, payload, {
-                removeOnComplete: queueKeep,
-                removeOnFail: queueKeep,
-                attempts: 10,
-                backoff: {
-                    type: 'exponential',
-                    delay: 5000
-                }
-            });
-        } catch (err) {
-            this.logger.error({ msg: 'Failed to add entry to documents queue', err });
+                await this.documentsQueue.add(ACCOUNT_DELETED_NOTIFY, payload, {
+                    removeOnComplete: queueKeep,
+                    removeOnFail: queueKeep,
+                    attempts: 10,
+                    backoff: {
+                        type: 'exponential',
+                        delay: 5000
+                    }
+                });
+            } catch (err) {
+                this.logger.error({ msg: 'Failed to add entry to documents queue', err });
+            }
         }
 
         await this.call({
@@ -1267,16 +1273,7 @@ class Account {
             } catch (err) {
                 // should not happen
                 if (logger.notifyError) {
-                    logger.notifyError(err, event => {
-                        if (this.account) {
-                            event.setUser(this.account);
-                        }
-
-                        event.addMetadata('ee', {
-                            path,
-                            mailboxListing: typeof mailboxListing
-                        });
-                    });
+                    logger.notifyError(err, { user: this.account, meta: { path, mailboxListing: typeof mailboxListing } });
                 }
 
                 let message = 'Failed to process stored mailbox listing';
@@ -1290,22 +1287,44 @@ class Account {
         return mailboxes;
     }
 
+    // Worker backends return false for entities they can not find (e.g. an unknown mailbox path
+    // or message ID on IMAP). Convert such results into a 404 error for the API.
+    assertFound(result, message, code) {
+        if (!result) {
+            let error = Boom.boomify(new Error(message), { statusCode: 404 });
+            error.output.payload.code = code;
+            throw error;
+        }
+        return result;
+    }
+
+    assertMessageFound(result) {
+        return this.assertFound(result, 'Requested message was not found', 'MessageNotFound');
+    }
+
+    assertFolderFound(result) {
+        return this.assertFound(result, 'Requested mailbox folder was not found', 'FolderNotFound');
+    }
+
     async updateMessage(message, updates) {
         await this.loadAccountData(this.account, true);
 
-        return await this.call({
+        let result = await this.call({
             cmd: 'updateMessage',
             account: this.account,
             message,
             updates,
             timeout: this.timeout
         });
+
+        // IMAP backend returns false for unknown messages and folders
+        return this.assertMessageFound(result);
     }
 
     async updateMessages(path, search, updates) {
         await this.loadAccountData(this.account, true);
 
-        return await this.call({
+        let result = await this.call({
             cmd: 'updateMessages',
             account: this.account,
             path,
@@ -1313,6 +1332,9 @@ class Account {
             updates,
             timeout: this.timeout
         });
+
+        // IMAP backend returns false for unknown folders
+        return this.assertFolderFound(result);
     }
 
     async listMailboxes(query) {
@@ -1335,7 +1357,8 @@ class Account {
 
     async moveMessage(message, target, options) {
         await this.loadAccountData(this.account, true);
-        return await this.call({
+
+        let result = await this.call({
             cmd: 'moveMessage',
             account: this.account,
             message,
@@ -1343,11 +1366,15 @@ class Account {
             options,
             timeout: this.timeout
         });
+
+        // IMAP backend returns false for unknown messages and folders
+        return this.assertMessageFound(result);
     }
 
     async moveMessages(source, search, target) {
         await this.loadAccountData(this.account, true);
-        return await this.call({
+
+        let result = await this.call({
             cmd: 'moveMessages',
             account: this.account,
             source,
@@ -1355,24 +1382,30 @@ class Account {
             target,
             timeout: this.timeout
         });
+
+        // IMAP backend returns false for unknown folders
+        return this.assertFolderFound(result);
     }
 
     async deleteMessage(message, force) {
         await this.loadAccountData(this.account, true);
 
-        return await this.call({
+        let result = await this.call({
             cmd: 'deleteMessage',
             account: this.account,
             message,
             force,
             timeout: this.timeout
         });
+
+        // IMAP backend returns false for unknown messages and folders
+        return this.assertMessageFound(result);
     }
 
     async deleteMessages(path, search, force) {
         await this.loadAccountData(this.account, true);
 
-        return await this.call({
+        let result = await this.call({
             cmd: 'deleteMessages',
             account: this.account,
             path,
@@ -1380,6 +1413,9 @@ class Account {
             force,
             timeout: this.timeout
         });
+
+        // IMAP backend returns false for unknown folders
+        return this.assertFolderFound(result);
     }
 
     async getQuota() {
@@ -1428,7 +1464,7 @@ class Account {
     }
 
     async getText(text, options) {
-        if (options.documentStore && (await settings.get('documentStoreEnabled'))) {
+        if (options.documentStore && (await isDocumentStoreEnabled())) {
             await this.loadAccountData(this.account, false);
 
             const { index, client } = this.esClient;
@@ -1447,13 +1483,7 @@ class Account {
                 results: getResult && getResult._source ? 1 : 0
             });
 
-            if (!getResult || !getResult._source) {
-                let message = 'Requested message was not found';
-                let error = Boom.boomify(new Error(message), { statusCode: 404 });
-                throw error;
-            }
-
-            let messageData = getResult._source;
+            let messageData = this.assertMessageFound(getResult && getResult._source);
             let response = {};
 
             response.hasMore = false;
@@ -1473,13 +1503,16 @@ class Account {
 
         await this.loadAccountData(this.account, true);
 
-        return await this.call({
+        let textData = await this.call({
             cmd: 'getText',
             account: this.account,
             text,
             options,
             timeout: this.timeout
         });
+
+        // IMAP backend returns false for unknown messages and folders
+        return this.assertMessageFound(textData);
     }
 
     async getMessage(message, options) {
@@ -1489,7 +1522,7 @@ class Account {
             options.preProcessHtml = true;
         }
 
-        if (options.documentStore && (await settings.get('documentStoreEnabled'))) {
+        if (options.documentStore && (await isDocumentStoreEnabled())) {
             await this.loadAccountData(this.account, false);
 
             const { index, client } = this.esClient;
@@ -1521,13 +1554,7 @@ class Account {
                 results: getResult && getResult._source ? 1 : 0
             });
 
-            if (!getResult || !getResult._source) {
-                let message = 'Requested message was not found';
-                let error = Boom.boomify(new Error(message), { statusCode: 404 });
-                throw error;
-            }
-
-            let messageData = getResult._source;
+            let messageData = this.assertMessageFound(getResult && getResult._source);
 
             // restore headers and text object as per the API response
             let headersObj = {};
@@ -1632,6 +1659,9 @@ class Account {
 
             if (options.markAsSeen && (!messageData.flags || !messageData.flags.includes('\\Seen'))) {
                 // mark message as seen
+                if (!messageData.flags) {
+                    messageData.flags = [];
+                }
                 messageData.flags.push('\\Seen');
                 // do not wait until the update is completed, return immediately
                 this.updateMessage(message, { flags: { add: ['\\Seen'] } }).catch(err => {
@@ -1661,13 +1691,7 @@ class Account {
             timeout: this.timeout
         });
 
-        if (!messageData) {
-            let message = 'Requested message was not found';
-            let error = Boom.boomify(new Error(message), { statusCode: 404 });
-            throw error;
-        }
-
-        return messageData;
+        return this.assertMessageFound(messageData);
     }
 
     async getMessages(messageIds, options) {
@@ -1682,7 +1706,7 @@ class Account {
     }
 
     async listMessages(query) {
-        if (query.documentStore && (await settings.get('documentStoreEnabled'))) {
+        if (query.documentStore && (await isDocumentStoreEnabled())) {
             await this.loadAccountData(this.account, false);
 
             const { index, client } = this.esClient;
@@ -1801,7 +1825,7 @@ class Account {
 
         await this.loadAccountData(this.account, true);
 
-        return await this.call(
+        let listing = await this.call(
             Object.assign(
                 {
                     cmd: 'listMessages',
@@ -1811,11 +1835,14 @@ class Account {
                 { timeout: this.timeout }
             )
         );
+
+        // IMAP and Gmail backends return false for unknown folders
+        return this.assertFolderFound(listing);
     }
 
     async searchMessages(query, searchOpts) {
         searchOpts = searchOpts || {};
-        if (query.documentStore && (await settings.get('documentStoreEnabled'))) {
+        if (query.documentStore && (await isDocumentStoreEnabled())) {
             if (!searchOpts.unified) {
                 await this.loadAccountData(this.account, false);
             }
@@ -2086,11 +2113,11 @@ class Account {
             let sizeMatch = {};
 
             if (query.search.larger) {
-                dateMatch.gte = query.search.larger;
+                sizeMatch.gte = query.search.larger;
             }
 
             if (query.search.smaller) {
-                dateMatch.lte = query.search.smaller;
+                sizeMatch.lte = query.search.smaller;
             }
 
             if (Object.keys(sizeMatch).length) {
@@ -2198,7 +2225,7 @@ class Account {
 
         await this.loadAccountData(this.account, true);
 
-        return await this.call(
+        let listing = await this.call(
             Object.assign(
                 {
                     cmd: 'listMessages',
@@ -2208,6 +2235,9 @@ class Account {
                 { timeout: this.timeout }
             )
         );
+
+        // IMAP and Gmail backends return false for unknown folders
+        return this.assertFolderFound(listing);
     }
 
     async uploadMessage(data) {
@@ -2322,7 +2352,7 @@ class Account {
             }
         } catch (err) {
             this.logger.error({ msg: 'Failed to get lock', lockKey, err });
-            if (Boom.isBoom) {
+            if (Boom.isBoom(err)) {
                 throw err;
             }
             let error = Boom.boomify(new Error('Failed to get flush lock, try again later'), { statusCode: 500 });
@@ -2377,11 +2407,13 @@ class Account {
                 // scan and delete keys
                 await redisScanDelete(this.redis, this.logger, `${REDIS_PREFIX}iam:${this.account}:*`);
 
-                if (await settings.get('documentStoreEnabled')) {
+                if (await isDocumentStoreEnabled()) {
                     // Flush ElasticSearch index for this account
                     const { index, client } = this.esClient;
                     if (!client) {
-                        return;
+                        // Account data in Redis was already flushed, only the index cleanup was skipped
+                        this.logger.error({ msg: 'Document store is enabled but the ElasticSearch client is not available', action: 'flush' });
+                        return true;
                     }
 
                     let deleteResult = {};
@@ -2592,7 +2624,7 @@ class Account {
                 account: accountData.account,
                 user: authData.user,
                 accessToken: authData.accessToken,
-                provider: accountData.oauth2.auth.provider,
+                provider: accountData.oauth2.provider,
                 registeredScopes: accountData.oauth2.scope,
                 cached: false
             };
@@ -2626,7 +2658,7 @@ class Account {
             account: accountData.account,
             user: accountData.oauth2.auth.user,
             accessToken,
-            provider: accountData.oauth2.auth.provider,
+            provider: accountData.oauth2.provider,
             registeredScopes: accountData.oauth2.scope,
             expires:
                 accountData.oauth2.expires && typeof accountData.oauth2.expires.toISOString === 'function'