emailengine-app 1.14.0

package/lib/consts.js

@@ -0,0 +1,22 @@
+'use strict';
+
+module.exports = {
+    MESSAGE_NEW_NOTIFY: 'messageNew',
+    MESSAGE_DELETED_NOTIFY: 'messageDeleted',
+    MESSAGE_UPDATED_NOTIFY: 'messageUpdated',
+
+    EMAIL_SENT_NOTIFY: 'messageSent',
+    EMAIL_FAILED_NOTIFY: 'messageFailed',
+    EMAIL_BOUNCE_NOTIFY: 'messageBounce',
+
+    MAILBOX_RESET_NOTIFY: 'mailboxReset',
+    MAILBOX_DELETED_NOTIFY: 'mailboxDeleted',
+    MAILBOX_NEW_NOTIFY: 'mailboxNew',
+
+    AUTH_ERROR_NOTIFY: 'authenticationError',
+    AUTH_SUCCESS_NOTIFY: 'authenticationSuccess',
+
+    CONNECT_ERROR_NOTIFY: 'connectError',
+
+    MAX_DAYS_STATS: 7
+};

package/lib/db.js

@@ -0,0 +1,72 @@
+'use strict';
+
+const fs = require('fs');
+const config = require('wild-config');
+const pathlib = require('path');
+
+config.dbs = config.dbs || {
+    redis: 'redis://127.0.0.1:6379/8'
+};
+
+const Queue = require('bull');
+const Redis = require('ioredis');
+
+const REDIS_CONF = process.env.EENGINE_REDIS || config.dbs.redis;
+
+const redis = new Redis(REDIS_CONF);
+
+const notifyQueue = new Queue('notify', typeof REDIS_CONF === 'object' ? { redis: REDIS_CONF } : REDIS_CONF);
+const submitQueue = new Queue('submit', typeof REDIS_CONF === 'object' ? { redis: REDIS_CONF } : REDIS_CONF);
+
+const zExpungeScript = fs.readFileSync(pathlib.join(__dirname, '/lua/z-expunge.lua'), 'utf-8');
+const zSetScript = fs.readFileSync(pathlib.join(__dirname, 'lua/z-set.lua'), 'utf-8');
+const zGetScript = fs.readFileSync(pathlib.join(__dirname, 'lua/z-get.lua'), 'utf-8');
+const zGetByUidScript = fs.readFileSync(pathlib.join(__dirname, 'lua/z-get-by-uid.lua'), 'utf-8');
+const zGetMailboxIdScript = fs.readFileSync(pathlib.join(__dirname, 'lua/z-get-mailbox-id.lua'), 'utf-8');
+const zGetMailboxPathScript = fs.readFileSync(pathlib.join(__dirname, 'lua/z-get-mailbox-path.lua'), 'utf-8');
+const sListAccountsScript = fs.readFileSync(pathlib.join(__dirname, 'lua/s-list-accounts.lua'), 'utf-8');
+const zPushScript = fs.readFileSync(pathlib.join(__dirname, 'lua/z-push.lua'), 'utf-8');
+
+redis.defineCommand('zExpunge', {
+    numberOfKeys: 2,
+    lua: zExpungeScript
+});
+
+redis.defineCommand('zSet', {
+    numberOfKeys: 1,
+    lua: zSetScript
+});
+
+redis.defineCommand('zGet', {
+    numberOfKeys: 1,
+    lua: zGetScript
+});
+
+redis.defineCommand('zPush', {
+    numberOfKeys: 1,
+    lua: zPushScript
+});
+
+redis.defineCommand('zGetByUid', {
+    numberOfKeys: 1,
+    lua: zGetByUidScript
+});
+
+redis.defineCommand('zGetMailboxId', {
+    numberOfKeys: 2,
+    lua: zGetMailboxIdScript
+});
+
+redis.defineCommand('zGetMailboxPath', {
+    numberOfKeys: 1,
+    lua: zGetMailboxPathScript
+});
+
+redis.defineCommand('sListAccounts', {
+    numberOfKeys: 1,
+    lua: sListAccountsScript
+});
+
+module.exports.redis = redis;
+module.exports.notifyQueue = notifyQueue;
+module.exports.submitQueue = submitQueue;

package/lib/encrypt.js

@@ -0,0 +1,100 @@
+'use strict';
+
+const crypto = require('crypto');
+const assert = require('assert');
+
+function parseEncryptedData(encryptedData) {
+    encryptedData = (encryptedData || '').toString();
+    if (!encryptedData || encryptedData.charAt(0) !== '$') {
+        // cleartext
+        return {
+            format: 'cleartext',
+            data: encryptedData
+        };
+    }
+
+    let [, format, cipher, authTag, iv, salt, encryptedText] = encryptedData.split('$');
+    if (!format || !cipher || !authTag || !iv || !encryptedText) {
+        // assume cleartext if format is not matching
+        return {
+            format: 'cleartext',
+            data: encryptedData
+        };
+    }
+
+    authTag = Buffer.from(authTag, 'hex');
+    iv = Buffer.from(iv, 'hex');
+    salt = Buffer.from(salt, 'hex');
+    encryptedText = Buffer.from(encryptedText, 'hex');
+
+    return {
+        format,
+        cipher,
+        authTag,
+        iv,
+        salt,
+        data: encryptedText
+    };
+}
+
+function getKeyFromPassword(password, salt, keyLen) {
+    return crypto.scryptSync(password, salt, keyLen);
+}
+
+function decrypt(encryptedData, secret) {
+    const decryptData = parseEncryptedData(encryptedData);
+
+    switch (decryptData.format) {
+        case 'cleartext':
+            return encryptedData;
+
+        case 'wd01':
+            try {
+                assert.strictEqual(decryptData.authTag.length, 16, 'Invalid auth tag length');
+                assert.strictEqual(decryptData.iv.length, 12, 'Invalid iv length');
+                assert.strictEqual(decryptData.salt.length, 16, 'Invalid salt length');
+
+                // convert password to 32B key
+                const key = getKeyFromPassword(secret, decryptData.salt, 32);
+
+                const decipher = crypto.createDecipheriv(decryptData.cipher, key, decryptData.iv, { authTagLength: decryptData.authTag.length });
+                decipher.setAuthTag(decryptData.authTag);
+
+                // try to decipher
+                return Buffer.concat([decipher.update(decryptData.data), decipher.final()]).toString('utf-8');
+            } catch (E) {
+                let err = new Error('Failed to decrypt data. ' + E.message);
+                err.responseCode = 500;
+                err.code = 'InternalConfigError';
+                throw err;
+            }
+
+        default: {
+            // assume cleartext
+            return encryptedData;
+        }
+    }
+}
+
+function encrypt(cleartext, secret) {
+    if (!secret) {
+        return cleartext;
+    }
+
+    const iv = crypto.randomBytes(12);
+    const salt = crypto.randomBytes(16);
+
+    const key = getKeyFromPassword(secret, salt, 32);
+
+    const format = 'wd01';
+    const algo = 'aes-256-gcm';
+
+    const cipher = crypto.createCipheriv(algo, key, iv, { authTagLength: 16 });
+    const encryptedText = Buffer.concat([cipher.update(cleartext), cipher.final()]);
+
+    const authTag = cipher.getAuthTag();
+
+    return ['', format, algo].concat([authTag, iv, salt, encryptedText].map(buf => buf.toString('hex'))).join('$');
+}
+
+module.exports = { encrypt, decrypt, parseEncryptedData };

package/lib/enum-message-flags.js

@@ -0,0 +1,6 @@
+'use strict';
+
+// Ordering matters. Up to 8 flags can be listed here.
+// Once added, do not remove or it messes up bit positions.
+
+module.exports = ['\\Seen', '\\Answered', '\\Flagged', '\\Deleted', '\\Draft', '$Forwarded', '$Junk', '$NotJunk'];

package/lib/get-raw-email.js

@@ -0,0 +1,292 @@
+'use strict';
+
+const MailComposer = require('nodemailer/lib/mail-composer');
+const { Splitter, Joiner } = require('mailsplit');
+const Transform = require('stream').Transform;
+const mimeFuncs = require('nodemailer/lib/mime-funcs');
+const MimeNode = require('nodemailer/lib/mime-node');
+const addressparser = require('nodemailer/lib/addressparser');
+const uuid = require('uuid');
+const os = require('os');
+const libmime = require('libmime');
+
+class HeadersRewriter extends Transform {
+    constructor(headersCb) {
+        let options = {
+            readableObjectMode: true,
+            writableObjectMode: true
+        };
+        super(options);
+        this.headersCb = headersCb;
+    }
+
+    _transform(obj, encoding, callback) {
+        if (obj.type !== 'node' || !obj.root || !typeof this.headersCb) {
+            this.push(obj);
+            return callback();
+        }
+        this.headersCb(obj.headers)
+            .then(() => {
+                this.push(obj);
+                return callback();
+            })
+            .catch(err => callback(err));
+    }
+
+    _flush(callback) {
+        return callback();
+    }
+}
+
+async function processMessage(data) {
+    let raw = typeof data.raw === 'string' ? Buffer.from(data.raw, 'base64') : data.raw;
+    let hasBcc = false;
+
+    let messageId, envelope;
+    let sendAt = data.sendAt;
+
+    const splitter = new Splitter();
+    const joiner = new Joiner();
+    const headersRewriter = new HeadersRewriter(async headers => {
+        let mn = new MimeNode();
+
+        let sendAtVal = headers.getFirst('x-ee-send-at');
+        headers.remove('x-ee-send-at');
+        if (sendAtVal) {
+            if (!isNaN(sendAtVal)) {
+                sendAtVal = Number(sendAtVal);
+            }
+            sendAtVal = new Date(sendAtVal);
+            if (sendAtVal.toString() !== 'Invalid Date') {
+                sendAt = sendAtVal;
+            }
+        }
+
+        for (let addrKey of ['from', 'to', 'cc', 'bcc']) {
+            if (!(addrKey in data)) {
+                continue;
+            }
+
+            let addrValue = [].concat(data[addrKey] || []);
+            if (!addrValue.length) {
+                // remove existing
+                headers.remove(addrKey);
+                continue;
+            }
+
+            // update or add new
+            let headerLine = mn._encodeHeaderValue(addrKey, addrValue);
+            if (headers.hasHeader(addrKey)) {
+                headers.update(
+                    addrKey.replace(/^./, c => c.toUpperCase()),
+                    headerLine
+                );
+            } else {
+                // push to bottom
+                headers.add(
+                    addrKey.replace(/^./, c => c.toUpperCase()),
+                    headerLine,
+                    Infinity
+                );
+            }
+        }
+
+        hasBcc = headers.hasHeader('bcc');
+
+        let addresses = {
+            from: headers
+                .get('from')
+                .flatMap(line => addressparser(line, { flatten: true }))
+                .filter(a => a)
+                .map(addr => {
+                    if (addr.name) {
+                        try {
+                            addr.name = libmime.decodeWords(addr.name);
+                        } catch (err) {
+                            // ignore
+                        }
+                    }
+                    return addr;
+                })
+                .find(addr => addr && addr.address),
+            to: headers
+                .get('to')
+                .flatMap(line => addressparser(line, { flatten: true }))
+                .filter(a => a)
+                .map(addr => {
+                    if (addr.name) {
+                        try {
+                            addr.name = libmime.decodeWords(addr.name);
+                        } catch (err) {
+                            // ignore
+                        }
+                    }
+                    return addr;
+                }),
+            cc: headers
+                .get('cc')
+                .flatMap(line => addressparser(line, { flatten: true }))
+                .filter(a => a)
+                .map(addr => {
+                    if (addr.name) {
+                        try {
+                            addr.name = libmime.decodeWords(addr.name);
+                        } catch (err) {
+                            // ignore
+                        }
+                    }
+                    return addr;
+                }),
+            bcc: headers
+                .get('bcc')
+                .flatMap(line => addressparser(line, { flatten: true }))
+                .filter(a => a)
+                .map(addr => {
+                    if (addr.name) {
+                        try {
+                            addr.name = libmime.decodeWords(addr.name);
+                        } catch (err) {
+                            // ignore
+                        }
+                    }
+                    return addr;
+                })
+        };
+
+        envelope = data.envelope || {
+            from: (addresses.from && addresses.from.address) || '',
+            to: Array.from(
+                new Set(
+                    []
+                        .concat(addresses.to)
+                        .concat(addresses.cc)
+                        .concat(addresses.bcc)
+                        .map(addr => addr.address)
+                )
+            )
+        };
+
+        envelope.from = [].concat(envelope.from || []).shift() || '';
+        envelope.to = [].concat(envelope.to || []);
+
+        // generate default message-id
+        if (!data.messageId && !headers.hasHeader('message-id')) {
+            let fromDomain = envelope.from ? envelope.from.split('@').pop().toLowerCase() : os.hostname();
+            messageId = `<${uuid.v4()}@${fromDomain}>`;
+            headers.add('Message-ID', messageId, Infinity);
+        }
+
+        if (data.messageId) {
+            messageId = (data.messageId || '').toString().trim().replace(/^<*/, '<').replace(/>*$/, '>');
+            headers.update('Message-ID', messageId);
+        }
+
+        if (!messageId) {
+            messageId = headers.getFirst('message-id');
+        }
+
+        // make sure that there is a Date header
+        if (!headers.hasHeader('date')) {
+            let dateVal = new Date().toUTCString().replace(/GMT/, '+0000');
+            headers.add('Date', dateVal, Infinity);
+        }
+        // ... and MIME-Version
+        if (!headers.hasHeader('mime-version')) {
+            headers.add('MIME-Version', '1.0', Infinity);
+        }
+
+        if (data.subject) {
+            headers.update('Subject', mimeFuncs.encodeWords(data.subject, 'Q', 64, true));
+        }
+
+        if (data.headers) {
+            for (let key of Object.keys(headers)) {
+                let casedKey = key.replace(/^.|-./g, c => c.toUpperCase());
+                switch (key) {
+                    case 'in-reply-to':
+                    case 'references':
+                        headers.update(casedKey, headers[key]);
+                        break;
+                    default:
+                        headers.add(casedKey, headers[key]);
+                        break;
+                }
+            }
+        }
+    });
+
+    let message = await new Promise((resolve, reject) => {
+        let chunks = [];
+        let chunklen = 0;
+
+        joiner.on('readable', () => {
+            let chunk;
+            while ((chunk = joiner.read()) !== null) {
+                chunks.push(chunk);
+                chunklen += chunk.length;
+            }
+        });
+
+        joiner.once('end', () => {
+            resolve(Buffer.concat(chunks, chunklen));
+        });
+
+        joiner.once('error', err => reject(err));
+        splitter.pipe(headersRewriter).pipe(joiner);
+        splitter.end(raw);
+    });
+
+    return { raw: message, hasBcc, messageId, envelope, sendAt };
+}
+
+async function removeBcc(raw) {
+    const splitter = new Splitter();
+    const joiner = new Joiner();
+    const headersRewriter = new HeadersRewriter(async headers => {
+        headers.remove('bcc');
+    });
+
+    return new Promise((resolve, reject) => {
+        let chunks = [];
+        let chunklen = 0;
+
+        joiner.on('readable', () => {
+            let chunk;
+            while ((chunk = joiner.read()) !== null) {
+                chunks.push(chunk);
+                chunklen += chunk.length;
+            }
+        });
+
+        joiner.once('end', () => {
+            resolve(Buffer.concat(chunks, chunklen));
+        });
+
+        joiner.once('error', err => reject(err));
+        splitter.pipe(headersRewriter).pipe(joiner);
+        splitter.end(raw);
+    });
+}
+
+async function getRawEmail(data) {
+    if (data.raw) {
+        return await processMessage(data);
+    }
+
+    let sendAt = data.sendAt;
+    delete data.sendAt;
+
+    const mail = new MailComposer(data);
+    let compiled = mail.compile();
+    compiled.keepBcc = true;
+
+    let messageId = compiled.messageId();
+
+    let email = await compiled.build();
+
+    let bcc = [].concat(data.bcc || []);
+
+    return { raw: email, hasBcc: !!bcc.length, messageId, envelope: compiled.getEnvelope(), sendAt };
+}
+
+module.exports = { getRawEmail, removeBcc };

package/lib/get-secret.js

@@ -0,0 +1,83 @@
+'use strict';
+
+const config = require('wild-config');
+const vault = require('node-vault');
+const logger = require('./logger');
+
+config.service = config.service || {};
+
+const ENCRYPT_SECRET = process.env.EENGINE_SECRET || config.service.secret;
+
+const VAULT_ADDR = process.env.VAULT_ADDR;
+const VAULT_ROLE_ID = process.env.VAULT_ROLE_ID;
+const VAULT_SECRET_ID = process.env.VAULT_SECRET_ID;
+const VAULT_PATH = process.env.VAULT_PATH;
+const VAULT_KEY = process.env.VAULT_KEY || 'secret';
+
+const vaultClient =
+    VAULT_ADDR && VAULT_ROLE_ID && VAULT_SECRET_ID && VAULT_PATH
+        ? vault({
+              apiVersion: 'v1',
+              endpoint: VAULT_ADDR
+          })
+        : false;
+
+const cache = new Map();
+
+async function getVaultClient(opts) {
+    if (!vaultClient) {
+        return false;
+    }
+
+    if (!vaultClient.token) {
+        const result = await vaultClient.approleLogin({
+            role_id: VAULT_ROLE_ID,
+            secret_id: VAULT_SECRET_ID
+        });
+        vault.token = result.auth.client_token;
+        if (opts && opts.text) {
+            console.error(`Retrieved access token from Vault for role ${VAULT_ROLE_ID}`);
+        } else {
+            logger.info({ msg: 'Retrieved access token from Vault', role_id: VAULT_ROLE_ID });
+        }
+    }
+
+    return vaultClient;
+}
+
+async function getSecret(opts) {
+    if (cache.has('secret')) {
+        return cache.get('secret');
+    }
+
+    if (process.env._VAULT_SECRET) {
+        if (opts && opts.text) {
+            console.error(`Using cached encryption secret from Vault`);
+        } else {
+            logger.info({ msg: 'Using cached encryption secret from Vault', role_id: VAULT_ROLE_ID });
+        }
+        cache.set('secret', process.env._VAULT_SECRET);
+        return process.env._VAULT_SECRET;
+    }
+
+    // check vault
+    let vaultClient = await getVaultClient(opts);
+    if (vaultClient) {
+        let vaultRes = await vaultClient.read(VAULT_PATH);
+        let secret = vaultRes && vaultRes.data && vaultRes.data.data && vaultRes.data.data[VAULT_KEY];
+
+        if (opts && opts.text) {
+            console.error(`Retrieved encryption secret from Vault for role ${VAULT_ROLE_ID}`);
+        } else {
+            logger.info({ msg: 'Retrieved encryption secret from Vault', role_id: VAULT_ROLE_ID });
+        }
+
+        process.env._VAULT_SECRET = secret;
+        cache.set('secret', process.env._VAULT_SECRET);
+        return secret;
+    }
+
+    return ENCRYPT_SECRET;
+}
+
+module.exports = getSecret;

package/lib/logger.js

@@ -0,0 +1,35 @@
+'use strict';
+
+const config = require('wild-config');
+const pino = require('pino');
+
+config.log = config.log || {
+    level: 'trace'
+};
+
+let logger = pino();
+logger.level = process.env.EENGINE_LOG_LEVEL || config.log.level;
+
+const { threadId } = require('worker_threads');
+
+if (threadId) {
+    logger = logger.child({ tid: threadId });
+}
+
+process.on('uncaughtException', err => {
+    logger.fatal({
+        msg: 'uncaughtException',
+        err
+    });
+    setTimeout(() => process.exit(1), 10);
+});
+
+process.on('unhandledRejection', err => {
+    logger.fatal({
+        msg: 'unhandledRejection',
+        err
+    });
+    setTimeout(() => process.exit(2), 10);
+});
+
+module.exports = logger;

package/lib/lua/s-list-accounts.lua

@@ -0,0 +1,51 @@
+local listKey = KEYS[1];
+
+local filterState = ARGV[1];
+local skip = tonumber(ARGV[2]) or 0;
+local count = tonumber(ARGV[3]) or 0;
+
+local total = redis.call("SCARD", listKey);
+
+local list = redis.call("SMEMBERS", listKey);
+
+if skip >= total then
+	return {total,skip, {}}
+end
+
+local shouldSkip = skip;
+local matching = 0;
+local result = {}
+
+for index, account in ipairs(list) do
+
+    local state;
+    if filterState ~= '*' then
+        -- load only if we actually need to compare account state value
+        state = redis.call("HGET", "iad:" .. account, "state"); 
+    end
+
+    if filterState == '*' or filterState == state then
+        -- state matches, can use this entry for listing        
+
+        if shouldSkip == 0 then
+            -- enough entries skipped, can use
+            if #result < count then
+                -- now we can actually use this record
+                result[#result + 1] = redis.call("HGETALL", "iad:" .. account);
+            else
+                -- max number entries in result buffer
+                if filterState == '*' then
+                    -- no point looking further, we already know the total count
+                    matching = total;
+                    break;
+                end
+            end
+        else
+            shouldSkip = shouldSkip - 1;
+        end
+
+        matching = matching + 1;
+    end    
+end
+
+return {matching, skip, result}

package/lib/lua/z-expunge.lua

@@ -0,0 +1,20 @@
+local listKey = KEYS[1];
+local mailboxKey = KEYS[2];
+
+local seq = tonumber(ARGV[1]) or 0;
+
+if redis.call("EXISTS", listKey) == 1 then
+    local list = redis.call("ZRANGE", listKey, seq-1, seq-1, "WITHSCORES");
+    local entry = list[1];
+    local uid = tonumber(list[2]) or 0;
+
+    if entry ~= nil and entry ~= "" and uid > 0 then 
+        local y = redis.call("ZREMRANGEBYRANK", listKey, seq-1, seq-1);
+        if y == 1 then
+            redis.call("HINCRBY", mailboxKey, "messages", -1);
+        end
+        return {uid, entry};
+    end;
+end;
+
+return nil;