emailengine-app 1.14.8 → 2.60.1

package/lib/mailbox.js

@@ -1,1546 +0,0 @@
-'use strict';
-
-const crypto = require('crypto');
-const { serialize, unserialize, compareExisting, normalizePath } = require('./tools');
-const msgpack = require('msgpack5')();
-const he = require('he');
-const libmime = require('libmime');
-const punycode = require('punycode/');
-const EmailReplyParser = require('email-reply-parser');
-const linkify = require('linkifyjs/html');
-const { htmlToText } = require('html-to-text');
-const addressparser = require('nodemailer/lib/addressparser');
-const settings = require('./settings');
-const { bounceDetect } = require('./bounce-detect');
-const appendList = require('./append-list');
-
-const {
-    MESSAGE_NEW_NOTIFY,
-    MAILBOX_DELETED_NOTIFY,
-    MESSAGE_DELETED_NOTIFY,
-    MESSAGE_UPDATED_NOTIFY,
-    MAILBOX_RESET_NOTIFY,
-    MAILBOX_NEW_NOTIFY,
-    EMAIL_BOUNCE_NOTIFY
-} = require('./consts');
-
-// Do not check for flag updates using full sync more often than this value
-const FULL_SYNC_DELAY = 30 * 60 * 1000;
-const MAX_HTML_PARSE_LENGTH = 2 * 1024 * 1024; // do not parse HTML messages larger than 2MB to plaintext
-
-class Mailbox {
-    constructor(connection, entry) {
-        this.status = false;
-        this.connection = connection;
-        this.path = entry.path;
-        this.listingEntry = entry;
-        this.syncDisabled = entry.syncDisabled;
-
-        this.imapClient = this.connection.imapClient;
-
-        this.logger = this.connection.mainLogger.child({
-            sub: 'mailbox',
-            path: this.path
-        });
-
-        this.isGmail = connection.isGmail;
-        this.isAllMail = this.isGmail && this.listingEntry.specialUse === '\\All';
-
-        this.selected = false;
-
-        this.redisKey = BigInt('0x' + crypto.createHash('sha1').update(normalizePath(this.path)).digest('hex')).toString(36);
-
-        this.runPartialSyncTimer = false;
-
-        this.synced = false;
-    }
-
-    getMailboxStatus() {
-        let mailboxInfo = this.imapClient.mailbox;
-
-        let status = {
-            path: this.path
-        };
-
-        status.highestModseq = mailboxInfo.highestModseq ? mailboxInfo.highestModseq : false;
-        status.uidValidity = mailboxInfo.uidValidity ? mailboxInfo.uidValidity : false;
-        status.uidNext = mailboxInfo.uidNext ? mailboxInfo.uidNext : false;
-        status.messages = mailboxInfo.exists ? mailboxInfo.exists : 0;
-
-        return status;
-    }
-
-    /**
-     * Loads last known mailbox state from Redis
-     * @returns {Object} mailbox state
-     */
-    async getStoredStatus() {
-        let data = await this.connection.redis.hgetall(this.getMailboxKey());
-        data = data || {};
-        return {
-            path: data.path || this.path,
-            uidValidity: data.uidValidity && !isNaN(data.uidValidity) ? BigInt(data.uidValidity) : false,
-            highestModseq: data.highestModseq && !isNaN(data.highestModseq) ? BigInt(data.highestModseq) : false,
-            messages: data.messages && !isNaN(data.messages) ? Number(data.messages) : false,
-            uidNext: data.uidNext && !isNaN(data.uidNext) ? Number(data.uidNext) : false,
-            lastFullSync: data.lastFullSync ? new Date(data.lastFullSync) : false
-        };
-    }
-
-    /**
-     * Updates known mailbox state in Redis
-     * @param {Object} data
-     */
-    async updateStoredStatus(data) {
-        if (!data || typeof data !== 'object') {
-            return false;
-        }
-
-        let list = Object.keys(data)
-            .map(key => {
-                switch (key) {
-                    case 'path':
-                    case 'uidValidity':
-                    case 'highestModseq':
-                    case 'messages':
-                    case 'uidNext':
-                        return [key, data[key].toString()];
-
-                    case 'lastFullSync':
-                        return [key, data[key].toISOString()];
-                }
-            })
-            .filter(entry => entry);
-
-        if (!list.length) {
-            return;
-        }
-
-        await this.connection.redis.hmset(this.getMailboxKey(), Object.fromEntries(list));
-    }
-
-    /**
-     * Sets message entry object. Entries are ordered by `uid` property
-     * @param {Object} data
-     * @param {Number} Sequence number for the added entry
-     */
-    async entryListSet(data) {
-        if (isNaN(data.uid)) {
-            return null;
-        }
-
-        return await this.connection.redis.zSet(this.getMessagesKey(), Number(data.uid), serialize(data));
-    }
-
-    /**
-     * Retrieves message entry object for the provided sequence value
-     * @param {Number} seq Sequence number
-     * @returns {Object|null} Message entry object
-     */
-    async entryListGet(seq, options) {
-        let range = Number(seq);
-        options = options || {};
-        let command = options.uid ? 'zGetByUidBuffer' : 'zGetBuffer';
-        let response = await this.connection.redis[command](this.getMessagesKey(), range);
-        if (response) {
-            try {
-                return {
-                    uid: Number(response[0]),
-                    entry: unserialize(response[1]),
-                    seq: Number(response[2])
-                };
-            } catch (err) {
-                return null;
-            }
-        }
-        return null;
-    }
-
-    /**
-     * Deletes entry from message list for the provided sequence value
-     * @param {Number} seq Sequence number
-     * @returns {Object|null} Message entry object that was deleted
-     */
-    async entryListExpunge(seq) {
-        let response = await this.connection.redis.zExpungeBuffer(this.getMessagesKey(), this.getMailboxKey(), seq);
-        if (response) {
-            try {
-                return unserialize(response[1]);
-            } catch (err) {
-                return null;
-            }
-        }
-        return null;
-    }
-
-    isSelected() {
-        return this.selected && this.imapClient.mailbox && normalizePath(this.imapClient.mailbox.path) === normalizePath(this.path);
-    }
-
-    getMessagesKey() {
-        return `iam:${this.connection.account}:l:${this.redisKey}`;
-    }
-
-    getMailboxKey() {
-        return `iam:${this.connection.account}:h:${this.redisKey}`;
-    }
-
-    getBounceKey() {
-        return `iar:b:${this.connection.account}`;
-    }
-
-    startIdle() {
-        if (!this.isSelected() || this.imapClient.idling) {
-            return;
-        }
-        this.imapClient.idle().catch(err => {
-            this.logger.error({ msg: 'IDLE error', err });
-        });
-    }
-
-    // clear mailbox records
-    async clear(opts) {
-        opts = opts || {};
-
-        clearTimeout(this.runPartialSyncTimer);
-
-        await this.connection.redis.del(this.getMailboxKey());
-        await this.connection.redis.del(this.getMessagesKey());
-
-        this.logger.debug({ msg: 'Deleted mailbox', path: this.listingEntry.path });
-
-        if (!opts.skipNotify) {
-            this.connection.notify(this, MAILBOX_DELETED_NOTIFY, {
-                path: this.listingEntry.path,
-                name: this.listingEntry.name,
-                specialUse: this.listingEntry.specialUse || false
-            });
-        }
-    }
-
-    async sync() {
-        if (this.selected) {
-            // expect current folder to be already synced
-            return false;
-        }
-
-        let status = await this.imapClient.status(this.path, {
-            uidNext: true,
-            messages: true,
-            highestModseq: true,
-            uidValidity: true
-        });
-
-        if (!status) {
-            // nothing to do here
-            return;
-        }
-
-        status.highestModseq = status.highestModseq || false;
-
-        if (this.syncDisabled) {
-            // only update counters
-            await this.updateStoredStatus(status);
-            return true;
-        }
-
-        let storedStatus = await this.getStoredStatus();
-        if (status.uidValidity === storedStatus.uidValidity) {
-            if (
-                status.uidNext === storedStatus.uidNext &&
-                status.messages === storedStatus.messages &&
-                storedStatus.lastFullSync > new Date(Date.now() - FULL_SYNC_DELAY)
-            ) {
-                // no reason to sync
-                return true;
-            }
-
-            if ((!status.messages && !storedStatus.messages) || (status.highestModseq && status.highestModseq === storedStatus.highestModseq)) {
-                // no reason to sync
-                return true;
-            }
-        }
-
-        let syncedPromise = new Promise((resolve, reject) => {
-            this.synced = resolve;
-            this.select(true).catch(err => reject(err));
-        });
-
-        await syncedPromise;
-    }
-
-    async select(skipIdle) {
-        let lock = await this.imapClient.getMailboxLock(this.path);
-        // have to release the lock immediatelly, otherwise difficult to process 'exists' / 'expunge' events
-        lock.release();
-
-        if (!skipIdle) {
-            // do not wait until command finishes before proceeding
-            this.startIdle();
-        }
-    }
-
-    logEvent(msg, event) {
-        const logObj = Object.assign({ msg }, event);
-        Object.keys(logObj).forEach(key => {
-            if (typeof logObj[key] === 'bigint') {
-                logObj[key] = logObj[key].toString();
-            }
-            if (typeof logObj[key].has === 'function') {
-                logObj[key] = Array.from(logObj[key]);
-            }
-        });
-        this.logger.trace(logObj);
-    }
-
-    async onExists(event) {
-        this.logEvent('Untagged EXISTS', event);
-
-        clearTimeout(this.runPartialSyncTimer);
-        this.runPartialSyncTimer = setTimeout(() => {
-            this.shouldRunPartialSyncAfterExists()
-                .then(shouldRun => {
-                    if (shouldRun) {
-                        this.logger.trace({ msg: 'Running partial sync' });
-                        return this.partialSync();
-                    }
-                    return false;
-                })
-                .then(() => this.select())
-                .catch(err => this.logger.error({ msg: 'Sync error', err }));
-        }, 1000);
-    }
-
-    async onExpunge(event) {
-        this.logEvent('Untagged EXPUNGE', event);
-
-        let deletedEntry = await this.entryListExpunge(event.seq);
-        if (deletedEntry) {
-            await this.processDeleted(deletedEntry);
-            await this.markUpdated();
-        }
-    }
-
-    async onFlags(event) {
-        this.logEvent('Untagged FETCH', event);
-
-        let storedMessage = await this.entryListGet(event.seq, { uid: false });
-        let changes;
-
-        // ignore Recent flag
-        event.flags.delete('\\Recent');
-
-        if (!storedMessage) {
-            // New! There should not be new messages.
-            // What should we do? Currently triggering partial sync.
-            return await this.onExists();
-        } else if ((changes = compareExisting(storedMessage.entry, event, ['flags']))) {
-            let messageData = storedMessage.entry;
-            messageData.flags = event.flags;
-            let seq = await this.entryListSet(messageData);
-            if (seq) {
-                await this.processChanges(storedMessage, changes);
-            }
-        }
-    }
-
-    async shouldRunPartialSyncAfterExists() {
-        let storedStatus = await this.getStoredStatus();
-        let mailboxStatus = this.getMailboxStatus();
-        return mailboxStatus.messages !== storedStatus.messages;
-    }
-
-    async partialSync(storedStatus) {
-        storedStatus = storedStatus || (await this.getStoredStatus());
-        let mailboxStatus = this.getMailboxStatus();
-
-        let lock = await this.imapClient.getMailboxLock(this.path);
-        try {
-            let newMessages = [];
-
-            let fields = { uid: true, flags: true, modseq: true, emailId: true, labels: true, internalDate: true };
-            let range = '1:*';
-            let opts = {
-                uid: true
-            };
-
-            if (this.imapClient.enabled.has('CONDSTORE') && storedStatus.highestModseq) {
-                opts.changedSince = storedStatus.highestModseq;
-            } else if (storedStatus.uidNext) {
-                range = `${storedStatus.uidNext}:*`;
-            }
-
-            if (mailboxStatus.messages) {
-                // only fetch messages if there is some
-                for await (let messageData of this.imapClient.fetch(range, fields, opts)) {
-                    // ignore Recent flag
-                    messageData.flags.delete('\\Recent');
-
-                    let storedMessage = await this.entryListGet(messageData.uid, { uid: true });
-
-                    let changes;
-                    if (!storedMessage) {
-                        // new!
-                        let seq = await this.entryListSet(messageData);
-                        if (seq) {
-                            newMessages.push(messageData);
-                        }
-                    } else if ((changes = compareExisting(storedMessage.entry, messageData))) {
-                        let seq = await this.entryListSet(messageData);
-                        if (seq) {
-                            await this.processChanges(messageData, changes);
-                        }
-                    }
-                }
-            }
-
-            await this.updateStoredStatus(this.getMailboxStatus());
-
-            let messageFetchOptions = {};
-            let notifyText = await settings.get('notifyText');
-            if (notifyText) {
-                messageFetchOptions.textType = '*';
-                let notifyTextSize = await settings.get('notifyTextSize');
-                if (notifyTextSize) {
-                    messageFetchOptions.maxBytes = notifyTextSize;
-                }
-            }
-
-            let notifyHeaders = await settings.get('notifyHeaders');
-            if (notifyHeaders) {
-                messageFetchOptions.headers = notifyHeaders.includes('*') ? true : notifyHeaders.length ? notifyHeaders : false;
-            }
-
-            // have to call after fetch is finished
-            for (let messageData of newMessages) {
-                if (this.connection.notifyFrom && messageData.internalDate < this.connection.notifyFrom) {
-                    // skip too old messages
-                    continue;
-                }
-                await this.processNew(messageData, messageFetchOptions);
-            }
-
-            if (newMessages.length) {
-                await this.markUpdated();
-            }
-        } finally {
-            lock.release();
-        }
-    }
-
-    async processDeleted(messageData) {
-        this.logger.debug({ msg: 'Deleted', uid: messageData.uid });
-
-        //FIXME: does not work as there is no messageId property
-        /*
-        if (messageData.messageId) {
-            try {
-                let deleted = await appendList.clear(this.connection.redis, this.getBounceKey(), messageData.messageId);
-                if (deleted) {
-                    this.logger.error({
-                        msg: 'Cleared bounce log for message',
-                        id: messageData.id,
-                        uid: messageData.uid,
-                        messageId: messageData.messageId
-                    });
-                }
-            } catch (err) {
-                this.logger.error({
-                    msg: 'Failed to clear bounce log',
-                    id: messageData.id,
-                    uid: messageData.uid,
-                    messageId: messageData.messageId,
-                    err
-                });
-            }
-        }
-        */
-
-        let packedUid = await this.connection.packUid(this, messageData.uid);
-        await this.connection.notify(this, MESSAGE_DELETED_NOTIFY, {
-            id: packedUid,
-            uid: messageData.uid
-        });
-    }
-
-    async processNew(messageData, options) {
-        this.logger.debug({ msg: 'New message', uid: messageData.uid, flags: Array.from(messageData.flags) });
-
-        options.skipLock = true;
-        options.headers = 'headers' in options ? options.headers : false;
-        options.skipVisible = true;
-
-        let messageInfo = await this.getMessage(messageData, options);
-        if (!messageInfo) {
-            this.logger.debug({ msg: 'Not found', uid: messageData.uid });
-            return;
-        }
-
-        let date = new Date(messageInfo.date);
-        if (this.connection.notifyFrom && date < this.connection.notifyFrom) {
-            // skip too old messages
-            return;
-        }
-
-        let bounceNotifyInfo;
-
-        // Check if this could be a bounce
-        if (this.mightBeABounce(messageInfo)) {
-            // parse for bounce
-            try {
-                let { content } = await this.imapClient.download(messageInfo.uid, false, {
-                    uid: true,
-                    // future feature
-                    chunkSize: options.chunkSize
-                });
-                if (content) {
-                    let bounce = await bounceDetect(content);
-
-                    let stored = 0;
-                    if (bounce.action && bounce.recipient && bounce.messageId) {
-                        let storedBounce = {
-                            i: messageInfo.id,
-                            r: bounce.recipient,
-                            t: Date.now(),
-                            a: bounce.action
-                        };
-
-                        if (bounce.response && bounce.response.message) {
-                            storedBounce.m = bounce.response.message;
-                        }
-
-                        if (bounce.response && bounce.response.status) {
-                            storedBounce.s = bounce.response.status;
-                        }
-
-                        // Store bounce info
-                        stored = await appendList.append(this.connection.redis, this.getBounceKey(), bounce.messageId, storedBounce);
-
-                        bounceNotifyInfo = Object.assign({ bounceMessage: messageInfo.id }, bounce);
-                    }
-
-                    this.logger.debug({
-                        msg: 'Detected bounce message',
-                        id: messageInfo.id,
-                        uid: messageInfo.uid,
-                        messageId: messageInfo.messageId,
-                        bounce,
-                        stored
-                    });
-                }
-            } catch (err) {
-                this.logger.error({
-                    msg: 'Failed to process potential bounce',
-                    id: messageInfo.id,
-                    uid: messageInfo.uid,
-                    messageId: messageInfo.messageId,
-                    err
-                });
-            }
-        }
-
-        await this.connection.notify(this, MESSAGE_NEW_NOTIFY, messageInfo);
-        if (bounceNotifyInfo) {
-            // send bounce notification _after_ bounce email notification
-            await this.connection.notify(false, EMAIL_BOUNCE_NOTIFY, bounceNotifyInfo);
-        }
-    }
-
-    async getMessageInfo(messageData, extended) {
-        if (!messageData) {
-            return false;
-        }
-
-        let packedUid = await this.connection.packUid(this, messageData.uid);
-        let { attachments, textId, encodedTextSize } = this.getAttachmentList(packedUid, messageData.bodyStructure);
-
-        let envelope = messageData.envelope || {};
-        let date = envelope.date || messageData.internalDate;
-
-        let result = {
-            id: packedUid,
-            uid: messageData.uid,
-
-            emailId: messageData.emailId || undefined,
-            threadId: messageData.threadId || undefined,
-
-            date: (date && date.toISOString()) || undefined,
-            flags: (extended && messageData.flags && messageData.flags.size && Array.from(messageData.flags)) || undefined,
-
-            unseen: messageData.flags && !messageData.flags.has('\\Seen') ? true : undefined,
-            flagged: messageData.flags && messageData.flags.has('\\Flagged') ? true : undefined,
-            answered: messageData.flags && messageData.flags.has('\\Answered') ? true : undefined,
-            draft: messageData.flags && messageData.flags.has('\\Draft') ? true : undefined,
-
-            size: messageData.size || undefined,
-            subject: envelope.subject || undefined,
-            from: envelope.from && envelope.from[0] ? envelope.from[0] : undefined,
-
-            replyTo: extended && envelope.replyTo && envelope.replyTo[0] ? envelope.replyTo[0] : undefined,
-            sender: extended && envelope.sender && envelope.sender[0] ? envelope.sender[0] : undefined,
-
-            to: envelope.to && envelope.to.length ? envelope.to : undefined,
-            cc: envelope.cc && envelope.cc.length ? envelope.cc : undefined,
-
-            bcc: extended && envelope.bcc && envelope.bcc.length ? envelope.bcc : undefined,
-
-            attachments: attachments && attachments.length ? attachments : undefined,
-            messageId: (envelope.messageId && envelope.messageId.toString().trim()) || undefined,
-            inReplyTo: envelope.inReplyTo || undefined,
-
-            labels: messageData.labels && messageData.labels.size ? Array.from(messageData.labels) : undefined,
-
-            headers: (extended && messageData.headers && libmime.decodeHeaders(messageData.headers.toString().trim())) || undefined,
-            text: textId
-                ? {
-                      id: textId,
-                      encodedSize: encodedTextSize
-                  }
-                : undefined
-        };
-
-        Object.keys(result).forEach(key => {
-            if (typeof result[key] === 'undefined') {
-                delete result[key];
-            }
-        });
-
-        // is there a related bounce as well?
-        try {
-            if (result.messageId) {
-                let bounces = await appendList.list(this.connection.redis, this.getBounceKey(), result.messageId);
-                if (bounces && bounces.length) {
-                    result.bounces = bounces.map(row => {
-                        let bounce = {
-                            message: row.i,
-                            recipient: row.r,
-                            action: row.a
-                        };
-                        if (row.m || row.s) {
-                            bounce.response = {};
-                        }
-                        if (row.m) {
-                            bounce.response.message = row.m;
-                        }
-                        if (row.s) {
-                            bounce.response.status = row.s;
-                        }
-                        bounce.date = new Date(row.t).toISOString();
-                        return bounce;
-                    });
-                }
-            }
-        } catch (E) {
-            this.logger.error({
-                msg: 'Failed to fetch bounces',
-                id: messageData.id,
-                uid: messageData.uid,
-                messageId: messageData.messageId,
-                err: E
-            });
-        }
-
-        return result;
-    }
-
-    getAttachmentList(packedUid, bodyStructure) {
-        let attachments = [];
-        let textParts = [[], [], []];
-        if (!bodyStructure) {
-            return attachments;
-        }
-
-        let idBuf = Buffer.from(packedUid.replace(/-/g, '+').replace(/_/g, '/'), 'base64');
-
-        let encodedTextSize = {};
-
-        let walk = (node, isRelated) => {
-            if (node.type === 'multipart/related') {
-                isRelated = true;
-            }
-
-            if (!/^multipart\//.test(node.type)) {
-                if (node.disposition === 'attachment' || !/^text\//.test(node.type)) {
-                    attachments.push({
-                        // append body part nr to message id
-                        id: Buffer.concat([idBuf, Buffer.from(node.part || '1')])
-                            .toString('base64')
-                            .replace(/\+/g, '-')
-                            .replace(/\//g, '_')
-                            .replace(/[=]+/g, ''),
-                        contentType: node.type,
-                        encodedSize: node.size,
-                        filename: (node.dispositionParameters && node.dispositionParameters.filename) || (node.parameters && node.parameters.name) || false,
-                        embedded: isRelated,
-                        inline: node.disposition === 'inline' || (!node.disposition && isRelated),
-                        contentId: node.id
-                    });
-                } else if ((!node.disposition || node.disposition === 'inline') && /^text\//.test(node.type)) {
-                    let type = node.type.substr(5);
-                    if (!encodedTextSize[type]) {
-                        encodedTextSize[type] = 0;
-                    }
-                    encodedTextSize[type] += node.size;
-                    switch (type) {
-                        case 'plain':
-                            textParts[0].push(node.part || '1');
-                            break;
-                        case 'html':
-                            textParts[1].push(node.part || '1');
-                            break;
-                        default:
-                            textParts[2].push(node.part || '1');
-                            break;
-                    }
-                }
-            }
-
-            if (node.childNodes) {
-                node.childNodes.forEach(childNode => walk(childNode, isRelated));
-            }
-        };
-
-        walk(bodyStructure, false);
-
-        return {
-            attachments,
-            textId: Buffer.concat([idBuf, msgpack.encode(textParts)])
-                .toString('base64')
-                .replace(/\+/g, '-')
-                .replace(/\//g, '_')
-                .replace(/[=]+/g, ''),
-            encodedTextSize
-        };
-    }
-
-    async processChanges(messageData, changes) {
-        let packedUid = await this.connection.packUid(this, messageData.uid);
-        await this.connection.notify(this, MESSAGE_UPDATED_NOTIFY, {
-            id: packedUid,
-            uid: messageData.uid,
-            changes
-        });
-        await this.markUpdated();
-    }
-
-    async fullSync() {
-        let range = '1:*';
-        let fields = { uid: true, flags: true, modseq: true, emailId: true, labels: true, internalDate: true };
-        let opts = {};
-
-        let lock = await this.imapClient.getMailboxLock(this.path);
-        try {
-            let mailboxStatus = this.getMailboxStatus();
-            let newMessages = [];
-
-            // full sync
-            let seqMax = 0;
-            let changes;
-
-            if (mailboxStatus.messages) {
-                // only fetch messages if there is some
-
-                for await (let messageData of this.imapClient.fetch(range, fields, opts)) {
-                    // ignore Recent flag
-                    messageData.flags.delete('\\Recent');
-
-                    if (messageData.seq > seqMax) {
-                        seqMax = messageData.seq;
-                    }
-
-                    let storedMessage = await this.entryListGet(messageData.uid, { uid: true });
-                    if (!storedMessage) {
-                        // new!
-                        let seq = await this.entryListSet(messageData);
-                        if (seq) {
-                            newMessages.push(messageData);
-                        }
-                    } else {
-                        let diff = storedMessage.seq - messageData.seq;
-                        for (let i = diff - 1; i >= 0; i--) {
-                            let seq = messageData.seq + i;
-                            let deletedEntry = await this.entryListExpunge(seq);
-                            if (deletedEntry) {
-                                await this.processDeleted(deletedEntry);
-                            }
-                        }
-
-                        if ((changes = compareExisting(storedMessage.entry, messageData))) {
-                            let seq = await this.entryListSet(messageData);
-                            if (seq) {
-                                await this.processChanges(messageData, changes);
-                            }
-                        }
-                    }
-                }
-            }
-
-            // delete unlisted messages
-            let storedMaxSeq = await this.connection.redis.zcard(this.getMessagesKey());
-            let diff = storedMaxSeq - seqMax;
-            for (let i = diff - 1; i >= 0; i--) {
-                let seq = seqMax + i + 1;
-                let deletedEntry = await this.entryListExpunge(seq);
-                if (deletedEntry) {
-                    await this.processDeleted(deletedEntry);
-                }
-            }
-
-            let status = this.getMailboxStatus();
-            status.lastFullSync = new Date();
-            await this.updateStoredStatus(status);
-
-            let messageFetchOptions = {};
-
-            let notifyText = await settings.get('notifyText');
-            if (notifyText) {
-                messageFetchOptions.textType = '*';
-                let notifyTextSize = await settings.get('notifyTextSize');
-                if (notifyTextSize) {
-                    messageFetchOptions.maxBytes = notifyTextSize;
-                }
-            }
-
-            let notifyHeaders = await settings.get('notifyHeaders');
-            if (notifyHeaders) {
-                messageFetchOptions.headers = notifyHeaders.includes('*') ? true : notifyHeaders.length ? notifyHeaders : false;
-            }
-
-            // have to call after fetch is finished
-            for (let messageData of newMessages) {
-                if (this.connection.notifyFrom && messageData.internalDate < this.connection.notifyFrom) {
-                    // skip too old messages
-                    continue;
-                }
-
-                await this.processNew(messageData, messageFetchOptions);
-            }
-
-            if (newMessages.length) {
-                await this.markUpdated();
-            }
-        } finally {
-            lock.release();
-        }
-    }
-
-    async onOpen() {
-        clearTimeout(this.runPartialSyncTimer);
-        this.selected = true;
-
-        let mailboxStatus = this.getMailboxStatus();
-
-        try {
-            let storedStatus = await this.getStoredStatus();
-
-            if (storedStatus.uidValidity && storedStatus.uidValidity !== mailboxStatus.uidValidity) {
-                // UIDVALIDITY has changed, full sync is required!
-                // delete mailbox status
-                let result = await this.connection.redis.multi().zcard(this.getMessagesKey()).del(this.getMessagesKey()).del(this.getMailboxKey()).exec();
-
-                let deletedMessages = (result[0] && Number(result[0][1])) || 0;
-                this.logger.info({
-                    msg: 'UIDVALIDITY change',
-                    deleted: deletedMessages,
-                    prevUidValidity: storedStatus.uidValidity && storedStatus.uidValidity.toString(),
-                    uidValidity: mailboxStatus.uidValidity && mailboxStatus.uidValidity.toString()
-                });
-
-                this.logger.debug({ msg: 'Mailbox reset', path: this.listingEntry.path });
-                await this.connection.notify(this, MAILBOX_RESET_NOTIFY, {
-                    path: this.listingEntry.path,
-                    name: this.listingEntry.name,
-                    specialUse: this.listingEntry.specialUse || false,
-                    uidValidity: mailboxStatus.uidValidity && mailboxStatus.uidValidity.toString(),
-                    prevUidValidity: storedStatus.uidValidity && storedStatus.uidValidity.toString()
-                });
-
-                // do not advertise messages as new
-                this.listingEntry.isNew = true;
-
-                // generates blank stored status as the Redis key was deleted
-                storedStatus = await this.getStoredStatus();
-            }
-
-            if (storedStatus.highestModseq && storedStatus.highestModseq === mailboxStatus.highestModseq) {
-                return false;
-            }
-
-            if (storedStatus.messages === 0 && mailboxStatus.messages === 0) {
-                return false;
-            }
-
-            if (
-                this.imapClient.enabled.has('CONDSTORE') &&
-                storedStatus.highestModseq < mailboxStatus.highestModseq &&
-                storedStatus.messages <= mailboxStatus.messages &&
-                mailboxStatus.uidNext - storedStatus.uidNext === mailboxStatus.messages - storedStatus.messages
-            ) {
-                // search for flag changes and new messages
-                return await this.partialSync(storedStatus);
-            }
-
-            if (
-                storedStatus.messages < mailboxStatus.messages &&
-                mailboxStatus.uidNext - storedStatus.uidNext === mailboxStatus.messages - storedStatus.messages
-            ) {
-                // seem to have new messages only
-                return await this.partialSync(storedStatus);
-            }
-
-            if (
-                storedStatus.messages === mailboxStatus.messages &&
-                storedStatus.uidNext === mailboxStatus.uidNext &&
-                storedStatus.lastFullSync &&
-                storedStatus.lastFullSync >= new Date(Date.now() - FULL_SYNC_DELAY)
-            ) {
-                // too soon from last full sync, message count seems the same
-                return false;
-            }
-
-            // Perform full sync. Only way of getting flag changes from non-CONDSTORE servers
-            return await this.fullSync();
-        } finally {
-            if (this.listingEntry.isNew) {
-                // fully synced, so not new anymore
-                this.listingEntry.isNew = false;
-                this.logger.debug({ msg: 'New mailbox', path: this.listingEntry.path });
-                this.connection.notify(this, MAILBOX_NEW_NOTIFY, {
-                    path: this.listingEntry.path,
-                    name: this.listingEntry.name,
-                    specialUse: this.listingEntry.specialUse || false,
-                    uidValidity: mailboxStatus.uidValidity.toString()
-                });
-            }
-
-            if (this.synced) {
-                this.synced();
-            } else {
-                await this.select();
-            }
-        }
-    }
-
-    async onClose() {
-        clearTimeout(this.runPartialSyncTimer);
-        this.selected = false;
-    }
-
-    async download(stream) {
-        return new Promise((resolve, reject) => {
-            let chunks = [];
-            let chunklen = 0;
-            stream.on('error', err => reject(err));
-            stream.on('readable', () => {
-                let chunk;
-                while ((chunk = stream.read()) !== null) {
-                    if (typeof chunk === 'string') {
-                        chunk = Buffer.from(chunk);
-                    }
-                    if (!chunk || !Buffer.isBuffer(chunk)) {
-                        // what's that?
-                        return;
-                    }
-                    chunks.push(chunk);
-                    chunklen += chunk.length;
-                }
-            });
-            stream.on('end', () => {
-                resolve(Buffer.concat(chunks, chunklen));
-            });
-        });
-    }
-
-    // User methods
-    // Call `clearTimeout(this.connection.completedTimer);` after locking mailbox
-    // Call this.onTaskCompleted() after selected mailbox is processed and lock is released
-
-    async getText(message, textParts, options) {
-        options = options || {};
-        let result = {};
-
-        let maxBytes = options.maxBytes || Infinity;
-        let reqMaxBytes = options.maxBytes && !isNaN(options.maxBytes) ? Number(options.maxBytes) + 4 : maxBytes;
-
-        let hasMore = false;
-
-        let lock;
-        if (!options.skipLock) {
-            lock = await this.imapClient.getMailboxLock(this.path);
-            clearTimeout(this.connection.completedTimer);
-        }
-
-        try {
-            for (let part of textParts) {
-                let { meta, content } = await this.imapClient.download(message.uid, part, {
-                    uid: true,
-                    // make sure we request enough bytes so we would have complete utf-8 codepoints
-                    maxBytes: reqMaxBytes,
-                    // future feature
-                    chunkSize: options.chunkSize
-                });
-
-                if (!content) {
-                    continue;
-                }
-                let text = await this.download(content);
-                text = text.toString().replace(/\r?\n/g, '\n');
-
-                let typeKey = (meta.contentType && meta.contentType.split('/')[1]) || 'plain';
-                if (!result[typeKey]) {
-                    result[typeKey] = [];
-                }
-
-                let typeSize = result[typeKey].reduce((sum, entry) => sum + entry.length, 0);
-                if (typeSize >= maxBytes) {
-                    hasMore = true;
-                    continue;
-                }
-                if (typeSize + text.length > maxBytes) {
-                    text = text.substr(0, maxBytes - typeSize);
-                    hasMore = true;
-                }
-                result[typeKey].push(text);
-            }
-        } finally {
-            if (lock) {
-                lock.release();
-            }
-        }
-
-        Object.keys(result).forEach(key => {
-            result[key] = result[key].join('\n');
-        });
-
-        result.hasMore = hasMore;
-
-        if (!options.skipLock) {
-            this.connection.onTaskCompleted();
-        }
-
-        return result;
-    }
-
-    async getAttachment(message, part, options) {
-        options = options || {};
-        let lock = await this.imapClient.getMailboxLock(this.path);
-        clearTimeout(this.connection.completedTimer);
-
-        let streaming = false;
-        let released = false;
-        try {
-            let { meta, content } = await this.imapClient.download(message.uid, part, {
-                uid: true,
-                maxBytes: options.maxBytes,
-                // future feature
-                chunkSize: options.chunkSize
-            });
-
-            if (!meta) {
-                return false;
-            }
-
-            content.headers = {
-                'content-type': meta.contentType || 'application/octet-stream',
-                'content-disposition': 'attachment' + (meta.filename ? `; filename=${he.encode(meta.filename)}` : '')
-            };
-
-            content.contentType = meta.contentType;
-            content.filename = meta.filename;
-            content.disposition = meta.disposition;
-            streaming = true;
-
-            content.once('end', () => {
-                if (!released) {
-                    released = true;
-                    lock.release();
-                }
-            });
-
-            content.once('error', () => {
-                if (!released) {
-                    released = true;
-                    lock.release();
-                    this.connection.onTaskCompleted();
-                }
-            });
-
-            return content;
-        } finally {
-            if (!streaming) {
-                lock.release();
-                this.connection.onTaskCompleted();
-            }
-        }
-    }
-
-    async getMessage(message, options) {
-        options = options || {};
-        let messageInfo;
-
-        try {
-            let lock;
-            if (!options.skipLock) {
-                lock = await this.imapClient.getMailboxLock(this.path);
-                clearTimeout(this.connection.completedTimer);
-            }
-
-            try {
-                let fields = options.fields || {
-                    uid: true,
-                    flags: true,
-                    size: true,
-                    bodyStructure: true,
-                    envelope: true,
-                    internalDate: true,
-                    headers: 'headers' in options ? options.headers : true,
-                    emailId: true,
-                    threadId: true,
-                    labels: true
-                };
-
-                let messageData = await this.imapClient.fetchOne(message.uid, fields, { uid: true });
-                if (!messageData) {
-                    return false;
-                }
-                messageInfo = await this.getMessageInfo(messageData, true);
-            } finally {
-                if (lock) {
-                    lock.release();
-                }
-            }
-
-            if (!messageInfo) {
-                return false;
-            }
-
-            // merge decoded text content with message data (if requested)
-            if (options.textType && messageInfo.text && messageInfo.text.id) {
-                let { textParts } = await this.connection.getMessageTextPaths(messageInfo.text.id);
-                if (textParts && textParts.length) {
-                    switch (options.textType) {
-                        case 'plain':
-                            textParts = textParts[0];
-                            break;
-                        case 'html':
-                            textParts = textParts[1];
-                            break;
-                        default:
-                            textParts = textParts.flatMap(entry => entry);
-                            break;
-                    }
-
-                    if (textParts && textParts.length) {
-                        let textContent = await this.getText(message, textParts, options);
-                        if (options.textType && options.textType !== '*') {
-                            textContent = {
-                                [options.textType]: textContent[options.textType] || '',
-                                hasMore: textContent.hasMore
-                            };
-                        }
-                        messageInfo.text = Object.assign(messageInfo.text, textContent);
-
-                        // Parse visible reply text as well if possible
-                        if (
-                            !options.skipVisible &&
-                            !['plain', 'html'].includes(options.textType) &&
-                            (messageInfo.inReplyTo || (messageInfo.headers && messageInfo.headers.references))
-                        ) {
-                            // might be a reply and not a specific text type was asked
-                            try {
-                                let plaintext = messageInfo.text.plain || '';
-                                if (!plaintext && messageInfo.text.html && messageInfo.text.html.length < MAX_HTML_PARSE_LENGTH) {
-                                    plaintext = htmlToText(messageInfo.text.html);
-                                }
-
-                                plaintext = plaintext.trim();
-                                if (plaintext) {
-                                    const email = new EmailReplyParser().read(plaintext);
-                                    messageInfo.text.visible = linkify(textToHtml(email.getVisibleText()), {
-                                        defaultProtocol: 'https'
-                                    });
-                                }
-                            } catch (err) {
-                                this.logger.error({ msg: 'Reply parsing error', err });
-                            }
-                        }
-                    }
-                }
-            }
-            return messageInfo;
-        } finally {
-            if (!options.skipLock) {
-                this.connection.onTaskCompleted();
-            }
-        }
-    }
-
-    async updateMessage(message, updates) {
-        updates = updates || {};
-        let lock = await this.imapClient.getMailboxLock(this.path);
-        clearTimeout(this.connection.completedTimer);
-
-        try {
-            let result = {};
-
-            if (updates.flags) {
-                if (updates.flags.set) {
-                    // If set exists the ignore add/delete calls
-                    let value = await this.imapClient.messageFlagsSet(message.uid, updates.flags.set, { uid: true });
-                    result.flags = {
-                        set: value
-                    };
-                } else {
-                    if (updates.flags.add && updates.flags.add.length) {
-                        let value = await this.imapClient.messageFlagsAdd(message.uid, updates.flags.add, { uid: true });
-                        if (!result.flags) {
-                            result.flags = {};
-                        }
-                        result.flags.add = value;
-                    }
-
-                    if (updates.flags.delete && updates.flags.delete.length) {
-                        let value = await this.imapClient.messageFlagsRemove(message.uid, updates.flags.delete, { uid: true });
-                        if (!result.flags) {
-                            result.flags = {};
-                        }
-                        result.flags.delete = value;
-                    }
-                }
-            }
-
-            if (updates.labels && this.isGmail) {
-                if (updates.labels.set) {
-                    // If set exists the ignore add/delete calls
-                    let value = await this.imapClient.messageFlagsSet(message.uid, updates.labels.set, { uid: true, useLabels: true });
-                    result.labels = {
-                        set: value
-                    };
-                } else {
-                    if (updates.labels.add && updates.labels.add.length) {
-                        let value = await this.imapClient.messageFlagsAdd(message.uid, updates.labels.add, { uid: true, useLabels: true });
-                        if (!result.labels) {
-                            result.labels = {};
-                        }
-                        result.labels.add = value;
-                    }
-
-                    if (updates.labels.delete && updates.labels.delete.length) {
-                        let value = await this.imapClient.messageFlagsRemove(message.uid, updates.labels.delete, { uid: true, useLabels: true });
-                        if (!result.labels) {
-                            result.labels = {};
-                        }
-                        result.labels.delete = value;
-                    }
-                }
-            }
-
-            return result;
-        } finally {
-            lock.release();
-            this.connection.onTaskCompleted();
-        }
-    }
-
-    async moveMessage(message, target) {
-        target = target || {};
-
-        let lock = await this.imapClient.getMailboxLock(this.path);
-        clearTimeout(this.connection.completedTimer);
-
-        try {
-            let result = {};
-
-            if (target.path) {
-                // If set exists the ignore add/delete calls
-                let value = await this.imapClient.messageMove(message.uid, target.path, { uid: true });
-                result.path = target.path;
-                if (value && value.uidMap && value.uidMap.has(message.uid)) {
-                    let uid = value.uidMap.get(message.uid);
-                    let packed = await this.connection.packUid(target.path, uid);
-                    result.id = packed;
-                    result.uid = uid;
-                }
-            }
-
-            return result;
-        } finally {
-            lock.release();
-            this.connection.onTaskCompleted();
-        }
-    }
-
-    async deleteMessage(message, force) {
-        let lock = await this.imapClient.getMailboxLock(this.path);
-        clearTimeout(this.connection.completedTimer);
-
-        try {
-            let result = {};
-
-            if (['\\Trash', '\\Junk'].includes(this.listingEntry.specialUse) || force) {
-                // delete
-                result.deleted = await this.imapClient.messageDelete(message.uid, { uid: true });
-            } else {
-                // move to trash
-                // find Trash folder path
-                let trashMailbox = await this.connection.getSpecialUseMailbox('\\Trash');
-                if (!trashMailbox || normalizePath(trashMailbox.path) === normalizePath(this.path)) {
-                    // no Trash found or already in trash
-                    result.deleted = await this.imapClient.messageDelete(message.uid, { uid: true });
-                } else {
-                    // we have a destionation, so can move message to there
-                    let moved = await await this.imapClient.messageMove(message.uid, trashMailbox.path, { uid: true });
-                    if (moved) {
-                        result.moved = {
-                            destination: moved.destination
-                        };
-                        if (moved && moved.uidMap && moved.uidMap.has(message.uid)) {
-                            result.moved.message = await this.connection.packUid(trashMailbox.path, moved.uidMap.get(message.uid));
-                        }
-                    }
-                }
-            }
-            return result;
-        } finally {
-            lock.release();
-            this.connection.onTaskCompleted();
-        }
-    }
-
-    async listMessages(options) {
-        options = options || {};
-        let page = Number(options.page) || 0;
-        let pageSize = Number(options.pageSize) || 20;
-
-        let lock = await this.imapClient.getMailboxLock(this.path);
-        clearTimeout(this.connection.completedTimer);
-
-        try {
-            let mailboxStatus = this.getMailboxStatus();
-
-            let messageCount = mailboxStatus.messages;
-            let uidList;
-            let opts = {};
-
-            if (options.search) {
-                uidList = await this.imapClient.search(options.search, { uid: true });
-                uidList = !uidList ? [] : uidList.sort((a, b) => b - a); // newer first
-                messageCount = uidList.length;
-            }
-
-            let pages = Math.ceil(messageCount / pageSize) || 1;
-
-            if (page < 0) {
-                page = 0;
-            }
-
-            if (page >= pages) {
-                page = pages - 1;
-            }
-
-            let messages = [];
-            let seqMax, seqMin, range;
-
-            if (!messageCount) {
-                return {
-                    page,
-                    pages,
-                    messages
-                };
-            }
-
-            if (options.search && uidList) {
-                let start = page * pageSize;
-                let uidRange = uidList.slice(start, start + pageSize).reverse();
-                range = uidRange.join(',');
-                opts.uid = true;
-            } else {
-                seqMax = messageCount - page * pageSize;
-                seqMin = seqMax - pageSize + 1;
-
-                if (seqMax >= messageCount) {
-                    seqMax = '*';
-                }
-
-                if (seqMin < 1) {
-                    seqMin = 1;
-                }
-
-                range = seqMin === seqMax ? `${seqMin}` : `${seqMin}:${seqMax}`;
-            }
-
-            let fields = {
-                uid: true,
-                flags: true,
-                size: true,
-                bodyStructure: true,
-                envelope: true,
-                internalDate: true,
-                emailId: true,
-                threadId: true,
-                labels: true
-            };
-
-            for await (let messageData of this.imapClient.fetch(range, fields, opts)) {
-                let messageInfo = await this.getMessageInfo(messageData);
-                messages.push(messageInfo);
-            }
-
-            return {
-                total: messageCount,
-                page,
-                pages,
-                // list newer first
-                messages: messages.reverse()
-            };
-        } finally {
-            lock.release();
-            this.connection.onTaskCompleted();
-        }
-    }
-
-    async buildContacts() {
-        let lock = await this.imapClient.getMailboxLock(this.path);
-        clearTimeout(this.connection.completedTimer);
-
-        let list = [];
-
-        try {
-            let range = '1:*';
-
-            let fields = {
-                uid: true,
-                envelope: true,
-                headers: ['from', 'to', 'cc']
-            };
-
-            for await (let messageData of this.imapClient.fetch(range, fields)) {
-                let addresses = parseAddressHeaders(messageData.headers);
-                if (addresses && addresses.length) {
-                    list = list.concat(addresses);
-                }
-            }
-
-            return list;
-        } finally {
-            lock.release();
-            this.connection.onTaskCompleted();
-        }
-    }
-
-    async markUpdated() {
-        try {
-            await this.connection.redis.hset(this.connection.getAccountKey(), 'sync', new Date().toISOString());
-        } catch (err) {
-            this.logger.error({ msg: 'Redis error', err });
-        }
-    }
-
-    mightBeABounce(messageInfo) {
-        if (this.path !== 'INBOX' && !(this.isAllMail && messageInfo.labels && messageInfo.labels.includes('\\Inbox'))) {
-            return false;
-        }
-
-        let name = (messageInfo.from && messageInfo.from.name) || '';
-        let address = (messageInfo.from && messageInfo.from.address) || '';
-
-        if (/Mail Delivery System|Mail Delivery Subsystem|Internet Mail Delivery/i.test(name)) {
-            return true;
-        }
-
-        if (/mailer-daemon@|postmaster@/i.test(address)) {
-            return true;
-        }
-
-        return false;
-    }
-}
-
-function textToHtml(str) {
-    let encoded = he
-        // encode special chars
-        .encode(str, {
-            useNamedReferences: true
-        });
-    let text = `<p>${
-        encoded
-            .replace(/\r?\n/g, '\n')
-            .trim() // normalize line endings
-            .replace(/[ \t]+$/gm, '')
-            .trim() // trim empty line endings
-            .replace(/\n\n+/g, '</p><p>')
-            .trim() // insert <p> to multiple linebreaks
-            .replace(/\n/g, '<br/>') // insert <br> to single linebreaks
-    }</p>`;
-
-    return text;
-}
-
-function parseAddressHeaders(headers) {
-    let decodedHeaders = headers && libmime.decodeHeaders(headers.toString().trim());
-    if (!decodedHeaders) {
-        return [];
-    }
-
-    let result = [];
-
-    Object.keys(decodedHeaders).forEach(key => {
-        let value = decodedHeaders[key];
-        switch (key) {
-            case 'from':
-            case 'to':
-            case 'cc':
-            case 'bcc':
-            case 'sender':
-            case 'reply-to':
-            case 'delivered-to':
-            case 'return-path':
-                value = addressparser(value);
-                decodeAddresses(value);
-                break;
-        }
-
-        value.forEach(entry => {
-            result.push(Object.assign({ type: key }, entry));
-        });
-    });
-
-    return result;
-}
-
-function decodeAddresses(addresses) {
-    for (let i = 0; i < addresses.length; i++) {
-        let address = addresses[i];
-        address.name = (address.name || '').toString().trim();
-
-        if (!address.address && /^(=\?([^?]+)\?[Bb]\?[^?]*\?=)(\s*=\?([^?]+)\?[Bb]\?[^?]*\?=)*$/.test(address.name)) {
-            let parsed = addressparser(libmime.decodeWords(address.name));
-            if (parsed.length) {
-                parsed.forEach(entry => addresses.push(entry));
-            }
-
-            // remove current element
-            addresses.splice(i, 1);
-            i--;
-            continue;
-        }
-
-        if (address.name) {
-            try {
-                address.name = libmime.decodeWords(address.name);
-            } catch (E) {
-                //ignore, keep as is
-            }
-        }
-        if (/@xn--/.test(address.address)) {
-            address.address =
-                address.address.substr(0, address.address.lastIndexOf('@') + 1) +
-                punycode.toUnicode(address.address.substr(address.address.lastIndexOf('@') + 1));
-        }
-        if (address.group) {
-            decodeAddresses(address.group);
-        }
-    }
-}
-
-module.exports.Mailbox = Mailbox;