imapkit 0.0.0-stage → 4.0.1

package/lib/threading.js

@@ -0,0 +1,357 @@
+'use strict';
+
+// THREAD and UID THREAD (RFC 5256 section 3) and the ORDEREDSUBJECT and REFERENCES algorithms
+
+const { states } = require('./command-states');
+const { getMessageData } = require('./mimeparser');
+const { collationKey, baseSubject, sentTime, parseMessageIds, searchMessages } = require('./sorting');
+
+/**
+ * Creates a thread tree node. A node without a message is a dummy
+ */
+function createNode(entry) {
+    return { entry: entry || null, parent: null, children: [] };
+}
+
+// Sent date order with the sequence number as tie-breaker (RFC 5256 section 2.2). A dummy is
+// ordered by its first child
+function compareNodes(a, b) {
+    while (!a.entry && a.children.length) {
+        a = a.children[0];
+    }
+    while (!b.entry && b.children.length) {
+        b = b.children[0];
+    }
+    return a.entry.date - b.entry.date || a.entry.seq - b.entry.seq;
+}
+
+/**
+ * Lists the nodes of trees in pre-order (parents before their children). Iterative, as message
+ * references can make a thread as deep as the mailbox is large.
+ *
+ * @param {Array} nodes Top level nodes
+ * @return {Array} All nodes
+ */
+function preOrder(nodes) {
+    const list = [];
+    const stack = nodes.slice().reverse();
+    while (stack.length) {
+        const node = stack.pop();
+        list.push(node);
+        for (let i = node.children.length - 1; i >= 0; i--) {
+            stack.push(node.children[i]);
+        }
+    }
+    return list;
+}
+
+/**
+ * Serializes thread nodes (RFC 5256 section 5): every thread is a thread-list, a node is followed by its
+ * only child, or by one thread-list per child when it has more. A dummy has no number of its own.
+ *
+ * @param {Array} roots Top level nodes
+ * @param {Boolean} isUid If true, list UIDs instead of sequence numbers
+ * @return {String} thread-data without the "THREAD" keyword
+ */
+function formatThreads(roots, isUid) {
+    const output = [];
+    // strings to write and nodes to expand, the next one is at the end
+    const stack = [];
+    const pushLists = nodes => {
+        for (let i = nodes.length - 1; i >= 0; i--) {
+            stack.push(')', nodes[i], '(');
+        }
+    };
+    pushLists(roots);
+    while (stack.length) {
+        const item = stack.pop();
+        if (typeof item === 'string') {
+            output.push(item);
+            continue;
+        }
+        const children = item.children;
+        if (children.length === 1) {
+            stack.push(children[0]);
+        } else {
+            pushLists(children);
+        }
+        if (item.entry) {
+            if (children.length) {
+                stack.push(' ');
+            }
+            output.push(isUid ? item.entry.message.uid : item.entry.seq);
+        }
+    }
+    return output.join('');
+}
+
+/**
+ * ORDEREDSUBJECT (RFC 5256 section 3): messages grouped by base subject, every thread is its first
+ * message with the others as children, threads ordered by the sent date of their first message
+ *
+ * @param {Array} entries Searched messages as `{ message, seq, date, subject }`
+ * @return {Array} Top level nodes
+ */
+function orderedSubject(entries) {
+    const compareSubjects = (a, b) => (a.subject.id < b.subject.id ? -1 : a.subject.id > b.subject.id ? 1 : 0);
+    const sorted = entries.slice().sort((a, b) => compareSubjects(a, b) || a.date - b.date || a.seq - b.seq);
+    const threads = [];
+    let current = null;
+    for (const entry of sorted) {
+        if (current && current.entry.subject.id === entry.subject.id) {
+            current.children.push(createNode(entry));
+        } else {
+            current = createNode(entry);
+            threads.push(current);
+        }
+    }
+    return threads.sort(compareNodes);
+}
+
+/**
+ * REFERENCES (RFC 5256 section 3), steps 1 to 6
+ *
+ * @param {Array} entries Searched messages in mailbox order as `{ message, seq, date, subject }`
+ * @return {Array} Top level nodes
+ */
+function references(entries) {
+    const byId = new Map();
+    const all = [];
+
+    const getNode = id => {
+        if (!byId.has(id)) {
+            const node = createNode();
+            byId.set(id, node);
+            all.push(node);
+        }
+        return byId.get(id);
+    };
+
+    const isDescendant = (node, ancestor) => {
+        for (let current = node; current; current = current.parent) {
+            if (current === ancestor) {
+                return true;
+            }
+        }
+        return false;
+    };
+
+    const unlink = node => {
+        if (node.parent) {
+            node.parent.children.splice(node.parent.children.indexOf(node), 1);
+            node.parent = null;
+        }
+    };
+
+    // links a parent and a child, unless that would introduce a loop. Only a node with children can have
+    // the parent as a descendant, so the walk up from the parent is skipped for the others
+    const link = (parent, child) => {
+        if (parent !== child && (!child.children.length || !isDescendant(parent, child))) {
+            parent.children.push(child);
+            child.parent = parent;
+        }
+    };
+
+    // (1) link the messages by their references
+    for (const entry of entries) {
+        const header = getMessageData(entry.message).tree.parsedHeader;
+
+        // (1.A) a message without a valid Message-ID, or with one that an earlier message already has, gets a unique one
+        const messageId = parseMessageIds(header['message-id'])[0];
+        let node = messageId !== undefined && getNode(messageId);
+        if (!node || node.entry) {
+            node = createNode();
+            all.push(node);
+        }
+        node.entry = entry;
+
+        // the References header, or else the first Message ID of In-Reply-To
+        let refs = parseMessageIds(header.references);
+        if (!refs.length) {
+            refs = parseMessageIds(header['in-reply-to']).slice(0, 1);
+        }
+
+        // (1.A) each reference is the parent of the next one, existing links are kept
+        const refNodes = refs.map(getNode);
+        for (let i = 1; i < refNodes.length; i++) {
+            if (!refNodes[i].parent) {
+                link(refNodes[i - 1], refNodes[i]);
+            }
+        }
+
+        // (1.B) the last reference is the parent of the message, replacing an existing link
+        const parent = refNodes[refNodes.length - 1] || null;
+        if (node.parent !== parent) {
+            unlink(node);
+            if (parent) {
+                link(parent, node);
+            }
+        }
+    }
+
+    // (2) messages without a parent are the top level
+    const roots = all.filter(node => !node.parent);
+
+    // (3) prune dummies: drop the ones without children, put the children of the others in their place,
+    // but not at the top level if there is more than one child
+    const replacement = (node, topLevel) => {
+        if (node.entry) {
+            return [node];
+        }
+        if (!node.children.length) {
+            return [];
+        }
+        return topLevel && node.children.length > 1 ? [node] : node.children;
+    };
+    // children before their parents, so a dummy is replaced by children that are already pruned
+    for (const node of preOrder(roots).reverse()) {
+        node.children = node.children.flatMap(child => replacement(child, false));
+        node.children.forEach(child => {
+            child.parent = node;
+        });
+    }
+    let top = roots.flatMap(node => replacement(node, true));
+    top.forEach(node => {
+        node.parent = null;
+    });
+
+    // (4) sort the top level by sent date, a dummy by its first child
+    top.forEach(node => {
+        if (!node.entry) {
+            node.children.sort(compareNodes);
+        }
+    });
+    top.sort(compareNodes);
+
+    // (5) gather threads with the same base subject, the subject of a dummy is the one of its first child
+    const threadSubject = node => (node.entry ? node.entry.subject : node.children[0].entry.subject);
+    const isReply = node => !!node.entry && node.entry.subject.isReply;
+    const table = new Map();
+
+    // (5.B) one message per base subject, a dummy or a message that is not a reply or forward is preferred
+    for (const node of top) {
+        const subject = threadSubject(node);
+        if (!subject.id) {
+            continue;
+        }
+        const old = table.get(subject.id);
+        if (!old || (old.entry && (!node.entry || (isReply(old) && !isReply(node))))) {
+            table.set(subject.id, node);
+        }
+    }
+
+    // (5.C) merge the other threads with the same subject into the one in the table
+    const removed = new Set();
+    for (const node of top) {
+        const subject = threadSubject(node);
+        if (!subject.id) {
+            continue;
+        }
+        const other = table.get(subject.id);
+        if (other === node) {
+            continue;
+        }
+        removed.add(node);
+        if (!other.entry && !node.entry) {
+            node.children.forEach(child => link(other, child));
+        } else if (!other.entry || (isReply(node) && !isReply(other))) {
+            link(other, node);
+        } else {
+            // both become children of a new dummy that takes the place of the message in the table
+            const dummy = createNode();
+            top[top.indexOf(other)] = dummy;
+            link(dummy, other);
+            link(dummy, node);
+            table.set(subject.id, dummy);
+        }
+    }
+    top = top.filter(node => !removed.has(node));
+
+    // (6) sort all sets of siblings by sent date, the deepest ones first
+    for (const node of preOrder(top).reverse()) {
+        node.children.sort(compareNodes);
+    }
+    top.sort(compareNodes);
+    return top;
+}
+
+const ALGORITHMS = {
+    ORDEREDSUBJECT: orderedSubject,
+    REFERENCES: references
+};
+
+/**
+ * Adds the THREAD and UID THREAD commands, unless another algorithm plugin did already, and the
+ * THREAD=<algorithm> capability
+ *
+ * @param {Object} server IMAP server
+ * @param {String} algorithm "ORDEREDSUBJECT" or "REFERENCES"
+ */
+function addThreadAlgorithm(server, algorithm) {
+    server.registerCapability('THREAD=' + algorithm);
+
+    if (server.threadAlgorithms) {
+        server.threadAlgorithms[algorithm] = ALGORITHMS[algorithm];
+        return;
+    }
+    server.threadAlgorithms = { [algorithm]: ALGORITHMS[algorithm] };
+
+    const threadHandler = (isUid, connection, parsed, data, callback) => {
+        const command = isUid ? 'UID THREAD' : 'THREAD';
+        const attributes = parsed.attributes || [];
+
+        // RFC 5256 section 5: thread-alg = "ORDEREDSUBJECT" / "REFERENCES" / thread-alg-ext, an atom
+        const name = attributes[0] && attributes[0].type === 'ATOM' ? attributes[0].value.toUpperCase() : '';
+        if (!Object.hasOwn(server.threadAlgorithms, name)) {
+            connection.sendStatus(
+                parsed,
+                data,
+                'BAD',
+                name ? 'Unsupported threading algorithm ' + name : command + ' expects a threading algorithm atom',
+                false,
+                command + ' FAILED'
+            );
+            return callback();
+        }
+
+        const result = searchMessages(connection, parsed, data, attributes.slice(1));
+        if (!result) {
+            return callback();
+        }
+
+        // base subjects are compared with the collation (RFC 5256 sections 3 and 7)
+        const entries = result.list.map(message => {
+            const base = baseSubject(getMessageData(message).tree.parsedHeader.subject);
+            return {
+                message,
+                seq: result.numbers[message.uid],
+                date: sentTime(message),
+                // a binary string of the collation key, these compare like the octets of the key
+                subject: { id: collationKey(base.subject).toString('binary'), isReply: base.isReply }
+            };
+        });
+
+        connection.send(
+            {
+                tag: '*',
+                command: 'THREAD',
+                // thread-data has no SP between thread-lists, so it can not be built from nested arrays
+                attributes: entries.length ? [{ type: 'TEXT', value: formatThreads(server.threadAlgorithms[name](entries), isUid) }] : []
+            },
+            command,
+            parsed,
+            data,
+            // the search result, so CONDSTORE knows about a MODSEQ search key (RFC 7162 section 3.1.9)
+            result
+        );
+        connection.sendStatus(parsed, data, 'OK', command + ' completed', false, command);
+        return callback();
+    };
+
+    // RFC 5256 section 3: EXPUNGE responses are not permitted while responding to THREAD, but are during UID THREAD.
+    // The search criteria start after the algorithm and the charset
+    server.setCommandHandler('THREAD', threadHandler.bind(null, false), { states: states.SELECTED, searchCriteria: 2, noExpunge: true });
+    server.setCommandHandler('UID THREAD', threadHandler.bind(null, true), { states: states.SELECTED, searchCriteria: 2 });
+}
+
+module.exports = { addThreadAlgorithm };

package/lib/utf8-session.js

@@ -0,0 +1,123 @@
+'use strict';
+
+const { isUtf8 } = require('buffer');
+const mailboxName = require('./mailbox-name');
+const { isEnabled } = require('./plugins/enable');
+
+/**
+ * UTF-8 support of a session, shared by the UTF8=ACCEPT (RFC 9755) and IMAP4rev2 (RFC 9051) plugins. Both
+ * turn on UTF-8 mailbox names and quoted strings with ENABLE, and they can be loaded in any order, so the
+ * state of a session is always computed from what it has enabled, see updateSession().
+ */
+
+// RFC 9755 section 3: mailbox names MUST NOT contain control characters, DEL, LINE SEPARATOR or
+// PARAGRAPH SEPARATOR. RFC 5198 section 2 (Net-Unicode) also forbids a leading BOM and unassigned code points
+// eslint-disable-next-line no-control-regex
+const INVALID_NAME_CHARS = /[\u0000-\u001f\u007f-\u009f\u2028\u2029]|^\ufeff|\p{Cn}/u;
+
+function badError(message) {
+    const err = new Error(message);
+    err.imapResponse = 'BAD';
+    return err;
+}
+
+/**
+ * Throws a BAD error for a mailbox name that is not Net-Unicode (RFC 9755 section 3, RFC 9051 section 5.1).
+ * RFC 5198 section 2 also requires Normalization Form C
+ *
+ * @param {String|Boolean} decoded Mailbox name as a unicode string, false if it could not be decoded
+ * @return {String} the decoded name
+ */
+function checkNetUnicode(decoded) {
+    if (decoded === false || INVALID_NAME_CHARS.test(decoded)) {
+        throw badError('Mailbox name must be valid Net-Unicode without control characters (RFC 9755 section 3)');
+    }
+    if (decoded.normalize('NFC') !== decoded) {
+        throw badError('Mailbox name must be in Unicode Normalization Form C (RFC 5198 section 2, RFC 9051 section 5.1)');
+    }
+    return decoded;
+}
+
+/**
+ * Checks a modified UTF-7 mailbox name from a client that did not enable UTF-8. The RFC 9755
+ * section 3 rules apply to these names as well, e.g. "&AA0-" encodes CR
+ *
+ * @param {String} name Mailbox name as a binary string
+ * @return {String} Storage name
+ * @throws {Error} BAD error if the name is not valid
+ */
+function importMutf7Name(name) {
+    const error = mailboxName(name);
+    if (error) {
+        throw badError(error);
+    }
+    checkNetUnicode(mailboxName.decode(name));
+    return name;
+}
+
+/**
+ * Converts a UTF-8 mailbox name from the client to the modified UTF-7 storage name
+ *
+ * @param {String} name Mailbox name as a binary string
+ * @return {String} Storage name
+ * @throws {Error} BAD error if the name is not valid
+ */
+function importUtf8Name(name) {
+    const buf = Buffer.from(name, 'binary');
+    return mailboxName.encode(checkNetUnicode(isUtf8(buf) && buf.toString('utf-8')));
+}
+
+/**
+ * Converts a modified UTF-7 storage name to UTF-8. A name that is not valid modified UTF-7
+ * is sent as it is.
+ *
+ * @param {String} path Storage name
+ * @return {String} Mailbox name as a binary string
+ */
+function exportUtf8Name(path) {
+    const decoded = mailboxName.decode(path);
+    return decoded === false ? path : Buffer.from(decoded, 'utf-8').toString('binary');
+}
+
+/**
+ * Sets the UTF-8 options of a session from the extensions it has enabled. Run it when the session starts,
+ * after ENABLE and after UNAUTHENTICATE (once the ENABLE plugin has cleared `connection.enabled`).
+ *
+ * - With the UTF8=ACCEPT plugin, quoted strings may hold UTF-8 even before ENABLE (RFC 9755 section 3) and
+ *   modified UTF-7 names must decode to Net-Unicode. Without it, 8-bit quoted strings are a syntax error until
+ *   ENABLE IMAP4rev2, like RFC 3501 says
+ * - ENABLE UTF8=ACCEPT or IMAP4rev2: mailbox names are UTF-8 in both directions and strings that are valid
+ *   UTF-8 are sent quoted (RFC 9755 section 3, RFC 9051 sections 4.3 and 5.1)
+ * - ENABLE UTF8=ACCEPT: SEARCH strings are UTF-8 and CHARSET is refused (RFC 9755 section 3)
+ * - ENABLE IMAP4rev2: SEARCH strings are UTF-8 unless a CHARSET says otherwise (RFC 9051 section 6.4.4)
+ *
+ * @param {Object} connection IMAP connection
+ */
+function updateSession(connection) {
+    const server = connection.server;
+    const accept = isEnabled(connection, 'UTF8=ACCEPT');
+    const rev2 = isEnabled(connection, 'IMAP4rev2');
+    const utf8Names = accept || rev2;
+
+    // UTF-8 names and quoted strings are on, plugins check this instead of the extension names
+    connection.utf8Enabled = utf8Names;
+    connection.parserOptions.utf8 = !!server.utf8Accept || rev2;
+    connection.compilerOptions.utf8 = utf8Names;
+    connection.searchCharset = accept ? 'UTF-8' : false;
+    connection.defaultSearchCharset = rev2 ? 'UTF-8' : false;
+
+    if (utf8Names) {
+        connection.importMailboxName = importUtf8Name;
+        connection.exportMailboxName = exportUtf8Name;
+    } else {
+        // back to the IMAPConnection methods
+        delete connection.exportMailboxName;
+        if (server.utf8Accept) {
+            connection.importMailboxName = importMutf7Name;
+        } else {
+            delete connection.importMailboxName;
+        }
+    }
+}
+
+module.exports = { updateSession };

package/lib/vanished.js

@@ -0,0 +1,57 @@
+'use strict';
+
+/**
+ * VANISHED responses (RFC 7162 section 3.2.10) in place of EXPUNGE notifications. Used by QRESYNC
+ * and by UIDONLY (RFC 9586 section 3.4), which both report expunged messages by UID.
+ */
+
+const { toSequenceSet } = require('./esearch');
+
+/**
+ * Checks if a queued notification is an EXPUNGE response for a removed message
+ *
+ * @param {Object} notification Queued notification
+ * @return {Boolean} true for an EXPUNGE notification
+ */
+function isExpungeNotification(notification) {
+    const name = !!notification.message && !!notification.attributes && notification.attributes[1];
+    return !!name && name.type === 'ATOM' && String(name.value).toUpperCase() === 'EXPUNGE';
+}
+
+/**
+ * Replaces the EXPUNGE notifications of a queue with VANISHED responses, consecutive EXPUNGE
+ * notifications become one VANISHED response. The queue is not changed
+ *
+ * @param {Array} queue Queued notifications
+ * @return {Array} notifications to send
+ */
+function toVanished(queue) {
+    if (!queue.some(isExpungeNotification)) {
+        return queue;
+    }
+    // groups of consecutive expunges, as lists of UIDs, between the other notifications
+    const groups = [];
+    queue.forEach(notification => {
+        if (!isExpungeNotification(notification)) {
+            groups.push(notification);
+        } else if (Array.isArray(groups[groups.length - 1])) {
+            groups[groups.length - 1].push(notification.message);
+        } else {
+            groups.push([notification.message]);
+        }
+    });
+    // the removed messages go with the response for output handlers, like `message` of EXPUNGE
+    return groups.map(group =>
+        Array.isArray(group)
+            ? {
+                  tag: '*',
+                  command: 'VANISHED',
+                  attributes: [{ type: 'SEQUENCE', value: toSequenceSet(group.map(message => message.uid)) }],
+                  notification: true,
+                  messages: group
+              }
+            : group
+    );
+}
+
+module.exports = { isExpungeNotification, toVanished };

package/package.json

@@ -1,6 +1,62 @@
 {
-  "name": "imapkit",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+    "name": "imapkit",
+    "version": "4.0.1",
+    "description": "Scriptable, strictly RFC compliant in-memory IMAP server for testing IMAP clients",
+    "main": "lib/server.js",
+    "bin": {
+        "imapkit": "bin/imapkit.js"
+    },
+    "files": [
+        "bin",
+        "cert",
+        "lib"
+    ],
+    "scripts": {
+        "prepare": "git rev-parse --git-dir >/dev/null 2>&1 && git config core.hooksPath .githooks || true",
+        "test": "npm run lint && npm run test:unit",
+        "test:unit": "node --test test/*.js",
+        "test:coverage": "node --test --experimental-test-coverage --test-coverage-include='lib/**' --test-coverage-lines=94 test/*.js",
+        "lint": "eslint",
+        "format": "prettier --write .",
+        "format:check": "prettier --check .",
+        "compare": "node compare/compare.js",
+        "dovecot:start": "bash compare/dovecot.sh start",
+        "dovecot:stop": "bash compare/dovecot.sh stop",
+        "update": "rm -rf node_modules package-lock.json && ncu -u && npm install"
+    },
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/postalsys/imapkit.git"
+    },
+    "bugs": {
+        "url": "https://github.com/postalsys/imapkit/issues"
+    },
+    "homepage": "https://imapkit.com",
+    "keywords": [
+        "imap",
+        "imap-server",
+        "mock",
+        "testing",
+        "test-server",
+        "imap4rev2",
+        "rfc3501",
+        "rfc9051",
+        "email"
+    ],
+    "author": "Postal Systems OÜ",
+    "license": "MIT",
+    "dependencies": {
+        "imap-handler": "1.3.1",
+        "smtp-server": "3.19.17"
+    },
+    "devDependencies": {
+        "@eslint/js": "10.0.1",
+        "eslint": "10.12.0",
+        "globals": "17.13.0",
+        "imapflow": "2.2.7",
+        "prettier": "3.9.9"
+    },
+    "engines": {
+        "node": ">=20.0.0"
+    }
+}