imapkit 4.0.0 → 4.0.2

package/README.md

@@ -224,7 +224,7 @@ Available commands:
 - **XTOYBIRD CONNECTION** dumps connection internals
 - **XTOYBIRD STORAGE** dumps storage as JSON
 - **XTOYBIRD USERADD "username" "password"** adds or updates user
-- **XTOYBIRD USERDEL "username"** removes an user
+- **XTOYBIRD USERDEL "username"** removes a user
 - **XTOYBIRD SHUTDOWN** Closes the server after the last client disconnects. New connections are rejected.
 
 Example usage for XTOYBIRD STORAGE:

package/bin/help.txt

@@ -26,7 +26,7 @@ IMAPKIT_SECURE, IMAPKIT_DEBUG, IMAPKIT_STORAGE, IMAPKIT_CONFIG,
 IMAPKIT_PLUGINS (comma separated) and IMAPKIT_SMTPPORT.
 
 NB! If port or smtpPort values are below 1024 you most probably need to
-use sudo or run the command in Administrator rights.
+use sudo or run the command with administrator rights.
 
 Available plugins
 -----------------

package/lib/commands/handlers/search.js

@@ -4,6 +4,7 @@ const { isUtf8 } = require('buffer');
 const { getMessageData, render } = require('../../mimeparser');
 const { monthIndex, dateKey, parseDateTime, parseHeaderDate } = require('../../dates');
 const { MAX_NUMBER, MAX_NUMBER64, isNumber } = require('../../numbers');
+const { decodeHeader, decodeUtf8 } = require('../../encoded-words');
 
 // RFC 3501 6.4.4 search keys and their arguments
 const searchKeys = {
@@ -142,7 +143,8 @@ function sendSearchError(connection, parsed, data, err, description) {
 }
 
 /**
- * Lower cases ASCII letters only, so that 8-bit octets in binary strings stay intact
+ * Lower cases ASCII letters only, so that 8-bit octets in binary strings stay intact. RFC 9051
+ * section 6.4.4 asks for case insensitive matching only within the ASCII range
  */
 function asciiLowerCase(str) {
     return str.replace(/[A-Z]+/g, chars => chars.toLowerCase());
@@ -371,11 +373,13 @@ module.exports = function (connection, messageSource, params, getMessageRange) {
         if (node.key === '_SEQ' || node.key === 'UID') {
             node.set = toMessageSet(node.args[0]);
         }
-        // lower case the string to look for once, not for every message
-        if (['BCC', 'BODY', 'CC', 'FROM', 'SUBJECT', 'TEXT', 'TO'].indexOf(node.key) >= 0) {
+        // lower case the string to look for once, not for every message. Header values are compared
+        // as Unicode text after decoding their encoded words, so the UTF-8 octets of the string are
+        // decoded too (RFC 9051 section 6.4.4)
+        if (['BODY', 'TEXT'].indexOf(node.key) >= 0) {
             node.needle = asciiLowerCase(node.args[0]);
-        } else if (node.key === 'HEADER') {
-            node.needle = asciiLowerCase(node.args[1]);
+        } else if (['BCC', 'CC', 'FROM', 'HEADER', 'SUBJECT', 'TO'].indexOf(node.key) >= 0) {
+            node.needle = asciiLowerCase(decodeUtf8(node.args[node.key === 'HEADER' ? 1 : 0]));
         }
         node.args.forEach(arg => {
             if (arg && typeof arg === 'object' && arg.key) {
@@ -387,16 +391,16 @@ module.exports = function (connection, messageSource, params, getMessageRange) {
 
     const hasFlag = (message, flag) => message.flags.indexOf(flag) >= 0;
 
-    // header lines as [lowercase name, unfolded value]
-    const getHeaders = message =>
-        (getMessageData(message).tree.header || []).map(line => {
-            const parts = line.split(':');
-            return [(parts.shift() || '').trim().toLowerCase(), parts.join(':').replace(/\r?\n(?=[ \t])/g, '')];
-        });
-
+    // compares the unfolded values of the header lines with that name, after decoding their encoded
+    // words. RFC 3501 and RFC 9051 section 6.4.4: [MIME-HDRS] strings in headers MUST be decoded before comparing text
     const matchHeader = (message, name, needle) => {
         name = name.toLowerCase();
-        return getHeaders(message).some(header => header[0] === name && contains(header[1], needle));
+        return (getMessageData(message).tree.header || []).some(line => {
+            const colon = line.indexOf(':');
+            const lineName = (colon < 0 ? line : line.substr(0, colon)).trim().toLowerCase();
+            const value = colon < 0 ? '' : line.substr(colon + 1).replace(/\r?\n(?=[ \t])/g, '');
+            return lineName === name && contains(decodeHeader(value), needle);
+        });
     };
 
     const matches = (node, message, index) => {

package/lib/encoded-words.js

@@ -0,0 +1,98 @@
+'use strict';
+
+// RFC 2047 encoded words in header values, decoded for SEARCH, SORT and THREAD
+
+// RFC 2047 section 2: encoded-word = "=?" charset "?" encoding "?" encoded-text "?=", RFC 2231 section 5
+// adds an optional "*" language suffix to the charset
+const ENCODED_WORD = /=\?([^?\s*]+)(?:\*[^?\s]*)?\?([BbQq])\?([^?\s]*)\?=/g;
+
+const utf8Decoder = new TextDecoder('utf-8');
+
+/**
+ * Reads a binary string (one char per octet) as UTF-8, invalid sequences become U+FFFD
+ */
+function decodeUtf8(value) {
+    return utf8Decoder.decode(Buffer.from(value, 'binary'));
+}
+
+// decoders by lower case charset name, false for a charset that TextDecoder does not know
+const decoders = new Map();
+
+/**
+ * Decodes the octets of an encoded word, or returns false if the charset is unknown
+ */
+function decodeCharset(charset, octets) {
+    charset = charset.toLowerCase();
+    if (!decoders.has(charset)) {
+        let decoder = false;
+        try {
+            decoder = new TextDecoder(charset);
+        } catch {
+            // unknown charset
+        }
+        decoders.set(charset, decoder);
+    }
+    const decoder = decoders.get(charset);
+    return decoder && decoder.decode(octets);
+}
+
+/**
+ * Decodes an RFC 2047 header value to a Unicode string. Text outside encoded words is read as UTF-8
+ * (invalid sequences become U+FFFD). Adjacent encoded words in the same charset are decoded together,
+ * so a multi-octet character may span them, and the white space between them is dropped (RFC 2047
+ * section 6.2). Encoded words in an unknown charset are kept as they are.
+ *
+ * @param {String} value Header value as a binary string
+ * @return {String} Decoded value
+ */
+function decodeHeader(value) {
+    value = (value || '').toString();
+    if (value.indexOf('=?') < 0 && !/[\u0080-\u00ff]/.test(value)) {
+        // nothing to decode
+        return value;
+    }
+    let result = '';
+    let pending = null;
+    let lastIndex = 0;
+
+    const flush = () => {
+        if (pending) {
+            const decoded = decodeCharset(pending.charset, Buffer.concat(pending.octets));
+            result += decoded === false ? pending.source : decoded;
+            pending = null;
+        }
+    };
+
+    ENCODED_WORD.lastIndex = 0;
+    let match;
+    while ((match = ENCODED_WORD.exec(value))) {
+        const between = value.substring(lastIndex, match.index);
+        const adjacent = pending && /^\s*$/.test(between);
+        if (!adjacent) {
+            flush();
+            result += decodeUtf8(between);
+        }
+        lastIndex = ENCODED_WORD.lastIndex;
+
+        const octets =
+            match[2].toUpperCase() === 'B'
+                ? Buffer.from(match[3], 'base64')
+                : Buffer.from(
+                      match[3].replace(/_/g, ' ').replace(/=([0-9a-fA-F]{2})/g, (m, hex) => String.fromCharCode(parseInt(hex, 16))),
+                      'binary'
+                  );
+
+        if (pending && pending.charset.toLowerCase() !== match[1].toLowerCase()) {
+            flush();
+        }
+        if (!pending) {
+            pending = { charset: match[1], octets: [], source: '' };
+        }
+        pending.octets.push(octets);
+        pending.source += (pending.source ? between : '') + match[0];
+    }
+    flush();
+    return result + decodeUtf8(value.substr(lastIndex));
+}
+
+module.exports = { decodeHeader, decodeUtf8 };

package/lib/plugins/auth-plain.js

@@ -2,7 +2,7 @@
 
 /**
  * @help Adds AUTH=PLAIN capability
- * @help Supports SALS-IR [RFC4959] as well
+ * @help Supports SASL-IR [RFC4959] as well
  */
 
 const utf8 = new TextDecoder('utf-8', { fatal: true });

package/lib/plugins/xtoybird.js

@@ -9,7 +9,7 @@ const { states } = require('../command-states');
  * @help  XTOYBIRD CONNECTION dumps connection internals
  * @help  XTOYBIRD STORAGE dumps storage as JSON
  * @help  XTOYBIRD USERADD "username" "password" adds or updates user
- * @help  XTOYBIRD USERDEL "username" removes an user
+ * @help  XTOYBIRD USERDEL "username" removes a user
  * @help  XTOYBIRD SHUTDOWN Closes the server after the last client
  * @help                  disconnects. New connections are rejected.
  * @help Commands are only allowed after login, and only for the owner

package/lib/sorting.js

@@ -8,88 +8,7 @@ const { processAddress } = require('./envelope');
 const { parseDateTime, parseHeaderDate, toTimestamp } = require('./dates');
 const makeSearch = require('./commands/handlers/search');
 const { badError, criteriaValues, sendSearchError } = makeSearch;
-
-// RFC 2047 section 2: encoded-word = "=?" charset "?" encoding "?" encoded-text "?=", RFC 2231 section 5
-// adds an optional "*" language suffix to the charset
-const ENCODED_WORD = /=\?([^?\s*]+)(?:\*[^?\s]*)?\?([BbQq])\?([^?\s]*)\?=/g;
-
-const utf8Decoder = new TextDecoder('utf-8');
-
-// decoders by lower case charset name, false for a charset that TextDecoder does not know
-const decoders = new Map();
-
-/**
- * Decodes the octets of an encoded word, or returns false if the charset is unknown
- */
-function decodeCharset(charset, octets) {
-    charset = charset.toLowerCase();
-    if (!decoders.has(charset)) {
-        let decoder = false;
-        try {
-            decoder = new TextDecoder(charset);
-        } catch {
-            // unknown charset
-        }
-        decoders.set(charset, decoder);
-    }
-    const decoder = decoders.get(charset);
-    return decoder && decoder.decode(octets);
-}
-
-/**
- * Decodes an RFC 2047 header value to a Unicode string. Text outside encoded words is read as UTF-8
- * (invalid sequences become U+FFFD). Adjacent encoded words in the same charset are decoded together,
- * so a multi-octet character may span them, and the white space between them is dropped (RFC 2047
- * section 6.2). Encoded words in an unknown charset are kept as they are.
- *
- * @param {String} value Header value as a binary string
- * @return {String} Decoded value
- */
-function decodeHeader(value) {
-    value = (value || '').toString();
-    let result = '';
-    let pending = null;
-    let lastIndex = 0;
-
-    const flush = () => {
-        if (pending) {
-            const decoded = decodeCharset(pending.charset, Buffer.concat(pending.octets));
-            result += decoded === false ? pending.source : decoded;
-            pending = null;
-        }
-    };
-
-    ENCODED_WORD.lastIndex = 0;
-    let match;
-    while ((match = ENCODED_WORD.exec(value))) {
-        const between = value.substring(lastIndex, match.index);
-        const adjacent = pending && /^\s*$/.test(between);
-        if (!adjacent) {
-            flush();
-            result += utf8Decoder.decode(Buffer.from(between, 'binary'));
-        }
-        lastIndex = ENCODED_WORD.lastIndex;
-
-        const octets =
-            match[2].toUpperCase() === 'B'
-                ? Buffer.from(match[3], 'base64')
-                : Buffer.from(
-                      match[3].replace(/_/g, ' ').replace(/=([0-9a-fA-F]{2})/g, (m, hex) => String.fromCharCode(parseInt(hex, 16))),
-                      'binary'
-                  );
-
-        if (pending && pending.charset.toLowerCase() !== match[1].toLowerCase()) {
-            flush();
-        }
-        if (!pending) {
-            pending = { charset: match[1], octets: [], source: '' };
-        }
-        pending.octets.push(octets);
-        pending.source += (pending.source ? between : '') + match[0];
-    }
-    flush();
-    return result + utf8Decoder.decode(Buffer.from(value.substr(lastIndex), 'binary'));
-}
+const { decodeHeader } = require('./encoded-words');
 
 // UnicodeData.txt titlecase mappings that differ from the single code point uppercase mapping
 // that String#toUpperCase gives: the Latin digraphs and the Greek letters with ypogegrammeni
@@ -361,7 +280,6 @@ function searchMessages(connection, parsed, data, attributes) {
 }
 
 module.exports = {
-    decodeHeader,
     collationKey,
     baseSubject,
     arrivalTime,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "imapkit",
-    "version": "4.0.0",
+    "version": "4.0.2",
     "description": "Scriptable, strictly RFC compliant in-memory IMAP server for testing IMAP clients",
     "main": "lib/server.js",
     "bin": {
@@ -46,7 +46,7 @@
     "author": "Postal Systems OÜ",
     "license": "MIT",
     "dependencies": {
-        "imap-handler": "1.3.1",
+        "imap-handler": "1.3.2",
         "smtp-server": "3.19.17"
     },
     "devDependencies": {