@zone-eu/mailsplit 5.4.27 → 5.4.29

package/lib/message-splitter.js

@@ -20,9 +20,17 @@ const MAX_PENDING_LINE_SIZE = 64 * 1024;
 // how many separate writes the pending line may be kept in before it is compacted
 const MAX_PENDING_LINE_CHUNKS = 1024;
 
+// The most SP and HTAB a delimiter line may carry between the boundary value, or its closing
+// "--", and the line ending. RFC 2046 5.1.1 allows the whitespace, and Python and postal-mime
+// accept any amount of it, so a line that ends in a few spaces delimits the same parts here
+// as there. The line still has to fit a header line (RFC 5322 2.1.1), so the pending line
+// kept as a possible delimiter stays bounded
+const BOUNDARY_LINE_TRAILING_WSP = 998;
+
 // what a delimiter line may carry after the boundary value: the "--" prefix, an optional
-// "--" suffix and the line terminator. This is the bound compareBoundary() accepts.
-const BOUNDARY_LINE_SUFFIX = 2 /* "--" prefix */ + 2 /* "--" suffix */ + 2; /* trailing <CR><LF> */
+// "--" suffix, the trailing whitespace and the line terminator. This is the bound
+// compareBoundary() accepts.
+const BOUNDARY_LINE_SUFFIX = 2 /* "--" prefix */ + 2 /* "--" suffix */ + BOUNDARY_LINE_TRAILING_WSP + 2; /* trailing <CR><LF> */
 
 // checkBoundary() additionally allows a line ending in front of the delimiter, so this is
 // the longest a delimiter line can ever be once the boundary value is subtracted
@@ -389,7 +397,8 @@ class MessageSplitter extends Transform {
      * @returns {1 | 2 | false}
      */
     compareBoundary(line, startpos, boundary) {
-        // --{boundary}\r\n or --{boundary}--\r\n
+        // --{boundary}\r\n or --{boundary}--\r\n, with SP and HTAB allowed in front of the
+        // line ending (RFC 2046 5.1.1), where Python and postal-mime accept them as well
         if (line.length < boundary.length + 3 + startpos || line.length > boundary.length + BOUNDARY_LINE_SUFFIX + startpos) {
             return false;
         }
@@ -399,34 +408,37 @@ class MessageSplitter extends Transform {
             }
         }
 
-        let pos = 0;
-        for (let i = boundary.length + 2 + startpos; i < line.length; i++) {
-            let c = line[i];
-            if (pos === 0 && (c === 0x0d || c === 0x0a)) {
-                // 1: next node
-                return 1;
-            }
-            if (pos === 0 && c !== 0x2d) {
-                // expecting "-"
-                return false;
-            }
-            if (pos === 1 && c !== 0x2d) {
-                // expecting "-"
-                return false;
-            }
-            if (pos === 2 && c !== 0x0d && c !== 0x0a) {
-                // expecting line terminator, either <CR> or <LF>
+        let i = boundary.length + 2 + startpos;
+        // 1: next node, 2: multipart end
+        let result = /** @type {1 | 2} */ (1);
+        if (line[i] === 0x2d) {
+            if (line[i + 1] !== 0x2d) {
+                // expecting "--"
                 return false;
             }
-            if (pos === 3 && c !== 0x0a) {
-                // expecting line terminator <LF>
-                return false;
-            }
-            pos++;
+            result = 2;
+            i += 2;
+        }
+
+        while (i < line.length && (line[i] === 0x20 || line[i] === 0x09)) {
+            i++;
         }
 
-        // 2: multipart end
-        return 2;
+        if (i === line.length) {
+            // only the closing delimiter may end the input without a line ending
+            return result === 2 ? 2 : false;
+        }
+
+        // expecting the line terminator and nothing after it: <CR><LF>, <CR> or <LF>, where
+        // the <CR> may be a run of them, the artifact of a second CR to CRLF conversion,
+        // which postal-mime reads as the line ending as well
+        while (i < line.length && line[i] === 0x0d) {
+            i++;
+        }
+        if (i < line.length && line[i] === 0x0a) {
+            i++;
+        }
+        return i === line.length ? result : false;
     }
 
     /**

package/lib/mime-node.d.ts

@@ -46,7 +46,7 @@ declare class MimeNode implements MimeNodeShape {
     /** Decoded filename from Content-Disposition or Content-Type parameters, or `false` when absent. */
     filename: string | false;
 
-    /** Whether this node is `text/*` with `format=flowed`. */
+    /** Whether this node is `text/plain` with `format=flowed`. */
     flowed: boolean;
 
     /** Whether flowed text uses `delsp=yes`. */

package/lib/mime-node.js

@@ -12,7 +12,7 @@ const pathlib = require('path');
 /** @typedef {import('..').PartNumberItem} PartNumberItem */
 /** @typedef {import('..').PartNumber} PartNumber */
 /** @typedef {import('..').SplitterOptions} SplitterOptions */
-/** @typedef {import('libmime').StructuredHeader} StructuredHeader */
+/** @typedef {import('libmime').ParsedHeaderValue} StructuredHeader */
 /** @typedef {import('stream').Transform | import('stream').PassThrough} ContentStream */
 /** @typedef {import('..').MimeNode} MimeNodeType */
 
@@ -27,6 +27,9 @@ const HEADER_LINE_MIN_COST = 64;
  * `value.replace(/\(.*\)/g, '')`, but in linear time. The regex backtracks quadratically
  * on a value with many `(` characters and no closing `)`.
  *
+ * Used for the Content-Transfer-Encoding only. The MIME headers with parameters go through
+ * libmime's parseHeaderValue with stripComments, which knows where a comment may stand.
+ *
  * @param {string} value
  * @returns {string}
  */
@@ -48,59 +51,6 @@ function stripComments(value) {
         .join('');
 }
 
-/**
- * Lists the parameters a header value gives in RFC 2231 form (`name*=`, `name*0=` or
- * `name*0*=`). libmime decodes those while parsing the header, so the result must not
- * go through `decodeWords` as well: a value like `filename*=utf-8''a%3D%3Futf-8%3FQ%3F%3D2Eexe%3F%3D`
- * would otherwise come out as `a.exe` while a parser that decodes once sees the encoded word.
- *
- * @param {string} value Raw header value.
- * @returns {Set<string>} Lowercased parameter names.
- */
-function rfc2231ParamNames(value) {
-    let names = new Set();
-    let quote = false;
-    let escaped = false;
-    let start = 0;
-
-    /**
-     * @param {number} end End offset of the segment starting at `start`.
-     * @returns {void}
-     */
-    let addSegment = end => {
-        let segment = value.substring(start, end);
-        let eq = segment.indexOf('=');
-        if (eq < 0) {
-            return;
-        }
-        let match = segment
-            .substring(0, eq)
-            .trim()
-            .toLowerCase()
-            .match(/^(.+?)\*(\d+\*?)?$/);
-        if (match) {
-            names.add(match[1]);
-        }
-    };
-
-    for (let i = 0; i < value.length; i++) {
-        let chr = value.charAt(i);
-        if (escaped) {
-            escaped = false;
-        } else if (chr === '\\') {
-            escaped = true;
-        } else if (chr === '"') {
-            quote = !quote;
-        } else if (chr === ';' && !quote) {
-            addSegment(i);
-            start = i + 1;
-        }
-    }
-    addSegment(value.length);
-
-    return names;
-}
-
 /**
  * Parsed MIME node with mutable headers and transfer-encoding helpers.
  */
@@ -216,8 +166,14 @@ class MimeNode {
         this._headersLines = [];
         let headers = this.headers;
 
+        // The comments of the MIME headers are dropped, so that `boundary=b (comment)` names
+        // the boundary `b` that postal-mime and Python's email.policy.default split by. Read as
+        // content, it matched no delimiter line and the parts both of them find stayed hidden
+        // here. Python's legacy compat32 policy, the default of message_from_bytes() without a
+        // policy, keeps the comment and splits by `--b (comment)` lines instead: a split
+        // against it is the price of agreeing with the two readers a renderer is built on
         let dispositionHeader = headers.getFirst('Content-Disposition');
-        this._parsedContentDisposition = this.libmime.parseHeaderValue(dispositionHeader);
+        this._parsedContentDisposition = this.libmime.parseHeaderValue(dispositionHeader, { stripComments: true });
         let parsedContentDisposition = /** @type {StructuredHeader} */ (this._parsedContentDisposition);
 
         // if content-type is missing default to plaintext
@@ -240,9 +196,12 @@ class MimeNode {
             }
         }
 
-        this._parsedContentType = this.libmime.parseHeaderValue(contentHeader);
+        this._parsedContentType = this.libmime.parseHeaderValue(contentHeader, { stripComments: true });
         let parsedContentType = /** @type {StructuredHeader} */ (this._parsedContentType);
 
+        // The encoding is a bare token, and one written in quotes stays quoted, so that
+        // "base64" is not decoded, as Python and postal-mime do not decode it. parseHeaderValue
+        // would unquote it, so the transfer encoding keeps its own comment stripper
         this.encoding = stripComments(headers.getFirst('Content-Transfer-Encoding')).toLowerCase().trim();
         this.contentType = (parsedContentType.value || '').toLowerCase().trim() || false;
         this.charset = parsedContentType.params.charset || false;
@@ -259,7 +218,11 @@ class MimeNode {
 
         this.filename = parsedContentDisposition.params.filename || parsedContentType.params.name || false;
 
-        if (parsedContentType.params.format && parsedContentType.params.format.toLowerCase().trim() === 'flowed') {
+        // format=flowed is defined for text/plain (RFC 3676), and a browser never unfolds an
+        // html part. Flagged on any text part, a text/html part with delsp=yes had its
+        // `<scri \r\npt>` joined into a script tag by the consumer that unfolded it, a tag
+        // neither Python nor a browser given the raw bytes sees
+        if (this.contentType === 'text/plain' && parsedContentType.params.format && parsedContentType.params.format.toLowerCase().trim() === 'flowed') {
             this.flowed = true;
             if (parsedContentType.params.delsp && parsedContentType.params.delsp.toLowerCase().trim() === 'yes') {
                 this.delSp = true;
@@ -267,10 +230,15 @@ class MimeNode {
         }
 
         if (this.filename) {
-            // a parameter given in RFC 2231 form was decoded by libmime already
+            // A parameter given in RFC 2231 form was decoded by libmime already, which names
+            // it in decodedParams, and must not go through decodeWords as well:
+            // filename*=utf-8''a%3D%3Futf-8%3FQ%3F%3D2Eexe%3F%3D would otherwise come out as
+            // a.exe while a parser that decodes once sees the encoded word. A second scan of
+            // the raw header for the name used to decide this, and read `(c)filename*=` as
+            // a parameter named `(c)filename` once the comment was dropped from the real one
             let decoded = parsedContentDisposition.params.filename
-                ? rfc2231ParamNames(dispositionHeader).has('filename')
-                : rfc2231ParamNames(contentHeader).has('name');
+                ? parsedContentDisposition.decodedParams.includes('filename')
+                : parsedContentType.decodedParams.includes('name');
             if (!decoded) {
                 try {
                     this.filename = this.libmime.decodeWords(this.filename);
@@ -285,7 +253,16 @@ class MimeNode {
                 this.contentType.substr(0, this.contentType.indexOf('/')) === 'multipart' &&
                 this.contentType.substr(this.contentType.indexOf('/') + 1)) ||
             false;
-        this._boundary = (parsedContentType.params.boundary && Buffer.from(parsedContentType.params.boundary)) || false;
+        // A boundary is ASCII and does not end in whitespace (RFC 2046 5.1.1, bchars), and
+        // Python matches no line against one that is not. One holding a non-ASCII character
+        // came out of a header read as latin1 or as UTF-8, and matched under `--ab\xC3\xA9cd`
+        // lines parts that a scanner built on Python never saw. Such a boundary counts as
+        // none, as a boundary parameter that is missing does. Trailing whitespace is dropped
+        // as Python drops it, a delimiter line may end in whitespace anyway
+        let boundary = (parsedContentType.params.boundary || '').trimEnd();
+        // a non-ASCII character takes more than one byte in UTF-8
+        let ascii = Buffer.byteLength(boundary) === boundary.length;
+        this._boundary = boundary && ascii ? Buffer.from(boundary) : false;
 
         this.rfc822 = this.contentType === 'message/rfc822';
 

package/lib/types.d.ts

@@ -107,7 +107,7 @@ export interface MimeNode {
     /** Decoded filename from Content-Disposition or Content-Type parameters, or `false` when absent. */
     filename: string | false;
 
-    /** Whether this node is `text/*` with `format=flowed`. */
+    /** Whether this node is `text/plain` with `format=flowed`. */
     flowed: boolean;
 
     /** Whether flowed text uses `delsp=yes`. */

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@zone-eu/mailsplit",
-    "version": "5.4.27",
+    "version": "5.4.29",
     "description": "Split email messages into an object stream",
     "main": "index.js",
     "types": "index.d.ts",
@@ -25,7 +25,7 @@
     "license": "(MIT OR EUPL-1.1+)",
     "dependencies": {
         "libbase64": "1.4.1",
-        "libmime": "5.5.2",
+        "libmime": "5.6.3",
         "libqp": "3.0.1"
     },
     "devDependencies": {