imapflow 1.6.5 → 1.6.6

package/lib/imap-flow.js

@@ -12,7 +12,7 @@ const logger = require('./logger');
 const libmime = require('libmime');
 const zlib = require('zlib');
 const { Headers } = require('@zone-eu/mailsplit');
-const { LimitedPassthrough } = require('./limited-passthrough');
+const { LimitedPassthrough, normalizeByteLimit } = require('./limited-passthrough');
 
 const { ImapStream } = require('./handler/imap-stream');
 const { parser, compiler } = require('./handler/imap-handler');
@@ -38,7 +38,11 @@ const {
     AuthenticationFailure,
     getColorFlags,
     hasCapability,
-    unrefTimer
+    unrefTimer,
+    parseUintValue,
+    isUnsafeKey,
+    getStringList,
+    MAX_UINT32_DIGITS
 } = require('./tools');
 
 const imapCommands = require('./imap-commands.js');
@@ -50,6 +54,10 @@ const UPGRADE_TIMEOUT = 10 * 1000;
 
 const SOCKET_TIMEOUT = 5 * 60 * 1000;
 
+// Ceiling for any throttle back-off wait. Both the connection-level back-off and the per-command
+// retries derive their delay from server-supplied hints, which are unbounded.
+const MAX_THROTTLE_DELAY = 5 * 60 * 1000;
+
 // Default threshold for warning that a mailbox lock has been held for a long
 // time. Intended to catch forgotten release() calls, not legitimate long ops
 // (e.g. fetching hundreds of thousands of messages). Configurable via the
@@ -323,17 +331,18 @@ class ImapFlow extends EventEmitter {
             logRaw: this.logRaw,
             secureConnection: this.secureConnection,
             maxLineLength: this.options.maxLineLength,
-            maxLiteralSize: this.options.maxLiteralSize
+            maxLiteralSize: this.options.maxLiteralSize,
+            maxResponseSize: this.options.maxResponseSize
         });
 
         this.reading = false;
         this.socket = false;
         this.writeSocket = false;
 
-        // Tracked throttle back-off timer (see reader()). Stored so close() can clear it
-        // and abort the wait instead of letting it keep the event loop alive for minutes.
-        this._throttleTimer = null;
-        this._throttleAbort = null;
+        // In-flight throttle back-offs (see throttleWait()). Tracked as a set because more than
+        // one can be pending at a time: the reader's connection-level back-off and a command
+        // retrying its own throttled request. close() clears them all.
+        this._throttleWaits = new Set();
 
         // Pending rejector of the in-flight STARTTLS upgrade promise (see upgradeToSTARTTLS()).
         // Stored so emitError() can route a streamer-originated error into the upgrade's single
@@ -816,6 +825,32 @@ class ImapFlow extends EventEmitter {
         }
     }
 
+    /**
+     * Waits out a throttle back-off.
+     *
+     * The delay is capped at MAX_THROTTLE_DELAY because it can come straight from a server hint
+     * (a Microsoft 365 "Suggested Backoff Time", say) and an uncapped hint would park the caller
+     * for weeks. The timer is unref'd and tracked so it can never outlive the client: a bare
+     * setTimeout here keeps a short-lived process alive for the full delay after close(), and
+     * leaves the caller waiting on a connection that is already gone.
+     *
+     * @param {Number} delay - Requested delay in milliseconds.
+     * @returns {Promise<Boolean>} True if close() aborted the wait, false on normal expiry.
+     */
+    async throttleWait(delay) {
+        delay = Math.min(Math.max(Number(delay) || 0, 0), MAX_THROTTLE_DELAY);
+
+        return await new Promise(resolve => {
+            let entry = { resolve };
+            entry.timer = setTimeout(() => {
+                this._throttleWaits.delete(entry);
+                resolve(false);
+            }, delay);
+            unrefTimer(entry.timer);
+            this._throttleWaits.add(entry);
+        });
+    }
+
     async reader() {
         let data;
         let processedCount = 0;
@@ -901,8 +936,10 @@ class ImapFlow extends EventEmitter {
         try {
             parsed = await parser(data.payload, { literals: data.literals });
         } catch (err) {
-            // can not make sense of this
-            this.log.error({ src: 's', msg: data.payload.toString(), err, cid: this.id });
+            // can not make sense of this. The payload can be up to the configured line
+            // cap (1GB by default), so log only a bounded prefix: a server looping
+            // unparseable garbage would otherwise turn this error log into a disk filler.
+            this.log.error({ src: 's', msg: data.payload.toString('latin1', 0, 1024), payloadBytes: data.payload.length, err, cid: this.id });
             // An unparseable untagged line is junk that can be skipped, but the line may
             // have been the in-flight command's tagged completion. Dropping that one
             // silently strands the command: currentRequest is never cleared, so trySend()
@@ -974,7 +1011,9 @@ class ImapFlow extends EventEmitter {
         }
 
         let section = parsed.attributes && parsed.attributes.length && parsed.attributes[0] && !parsed.attributes[0].value && parsed.attributes[0].section;
-        if (section && section.length && section[0].type === 'ATOM' && typeof section[0].value === 'string') {
+        // section[0] can be a parsed NIL (null), e.g. from a "[NIL]" response code - the
+        // dereference must be guarded or one such line tears down the whole connection
+        if (section && section.length && section[0] && section[0].type === 'ATOM' && typeof section[0].value === 'string') {
             let sectionHandler = this.getSectionHandler(section[0].value.toUpperCase().trim());
             if (sectionHandler) {
                 try {
@@ -1124,27 +1163,12 @@ class ImapFlow extends EventEmitter {
                         err.code = 'ETHROTTLE';
                         err.throttleReset = throttleDelay;
 
-                        let delayResponse = throttleDelay;
-                        if (delayResponse > 5 * 60 * 1000) {
-                            // Cap wait at 5 minutes to avoid hanging connections indefinitely.
-                            // The server-suggested delay can be very large.
-                            delayResponse = 5 * 60 * 1000;
-                        }
+                        // The server-suggested delay can be very large, so throttleWait() caps it
+                        let delayResponse = Math.min(throttleDelay, MAX_THROTTLE_DELAY);
 
                         this.log.warn({ msg: 'Throttling detected', cid: this.id, throttleDelay, delayResponse, err });
 
-                        // Tracked, abortable wait. Storing the timer lets close() clear it so
-                        // the back-off never keeps the event loop alive, and storing the resolve
-                        // lets close() abort the wait promptly (aborted=true) instead of blocking
-                        // the reader for up to 5 minutes and rejecting long after the connection
-                        // is gone. Normal expiry resolves with aborted=false.
-                        let aborted = await new Promise(resolve => {
-                            this._throttleAbort = resolve;
-                            this._throttleTimer = setTimeout(() => resolve(false), delayResponse);
-                            unrefTimer(this._throttleTimer);
-                        });
-                        this._throttleTimer = null;
-                        this._throttleAbort = null;
+                        let aborted = await this.throttleWait(delayResponse);
 
                         if (aborted) {
                             // Connection closed during back-off: reject promptly with a
@@ -1571,6 +1595,12 @@ class ImapFlow extends EventEmitter {
             let opts = Object.assign(
                 {
                     socket: this.socket,
+                    // host is required even though the socket is already connected: without
+                    // it, a connection made to an IP literal (servername=false) has its
+                    // certificate verified against Node's fallback name "localhost" instead
+                    // of the IP - accepting any "localhost" certificate for any IP-hosted
+                    // server, and rejecting legitimate IP-SAN certificates.
+                    host: this.host,
                     servername: this.servername,
                     port: this.port
                 },
@@ -1894,11 +1924,18 @@ class ImapFlow extends EventEmitter {
             return;
         }
 
-        if (!untagged || !untagged.command || isNaN(untagged.command)) {
+        if (!untagged) {
             return;
         }
 
-        let count = Number(untagged.command);
+        // Not a usable count: anything but a bounded digit run. A digit run long enough
+        // coerces to Infinity, which would corrupt mailbox state (resolveRange('*') would
+        // compile to the literal "Infinity" and every range-based command would fail until
+        // the next SELECT)
+        let count = parseUintValue(untagged.command, MAX_UINT32_DIGITS);
+        if (count === false) {
+            return;
+        }
         if (count === this.mailbox.exists) {
             // nothing changed?
             return;
@@ -1920,11 +1957,12 @@ class ImapFlow extends EventEmitter {
             return;
         }
 
-        if (!untagged || !untagged.command || isNaN(untagged.command)) {
+        if (!untagged) {
             return;
         }
 
-        let seq = Number(untagged.command);
+        // Same bound untaggedExists() applies: only a bounded decimal run is a usable sequence number
+        let seq = parseUintValue(untagged.command, MAX_UINT32_DIGITS);
         if (seq && seq <= this.mailbox.exists) {
             this.mailbox.exists--;
             let payload = {
@@ -1955,8 +1993,14 @@ class ImapFlow extends EventEmitter {
         let tags = [];
         let uids = false;
 
+        // A malformed VANISHED can carry no attributes at all, and one carrying only the
+        // (EARLIER) tag leaves `uids` false - expandRange() handles that and yields nothing
+        if (!untagged.attributes || !untagged.attributes.length) {
+            return;
+        }
+
         if (untagged.attributes.length > 1 && Array.isArray(untagged.attributes[0])) {
-            tags = untagged.attributes[0].map(entry => (typeof entry.value === 'string' ? entry.value.toUpperCase() : false)).filter(value => value);
+            tags = getStringList(untagged.attributes[0]).map(value => value.toUpperCase());
             untagged.attributes.shift();
         }
 
@@ -2320,14 +2364,13 @@ class ImapFlow extends EventEmitter {
             clearTimeout(this.connectTimeout);
             clearTimeout(this.greetingTimeout);
 
-            // Abort any in-flight throttle back-off so the reader unblocks and the
-            // throttled request is rejected promptly rather than after the full delay.
-            clearTimeout(this._throttleTimer);
-            this._throttleTimer = null;
-            if (typeof this._throttleAbort === 'function') {
-                this._throttleAbort(true);
-                this._throttleAbort = null;
+            // Abort every in-flight throttle back-off so each waiter unblocks and its request is
+            // settled promptly rather than after the full delay.
+            for (let entry of this._throttleWaits) {
+                clearTimeout(entry.timer);
+                entry.resolve(true);
             }
+            this._throttleWaits.clear();
 
             this.usable = false;
             // close() takes over ownership of the idling state: dropping the session token means a
@@ -3593,7 +3636,8 @@ class ImapFlow extends EventEmitter {
         let processed = 0;
 
         let chunkSize = Number(options.chunkSize) || 64 * 1024;
-        let maxBytes = Number(options.maxBytes) || Infinity;
+        // Normalized once here so every bounded stage of the pipeline below agrees on the budget
+        let maxBytes = normalizeByteLimit(options.maxBytes);
 
         let uid = false;
 
@@ -3783,24 +3827,43 @@ class ImapFlow extends EventEmitter {
                 output = stream = new PassThrough();
         }
 
+        // Every byte-bounded stage of the pipeline. The fetch loop below stops as soon as any of
+        // them has taken all it will accept. The limiter at the tail is not enough on its own: a
+        // transform in the middle that buffers its whole input before emitting anything (the
+        // format=flowed decoder, the Japanese charset decoder) leaves the tail limiter reporting
+        // `limited === false` however much the server sends, so a download with a small maxBytes
+        // would still pull the entire part off the wire.
+        let limiters = [];
+        let isLimited = () => limiters.some(entry => entry.limited);
+
+        // Appending a stage means forwarding the current tail's errors to it before piping, so a
+        // failure anywhere reaches the stream the caller is reading
+        let pipeStage = stage => {
+            output.on('error', err => {
+                stage.emit('error', err);
+            });
+            output = output.pipe(stage);
+            return stage;
+        };
+
         let isTextNode = ['text/html', 'text/plain', 'text/x-amp-html'].includes(meta.contentType) || (part === '1' && !meta.contentType);
         if ((!meta.disposition || meta.disposition === 'inline') && isTextNode) {
             // RFC 3676 format=flowed text: unwrap soft line breaks
             if (meta.flowed) {
-                let flowDecoder = new FlowedDecoder({
-                    delSp: meta.delSp
-                });
-                output.on('error', err => {
-                    flowDecoder.emit('error', err);
-                });
-                output = output.pipe(flowDecoder);
+                // FlowedDecoder buffers its whole input before emitting, and being third party it
+                // carries no bound of its own, so bound what it can ever be handed. Unwrapping only
+                // removes bytes, so capping its input at maxBytes cannot push the delivered output
+                // above the cap either.
+                limiters.push(pipeStage(new LimitedPassthrough({ maxBytes })));
+
+                pipeStage(new FlowedDecoder({ delSp: meta.delSp }));
             }
 
             // Convert non-UTF-8 charsets to UTF-8 via a streaming decoder.
             // ASCII and UTF-8 need no conversion. Unknown charsets are left as-is.
             if (meta.charset && !['ascii', 'usascii', 'utf8'].includes(meta.charset.toLowerCase().replace(/[^a-z0-9]+/g, ''))) {
                 try {
-                    let decoder = getDecoder(meta.charset);
+                    let decoder = getDecoder(meta.charset, maxBytes);
                     // Safety listener attached first so the decoder always has at least
                     // one 'error' listener. Prevents Node.js from throwing
                     // ERR_UNHANDLED_ERROR if a later pipe setup step throws and leaves
@@ -3810,10 +3873,10 @@ class ImapFlow extends EventEmitter {
                     decoder.on('error', err => {
                         this.log.warn({ err, charset: meta.charset, cid: this.id });
                     });
-                    output.on('error', err => {
-                        decoder.emit('error', err);
-                    });
-                    output = output.pipe(decoder);
+                    // The Japanese decoder buffers its whole input as well, and reports the same
+                    // `limited` flag the limiters do so the fetch loop can stop once it is full.
+                    // A streaming decoder has no such flag, which reads as false and is correct.
+                    limiters.push(pipeStage(decoder));
                     // force to utf-8 for output
                     meta.charset = 'utf-8';
                 } catch {
@@ -3822,11 +3885,8 @@ class ImapFlow extends EventEmitter {
             }
         }
 
-        let limiter = new LimitedPassthrough({ maxBytes });
-        output.on('error', err => {
-            limiter.emit('error', err);
-        });
-        output = output.pipe(limiter);
+        let limiter = pipeStage(new LimitedPassthrough({ maxBytes }));
+        limiters.push(limiter);
 
         // Cleanup function
         const cleanup = () => {
@@ -3841,7 +3901,7 @@ class ImapFlow extends EventEmitter {
         output.once('close', cleanup);
 
         let writeChunk = chunk => {
-            if (limiter.limited || fetchAborted || stream.destroyed) {
+            if (isLimited() || fetchAborted || stream.destroyed) {
                 return true;
             }
             return stream.write(chunk);
@@ -3851,7 +3911,7 @@ class ImapFlow extends EventEmitter {
         // Stops when the server returns a short chunk (< chunkSize), the byte
         // limiter is satisfied, or the consumer destroys the output stream.
         let fetchAllParts = async () => {
-            while (hasMore && !limiter.limited && !fetchAborted) {
+            while (hasMore && !isLimited() && !fetchAborted) {
                 let { chunk } = await getNextPart();
                 if (!chunk || fetchAborted) {
                     break;
@@ -4012,6 +4072,12 @@ class ImapFlow extends EventEmitter {
 
         for (let [part, content] of response.bodyParts) {
             let keyParts = part.split('.mime');
+            // The server chooses the BODY[...] keys it answers with: never let one be a
+            // prototype-chain name, or the assignments below write onto Object.prototype
+            // (process-wide pollution) instead of the result object.
+            if (isUnsafeKey(keyParts[0])) {
+                continue;
+            }
             if (keyParts.length === 1) {
                 // content
                 let key = keyParts[0];
@@ -4080,7 +4146,11 @@ class ImapFlow extends EventEmitter {
         }
 
         for (let part of Object.keys(data)) {
-            let meta = data[part].meta;
+            // `meta` is only built from the companion BODY[<part>.MIME] item. A server may
+            // legally answer with fewer items than were requested, and one part arriving
+            // without its MIME headers must not cost the caller the whole download.
+            let meta = data[part].meta || {};
+            data[part].meta = meta;
 
             // parts that arrived via FETCH BINARY (response.binaryParts) are already
             // decoded by the server - decoding again would corrupt the data

package/lib/jp-decoder.js

@@ -2,6 +2,7 @@
 
 const { Transform } = require('stream');
 const encodingJapanese = require('encoding-japanese');
+const { normalizeByteLimit } = require('./limited-passthrough.js');
 
 // A Transform stream for decoding Japanese character sets (Shift_JIS, EUC-JP, ISO-2022-JP).
 // Unlike iconv-lite which can decode incrementally, encoding-japanese requires the complete
@@ -9,23 +10,47 @@ const encodingJapanese = require('encoding-japanese');
 // which uses escape sequences to switch between ASCII and multi-byte modes). Therefore,
 // this stream buffers all input during _transform and performs the actual decoding in _flush.
 class JPDecoder extends Transform {
-    constructor(charset) {
+    constructor(charset, maxBytes) {
         super();
 
         this.charset = charset;
         this.chunks = [];
         this.chunklen = 0;
+
+        // Upper bound for the buffered bytes, normalized the same way LimitedPassthrough
+        // normalizes its own. The whole-input buffering defeats a downstream maxBytes limiter
+        // (nothing is emitted until _flush), so without an internal bound a server could force
+        // unbounded memory use through a caller that asked for a limited download. Excess input
+        // is truncated, mirroring the truncation a maxBytes download applies anyway.
+        this.maxBytes = normalizeByteLimit(maxBytes);
+
+        // Also mirroring LimitedPassthrough: true once the bound is reached and every further
+        // chunk is being discarded. The download loop reads this to stop pulling from the
+        // server, which the limiter at the tail of the pipeline cannot tell it, because nothing
+        // is emitted from here until _flush().
+        this.limited = false;
     }
 
-    // Buffer all incoming chunks; no decoding happens here because Japanese charsets
-    // require the complete input for accurate conversion.
+    // Buffer all incoming chunks (up to maxBytes); no decoding happens here because
+    // Japanese charsets require the complete input for accurate conversion.
     _transform(chunk, encoding, done) {
         if (typeof chunk === 'string') {
             chunk = Buffer.from(chunk, encoding);
         }
 
-        this.chunks.push(chunk);
-        this.chunklen += chunk.length;
+        if (this.chunklen + chunk.length > this.maxBytes) {
+            chunk = chunk.slice(0, Math.max(0, this.maxBytes - this.chunklen));
+        }
+
+        if (chunk.length) {
+            this.chunks.push(chunk);
+            this.chunklen += chunk.length;
+        }
+
+        if (this.chunklen >= this.maxBytes) {
+            this.limited = true;
+        }
+
         done();
     }
 

package/lib/limited-passthrough.js

@@ -2,6 +2,23 @@
 
 const { Transform } = require('stream');
 
+/**
+ * Normalizes a byte budget for the download pipeline. Any finite positive number is honored and
+ * floored, because byte counts are integers: with a fractional bound a counter can only ever
+ * reach its floor, so a stage would never report itself full and a loop polling that flag would
+ * keep pulling forever. Anything else - 0, NaN, a non-numeric value - means "no limit".
+ *
+ * Lives here rather than in tools.js because tools.js requires jp-decoder.js, which needs this.
+ *
+ * @param {*} value - The configured budget.
+ * @returns {Number} The normalized budget, or Infinity when unbounded.
+ */
+const normalizeByteLimit = value => {
+    let bytes = Number(value);
+    // Math.max keeps a sub-1 budget from flooring to 0, which would read back as "no limit"
+    return Number.isFinite(bytes) && bytes > 0 ? Math.max(Math.floor(bytes), 1) : Infinity;
+};
+
 // A Transform stream that passes through data up to a maximum byte limit,
 // then silently discards all subsequent chunks. Used to enforce download
 // size limits when fetching message content from the IMAP server.
@@ -9,7 +26,7 @@ class LimitedPassthrough extends Transform {
     constructor(options) {
         super();
         this.options = options || {};
-        this.maxBytes = this.options.maxBytes || Infinity;
+        this.maxBytes = normalizeByteLimit(this.options.maxBytes);
         this.processed = 0;
         // Once set to true, all subsequent chunks are dropped without error
         this.limited = false;
@@ -42,3 +59,4 @@ class LimitedPassthrough extends Transform {
 }
 
 module.exports.LimitedPassthrough = LimitedPassthrough;
+module.exports.normalizeByteLimit = normalizeByteLimit;