imapflow 1.6.5 → 1.7.0

package/lib/imap-flow.js

@@ -12,7 +12,7 @@ const logger = require('./logger');
 const libmime = require('libmime');
 const zlib = require('zlib');
 const { Headers } = require('@zone-eu/mailsplit');
-const { LimitedPassthrough } = require('./limited-passthrough');
+const { LimitedPassthrough, normalizeByteLimit } = require('./limited-passthrough');
 
 const { ImapStream } = require('./handler/imap-stream');
 const { parser, compiler } = require('./handler/imap-handler');
@@ -38,7 +38,11 @@ const {
     AuthenticationFailure,
     getColorFlags,
     hasCapability,
-    unrefTimer
+    unrefTimer,
+    parseUintValue,
+    isUnsafeKey,
+    getStringList,
+    MAX_UINT32_DIGITS
 } = require('./tools');
 
 const imapCommands = require('./imap-commands.js');
@@ -50,12 +54,29 @@ const UPGRADE_TIMEOUT = 10 * 1000;
 
 const SOCKET_TIMEOUT = 5 * 60 * 1000;
 
+// Ceiling for any throttle back-off wait. Both the connection-level back-off and the per-command
+// retries derive their delay from server-supplied hints, which are unbounded.
+const MAX_THROTTLE_DELAY = 5 * 60 * 1000;
+
 // Default threshold for warning that a mailbox lock has been held for a long
 // time. Intended to catch forgotten release() calls, not legitimate long ops
 // (e.g. fetching hundreds of thousands of messages). Configurable via the
 // ImapFlow constructor option `maxLockHoldTime`. Set to 0 or false to disable.
 const HELD_LOCK_WARN_MS = 30 * 60 * 1000;
 
+// How long the connection has to stay inactive before auto-IDLE starts. Long enough that a caller
+// running a sequence of commands is not interrupted by an IDLE it immediately has to break.
+// Configurable via the ImapFlow constructor option `autoIdleDelay`.
+const AUTO_IDLE_DELAY = 15 * 1000;
+
+// Headroom kept between the auto-IDLE delay and the socket inactivity watchdog, so IDLE reaches
+// the wire before the watchdog can fire. See normalizeAutoIdleDelay().
+const AUTO_IDLE_SOCKET_MARGIN = 1000;
+
+// The largest delay setTimeout can honor (2^31 - 1 ms). Anything above fires after 1 ms instead,
+// so the auto-IDLE delay cap has to stay inside this range even when socketTimeout is not.
+const MAX_TIMER_DELAY = 2 ** 31 - 1;
+
 const states = {
     NOT_AUTHENTICATED: 0x01,
     AUTHENTICATED: 0x02,
@@ -63,6 +84,49 @@ const states = {
     LOGOUT: 0x04
 };
 
+/**
+ * Normalizes the configured auto-IDLE delay into a value `setTimeout` can honor. Anything Node
+ * would silently turn into a 1ms timer - NaN, a negative number, a value above the 32-bit range -
+ * falls back to the default instead, because a 1ms delay means an IDLE/DONE round trip around
+ * every single command. The delay is also capped below `socketTimeout`, see AUTO_IDLE_SOCKET_MARGIN.
+ *
+ * @param {*} value - The configured `autoIdleDelay` option.
+ * @param {Number} socketTimeout - The normalized socket inactivity timeout.
+ * @param {Object} log - Logger, used to report a value that could not be used as given.
+ * @param {String} cid - Connection id for the log entry.
+ * @returns {Number} Delay in milliseconds.
+ */
+const normalizeAutoIdleDelay = (value, socketTimeout, log, cid) => {
+    const maxDelay = Math.max(0, Math.min(socketTimeout, MAX_TIMER_DELAY) - AUTO_IDLE_SOCKET_MARGIN);
+    const configured = value !== undefined && value !== null;
+
+    // Numeric strings are accepted, because configuration usually arrives from an environment
+    // variable or a JSON file. Booleans and blank strings are not: Number() would read them as 0,
+    // i.e. "IDLE around every command", the opposite of the "off" they suggest.
+    let delay = typeof value === 'number' || (typeof value === 'string' && value.trim()) ? Number(value) : NaN;
+    let reason = null;
+
+    if (!Number.isFinite(delay) || delay < 0) {
+        reason = 'not a non-negative finite number';
+        delay = AUTO_IDLE_DELAY;
+    }
+
+    if (delay > maxDelay) {
+        // An invalid value keeps its own reason: the cap then applies to the fallback default,
+        // not to anything the caller asked for.
+        reason = reason || `above socketTimeout (${socketTimeout} ms)`;
+        delay = maxDelay;
+    }
+
+    // Only an explicitly configured value is worth warning about. Capping the default because the
+    // caller picked a short socketTimeout is expected behavior, not a misconfiguration.
+    if (configured && reason) {
+        log.warn({ msg: 'Adjusted unusable autoIdleDelay option', requested: value, autoIdleDelay: delay, reason, cid });
+    }
+
+    return Math.floor(delay);
+};
+
 /**
  * @typedef {Object} MailboxObject
  * @global
@@ -180,6 +244,15 @@ class ImapFlow extends EventEmitter {
      * @property {Boolean} [disableAutoIdle=false]
      *     If `true`, do not start IDLE automatically. Useful when only specific operations are needed.
      *
+     * @property {Number} [autoIdleDelay=15000]
+     *     How long (in milliseconds) the connection has to be inactive before IDLE is started automatically.
+     *     Keep it above the pause your own code usually leaves between two commands, otherwise every command is
+     *     followed by an IDLE that the next command has to break, costing two extra round-trips per command.
+     *     To turn auto-IDLE off entirely use `disableAutoIdle` rather than a very large delay: the value is
+     *     capped below `socketTimeout`, because auto-IDLE has to start before the inactivity watchdog fires.
+     *     On servers without IDLE support this controls when the polling fallback starts, not how often it
+     *     polls - the poll interval is `maxIdleTime`, capped at 2 minutes.
+     *
      * @property {Object} [tls]
      *     Additional TLS options. For details, see [Node.js TLS connect](https://nodejs.org/api/tls.html#tls_tls_connect_options_callback).
      *
@@ -323,17 +396,18 @@ class ImapFlow extends EventEmitter {
             logRaw: this.logRaw,
             secureConnection: this.secureConnection,
             maxLineLength: this.options.maxLineLength,
-            maxLiteralSize: this.options.maxLiteralSize
+            maxLiteralSize: this.options.maxLiteralSize,
+            maxResponseSize: this.options.maxResponseSize
         });
 
         this.reading = false;
         this.socket = false;
         this.writeSocket = false;
 
-        // Tracked throttle back-off timer (see reader()). Stored so close() can clear it
-        // and abort the wait instead of letting it keep the event loop alive for minutes.
-        this._throttleTimer = null;
-        this._throttleAbort = null;
+        // In-flight throttle back-offs (see throttleWait()). Tracked as a set because more than
+        // one can be pending at a time: the reader's connection-level back-off and a command
+        // retrying its own throttled request. close() clears them all.
+        this._throttleWaits = new Set();
 
         // Pending rejector of the in-flight STARTTLS upgrade promise (see upgradeToSTARTTLS()).
         // Stored so emitError() can route a streamer-originated error into the upgrade's single
@@ -429,6 +503,14 @@ class ImapFlow extends EventEmitter {
         this.idRequested = false;
 
         this.maxIdleTime = this.options.maxIdleTime || false;
+        this.autoIdleDelay = normalizeAutoIdleDelay(this.options.autoIdleDelay, this.socketTimeout, this.log, this.id);
+
+        // Wall-clock time of the last fallback poll, owned by lib/commands/idle.js
+        this._lastPollAt = 0;
+
+        // Download streams still fetching chunks. Counted, not a flag, so overlapping downloads
+        // cannot clear each other's suppression of auto-IDLE.
+        this._openDownloads = 0;
         this.missingIdleCommand = (this.options.missingIdleCommand || '').toString().toUpperCase().trim() || 'NOOP';
 
         this.disableBinary = !!this.options.disableBinary;
@@ -816,6 +898,32 @@ class ImapFlow extends EventEmitter {
         }
     }
 
+    /**
+     * Waits out a throttle back-off.
+     *
+     * The delay is capped at MAX_THROTTLE_DELAY because it can come straight from a server hint
+     * (a Microsoft 365 "Suggested Backoff Time", say) and an uncapped hint would park the caller
+     * for weeks. The timer is unref'd and tracked so it can never outlive the client: a bare
+     * setTimeout here keeps a short-lived process alive for the full delay after close(), and
+     * leaves the caller waiting on a connection that is already gone.
+     *
+     * @param {Number} delay - Requested delay in milliseconds.
+     * @returns {Promise<Boolean>} True if close() aborted the wait, false on normal expiry.
+     */
+    async throttleWait(delay) {
+        delay = Math.min(Math.max(Number(delay) || 0, 0), MAX_THROTTLE_DELAY);
+
+        return await new Promise(resolve => {
+            let entry = { resolve };
+            entry.timer = setTimeout(() => {
+                this._throttleWaits.delete(entry);
+                resolve(false);
+            }, delay);
+            unrefTimer(entry.timer);
+            this._throttleWaits.add(entry);
+        });
+    }
+
     async reader() {
         let data;
         let processedCount = 0;
@@ -901,8 +1009,10 @@ class ImapFlow extends EventEmitter {
         try {
             parsed = await parser(data.payload, { literals: data.literals });
         } catch (err) {
-            // can not make sense of this
-            this.log.error({ src: 's', msg: data.payload.toString(), err, cid: this.id });
+            // can not make sense of this. The payload can be up to the configured line
+            // cap (1GB by default), so log only a bounded prefix: a server looping
+            // unparseable garbage would otherwise turn this error log into a disk filler.
+            this.log.error({ src: 's', msg: data.payload.toString('latin1', 0, 1024), payloadBytes: data.payload.length, err, cid: this.id });
             // An unparseable untagged line is junk that can be skipped, but the line may
             // have been the in-flight command's tagged completion. Dropping that one
             // silently strands the command: currentRequest is never cleared, so trySend()
@@ -974,7 +1084,9 @@ class ImapFlow extends EventEmitter {
         }
 
         let section = parsed.attributes && parsed.attributes.length && parsed.attributes[0] && !parsed.attributes[0].value && parsed.attributes[0].section;
-        if (section && section.length && section[0].type === 'ATOM' && typeof section[0].value === 'string') {
+        // section[0] can be a parsed NIL (null), e.g. from a "[NIL]" response code - the
+        // dereference must be guarded or one such line tears down the whole connection
+        if (section && section.length && section[0] && section[0].type === 'ATOM' && typeof section[0].value === 'string') {
             let sectionHandler = this.getSectionHandler(section[0].value.toUpperCase().trim());
             if (sectionHandler) {
                 try {
@@ -1124,27 +1236,12 @@ class ImapFlow extends EventEmitter {
                         err.code = 'ETHROTTLE';
                         err.throttleReset = throttleDelay;
 
-                        let delayResponse = throttleDelay;
-                        if (delayResponse > 5 * 60 * 1000) {
-                            // Cap wait at 5 minutes to avoid hanging connections indefinitely.
-                            // The server-suggested delay can be very large.
-                            delayResponse = 5 * 60 * 1000;
-                        }
+                        // The server-suggested delay can be very large, so throttleWait() caps it
+                        let delayResponse = Math.min(throttleDelay, MAX_THROTTLE_DELAY);
 
                         this.log.warn({ msg: 'Throttling detected', cid: this.id, throttleDelay, delayResponse, err });
 
-                        // Tracked, abortable wait. Storing the timer lets close() clear it so
-                        // the back-off never keeps the event loop alive, and storing the resolve
-                        // lets close() abort the wait promptly (aborted=true) instead of blocking
-                        // the reader for up to 5 minutes and rejecting long after the connection
-                        // is gone. Normal expiry resolves with aborted=false.
-                        let aborted = await new Promise(resolve => {
-                            this._throttleAbort = resolve;
-                            this._throttleTimer = setTimeout(() => resolve(false), delayResponse);
-                            unrefTimer(this._throttleTimer);
-                        });
-                        this._throttleTimer = null;
-                        this._throttleAbort = null;
+                        let aborted = await this.throttleWait(delayResponse);
 
                         if (aborted) {
                             // Connection closed during back-off: reject promptly with a
@@ -1228,10 +1325,22 @@ class ImapFlow extends EventEmitter {
         /**
          * Socket timeout event handler.
          *
-         * When a socket timeout occurs during IDLE, the handler attempts to recover the connection
-         * by sending a NOOP command and then returning to IDLE state.
+         * A quiet socket is only a dead connection when something was supposed to be talking. An
+         * idling session, a download whose consumer stopped draining, and a held mailbox lock
+         * whose owner is busy between commands are all expected to go quiet, so the handler keeps
+         * such a connection alive with a NOOP instead of tearing it down. An in-flight command is
+         * the opposite: its reply is overdue, a recovery NOOP would only queue up behind it and
+         * never reach the wire, so the timeout is reported as an error. The IDLE command itself is
+         * the one exception - it stays in flight for as long as idling lasts, and run() breaks it
+         * through preCheck() before the NOOP is dispatched.
+         *
+         * IDLE is not restarted here: run() re-arms auto-IDLE once the NOOP settles, and
+         * autoidle() knows whether the connection is actually free for IDLE - an open download or
+         * a held lock keeps just the keepalive, and with disableAutoIdle nothing restarts at all.
+         * If the server is dead the NOOP never settles, and the next timeout fires with the NOOP
+         * as the stuck in-flight command, which lands in the error branch below.
          *
-         * @fires ImapFlow#error Emits error event unless the current command is IDLE
+         * @fires ImapFlow#error Emits error event if the connection cannot be recovered
          */
         this._socketTimeout =
             this._socketTimeout ||
@@ -1239,26 +1348,21 @@ class ImapFlow extends EventEmitter {
                 const err = new Error('Socket timeout');
                 err.code = 'ETIMEOUT';
 
-                if (this.idling) {
+                const quietExpected = this.idling || this._openDownloads || this.currentLock;
+                const commandStuck = this.currentRequest && !(this.idling && this.currentRequest.command === 'IDLE');
+
+                if (quietExpected && !commandStuck) {
                     if (!this.usable || !this.socket || this.socket.destroyed) {
                         this.emitError(err);
                         return;
                     }
-                    // Attempt to recover IDLE connections. During true IDLE the NOOP cannot
-                    // reach the server until IDLE has been terminated: run() awaits preCheck(),
-                    // which sends DONE and only resolves once the server has completed the IDLE
-                    // command. Fallback polling has no such handshake - preCheck() there just
-                    // cancels the polling session.
-                    this.run('NOOP')
-                        .then(() => this.idle())
-                        .catch(err => {
-                            this.log.warn({ msg: 'IDLE recovery failed after timeout', err, cid: this.id });
-                            if (!this.isClosed) {
-                                this.close();
-                            }
-                        });
+                    this.run('NOOP').catch(err => {
+                        this.log.warn({ msg: 'Connection recovery failed after timeout', err, cid: this.id });
+                        if (!this.isClosed) {
+                            this.close();
+                        }
+                    });
                 } else {
-                    // Close immediately for non-IDLE operations
                     this.log.debug({ msg: 'Socket timeout', cid: this.id });
                     this.emitError(err);
                 }
@@ -1571,6 +1675,12 @@ class ImapFlow extends EventEmitter {
             let opts = Object.assign(
                 {
                     socket: this.socket,
+                    // host is required even though the socket is already connected: without
+                    // it, a connection made to an IP literal (servername=false) has its
+                    // certificate verified against Node's fallback name "localhost" instead
+                    // of the IP - accepting any "localhost" certificate for any IP-hosted
+                    // server, and rejecting legitimate IP-SAN certificates.
+                    host: this.host,
                     servername: this.servername,
                     port: this.port
                 },
@@ -1894,11 +2004,18 @@ class ImapFlow extends EventEmitter {
             return;
         }
 
-        if (!untagged || !untagged.command || isNaN(untagged.command)) {
+        if (!untagged) {
             return;
         }
 
-        let count = Number(untagged.command);
+        // Not a usable count: anything but a bounded digit run. A digit run long enough
+        // coerces to Infinity, which would corrupt mailbox state (resolveRange('*') would
+        // compile to the literal "Infinity" and every range-based command would fail until
+        // the next SELECT)
+        let count = parseUintValue(untagged.command, MAX_UINT32_DIGITS);
+        if (count === false) {
+            return;
+        }
         if (count === this.mailbox.exists) {
             // nothing changed?
             return;
@@ -1920,11 +2037,12 @@ class ImapFlow extends EventEmitter {
             return;
         }
 
-        if (!untagged || !untagged.command || isNaN(untagged.command)) {
+        if (!untagged) {
             return;
         }
 
-        let seq = Number(untagged.command);
+        // Same bound untaggedExists() applies: only a bounded decimal run is a usable sequence number
+        let seq = parseUintValue(untagged.command, MAX_UINT32_DIGITS);
         if (seq && seq <= this.mailbox.exists) {
             this.mailbox.exists--;
             let payload = {
@@ -1955,8 +2073,14 @@ class ImapFlow extends EventEmitter {
         let tags = [];
         let uids = false;
 
+        // A malformed VANISHED can carry no attributes at all, and one carrying only the
+        // (EARLIER) tag leaves `uids` false - expandRange() handles that and yields nothing
+        if (!untagged.attributes || !untagged.attributes.length) {
+            return;
+        }
+
         if (untagged.attributes.length > 1 && Array.isArray(untagged.attributes[0])) {
-            tags = untagged.attributes[0].map(entry => (typeof entry.value === 'string' ? entry.value.toUpperCase() : false)).filter(value => value);
+            tags = getStringList(untagged.attributes[0]).map(value => value.toUpperCase());
             untagged.attributes.shift();
         }
 
@@ -2076,6 +2200,16 @@ class ImapFlow extends EventEmitter {
         return range;
     }
 
+    // The single definition of "the connection is not free". A held or queued mailbox lock, a
+    // command in flight or queued, and an open download stream all mean a caller is
+    // mid-sequence: starting IDLE there injects an IDLE/DONE round trip - or, with
+    // `missingIdleCommand` set to SELECT or STATUS, a mailbox poll - between two of that
+    // caller's own commands. Every one of those states ends by calling autoidle() again, so
+    // declining while busy postpones IDLE, it never cancels it.
+    connectionBusy() {
+        return !!(this.currentLock || this.locks.length || this.currentRequest || this.requestQueue.length || this._openDownloads);
+    }
+
     // Timer process-liveness policy: connection establishment and greeting deadlines keep the
     // process alive, because a caller is waiting on connect() to settle. Background timers
     // (auto-IDLE, IDLE restart, fallback polling, throttle back-off, the held-lock diagnostic) are
@@ -2086,9 +2220,22 @@ class ImapFlow extends EventEmitter {
         if (this.options.disableAutoIdle || this.state !== this.states.SELECTED) {
             return;
         }
+
+        if (this.connectionBusy()) {
+            return;
+        }
+
         this.idleStartTimer = setTimeout(() => {
+            // Re-checked at fire time: paths that take ownership of the connection clear this
+            // timer, but the guard must not depend on every one of them doing so - a single
+            // missed clearTimeout would inject IDLE between a caller's own commands. Declining
+            // postpones rather than cancels: whatever made the connection busy calls autoidle()
+            // again when it finishes.
+            if (this.state !== this.states.SELECTED || this.connectionBusy()) {
+                return;
+            }
             this.idle().catch(err => this.log.warn({ err, cid: this.id }));
-        }, 15 * 1000);
+        }, this.autoIdleDelay);
         unrefTimer(this.idleStartTimer);
     }
 
@@ -2320,14 +2467,13 @@ class ImapFlow extends EventEmitter {
             clearTimeout(this.connectTimeout);
             clearTimeout(this.greetingTimeout);
 
-            // Abort any in-flight throttle back-off so the reader unblocks and the
-            // throttled request is rejected promptly rather than after the full delay.
-            clearTimeout(this._throttleTimer);
-            this._throttleTimer = null;
-            if (typeof this._throttleAbort === 'function') {
-                this._throttleAbort(true);
-                this._throttleAbort = null;
+            // Abort every in-flight throttle back-off so each waiter unblocks and its request is
+            // settled promptly rather than after the full delay.
+            for (let entry of this._throttleWaits) {
+                clearTimeout(entry.timer);
+                entry.resolve(true);
             }
+            this._throttleWaits.clear();
 
             this.usable = false;
             // close() takes over ownership of the idling state: dropping the session token means a
@@ -3593,7 +3739,8 @@ class ImapFlow extends EventEmitter {
         let processed = 0;
 
         let chunkSize = Number(options.chunkSize) || 64 * 1024;
-        let maxBytes = Number(options.maxBytes) || Infinity;
+        // Normalized once here so every bounded stage of the pipeline below agrees on the budget
+        let maxBytes = normalizeByteLimit(options.maxBytes);
 
         let uid = false;
 
@@ -3783,24 +3930,43 @@ class ImapFlow extends EventEmitter {
                 output = stream = new PassThrough();
         }
 
+        // Every byte-bounded stage of the pipeline. The fetch loop below stops as soon as any of
+        // them has taken all it will accept. The limiter at the tail is not enough on its own: a
+        // transform in the middle that buffers its whole input before emitting anything (the
+        // format=flowed decoder, the Japanese charset decoder) leaves the tail limiter reporting
+        // `limited === false` however much the server sends, so a download with a small maxBytes
+        // would still pull the entire part off the wire.
+        let limiters = [];
+        let isLimited = () => limiters.some(entry => entry.limited);
+
+        // Appending a stage means forwarding the current tail's errors to it before piping, so a
+        // failure anywhere reaches the stream the caller is reading
+        let pipeStage = stage => {
+            output.on('error', err => {
+                stage.emit('error', err);
+            });
+            output = output.pipe(stage);
+            return stage;
+        };
+
         let isTextNode = ['text/html', 'text/plain', 'text/x-amp-html'].includes(meta.contentType) || (part === '1' && !meta.contentType);
         if ((!meta.disposition || meta.disposition === 'inline') && isTextNode) {
             // RFC 3676 format=flowed text: unwrap soft line breaks
             if (meta.flowed) {
-                let flowDecoder = new FlowedDecoder({
-                    delSp: meta.delSp
-                });
-                output.on('error', err => {
-                    flowDecoder.emit('error', err);
-                });
-                output = output.pipe(flowDecoder);
+                // FlowedDecoder buffers its whole input before emitting, and being third party it
+                // carries no bound of its own, so bound what it can ever be handed. Unwrapping only
+                // removes bytes, so capping its input at maxBytes cannot push the delivered output
+                // above the cap either.
+                limiters.push(pipeStage(new LimitedPassthrough({ maxBytes })));
+
+                pipeStage(new FlowedDecoder({ delSp: meta.delSp }));
             }
 
             // Convert non-UTF-8 charsets to UTF-8 via a streaming decoder.
             // ASCII and UTF-8 need no conversion. Unknown charsets are left as-is.
             if (meta.charset && !['ascii', 'usascii', 'utf8'].includes(meta.charset.toLowerCase().replace(/[^a-z0-9]+/g, ''))) {
                 try {
-                    let decoder = getDecoder(meta.charset);
+                    let decoder = getDecoder(meta.charset, maxBytes);
                     // Safety listener attached first so the decoder always has at least
                     // one 'error' listener. Prevents Node.js from throwing
                     // ERR_UNHANDLED_ERROR if a later pipe setup step throws and leaves
@@ -3810,10 +3976,10 @@ class ImapFlow extends EventEmitter {
                     decoder.on('error', err => {
                         this.log.warn({ err, charset: meta.charset, cid: this.id });
                     });
-                    output.on('error', err => {
-                        decoder.emit('error', err);
-                    });
-                    output = output.pipe(decoder);
+                    // The Japanese decoder buffers its whole input as well, and reports the same
+                    // `limited` flag the limiters do so the fetch loop can stop once it is full.
+                    // A streaming decoder has no such flag, which reads as false and is correct.
+                    limiters.push(pipeStage(decoder));
                     // force to utf-8 for output
                     meta.charset = 'utf-8';
                 } catch {
@@ -3822,11 +3988,8 @@ class ImapFlow extends EventEmitter {
             }
         }
 
-        let limiter = new LimitedPassthrough({ maxBytes });
-        output.on('error', err => {
-            limiter.emit('error', err);
-        });
-        output = output.pipe(limiter);
+        let limiter = pipeStage(new LimitedPassthrough({ maxBytes }));
+        limiters.push(limiter);
 
         // Cleanup function
         const cleanup = () => {
@@ -3841,7 +4004,7 @@ class ImapFlow extends EventEmitter {
         output.once('close', cleanup);
 
         let writeChunk = chunk => {
-            if (limiter.limited || fetchAborted || stream.destroyed) {
+            if (isLimited() || fetchAborted || stream.destroyed) {
                 return true;
             }
             return stream.write(chunk);
@@ -3851,7 +4014,7 @@ class ImapFlow extends EventEmitter {
         // Stops when the server returns a short chunk (< chunkSize), the byte
         // limiter is satisfied, or the consumer destroys the output stream.
         let fetchAllParts = async () => {
-            while (hasMore && !limiter.limited && !fetchAborted) {
+            while (hasMore && !isLimited() && !fetchAborted) {
                 let { chunk } = await getNextPart();
                 if (!chunk || fetchAborted) {
                     break;
@@ -3903,6 +4066,23 @@ class ImapFlow extends EventEmitter {
             }
         };
 
+        // A download is a sequence of chunk FETCHes with a backpressure wait in between. Those
+        // gaps look exactly like an inactive connection, so without this auto-IDLE would start
+        // between chunks and the next chunk would have to break it again - two extra round
+        // trips per chunk, for as long as the consumer is slow. Counted before control returns
+        // to the event loop: the head chunk's own FETCH already armed the auto-IDLE timer, and
+        // with a very short autoIdleDelay that timer could otherwise fire before the deferred
+        // chunk loop below has marked the download open.
+        this._openDownloads++;
+        let downloadDone = false;
+        let finishDownload = () => {
+            if (!downloadDone) {
+                downloadDone = true;
+                this._openDownloads--;
+                this.autoidle();
+            }
+        };
+
         // Kick off the download pipeline asynchronously. The first chunk was
         // already fetched above (to get metadata); write it to the decoder
         // stream and then fetch remaining chunks via fetchAllParts().
@@ -3927,6 +4107,7 @@ class ImapFlow extends EventEmitter {
                     /* c8 ignore stop */
                 })
                 .finally(() => {
+                    finishDownload();
                     if (!fetchAborted && stream && !stream.destroyed) {
                         stream.end();
                     }
@@ -3939,6 +4120,7 @@ class ImapFlow extends EventEmitter {
                 writeResult = writeChunk(chunk);
             } catch (err) {
                 stream.emit('error', err);
+                finishDownload();
                 /* c8 ignore next 3 */ // emitting the error above triggers cleanup (fetchAborted=true), so this end() guard is already false here
                 if (!fetchAborted && stream && !stream.destroyed) {
                     stream.end();
@@ -3946,12 +4128,14 @@ class ImapFlow extends EventEmitter {
                 return;
             }
 
-            /* c8 ignore next 7 */ // `stream` is piped to the limiter before this runs, so the head write drains synchronously and always returns true (verified for chunkSize up to 8MB); the drain-wait branch is unreachable
+            /* c8 ignore next 9 */ // `stream` is piped to the limiter before this runs, so the head write drains synchronously and always returns true (verified for chunkSize up to 8MB); the drain-wait branch is unreachable
             if (!writeResult) {
                 // Initial chunk filled the buffer, wait for drain
                 stream.once('drain', () => {
                     if (!fetchAborted) {
                         runFetchAllParts();
+                    } else {
+                        finishDownload();
                     }
                 });
             } else {
@@ -4012,6 +4196,12 @@ class ImapFlow extends EventEmitter {
 
         for (let [part, content] of response.bodyParts) {
             let keyParts = part.split('.mime');
+            // The server chooses the BODY[...] keys it answers with: never let one be a
+            // prototype-chain name, or the assignments below write onto Object.prototype
+            // (process-wide pollution) instead of the result object.
+            if (isUnsafeKey(keyParts[0])) {
+                continue;
+            }
             if (keyParts.length === 1) {
                 // content
                 let key = keyParts[0];
@@ -4080,7 +4270,11 @@ class ImapFlow extends EventEmitter {
         }
 
         for (let part of Object.keys(data)) {
-            let meta = data[part].meta;
+            // `meta` is only built from the companion BODY[<part>.MIME] item. A server may
+            // legally answer with fewer items than were requested, and one part arriving
+            // without its MIME headers must not cost the caller the whole download.
+            let meta = data[part].meta || {};
+            data[part].meta = meta;
 
             // parts that arrived via FETCH BINARY (response.binaryParts) are already
             // decoded by the server - decoding again would corrupt the data
@@ -4112,18 +4306,22 @@ class ImapFlow extends EventEmitter {
 
         clearTimeout(this.idleStartTimer);
 
-        if (typeof this.preCheck === 'function') {
-            await this.preCheck();
-        }
-
-        let result = await this.runInternal(command, ...args);
+        try {
+            // The preCheck (breaking an active IDLE) sits inside the try on purpose: the
+            // clearTimeout above is unconditional, so every exit - a failed command or a
+            // preCheck that rejects - must still reach the finally, or auto-IDLE would stay
+            // disarmed on an otherwise healthy connection until some later command succeeded.
+            if (typeof this.preCheck === 'function') {
+                await this.preCheck();
+            }
 
-        if (command !== 'IDLE') {
-            // do not autostart IDLE, if IDLE itself was stopped
-            this.autoidle();
+            return await this.runInternal(command, ...args);
+        } finally {
+            if (command !== 'IDLE') {
+                // do not autostart IDLE, if IDLE itself was stopped
+                this.autoidle();
+            }
         }
-
-        return result;
     }
 
     /**
@@ -4244,6 +4442,10 @@ class ImapFlow extends EventEmitter {
                             idling: this.idling
                         });
                         this.currentLock = false;
+                        // autoidle() will not arm while a lock is held, so the release is what
+                        // restarts it. It re-checks the queue itself, so a lock waiting behind
+                        // this one still keeps IDLE off.
+                        this.autoidle();
                         // Use setImmediate to avoid stack overflow
                         setImmediate(() => {
                             this.processLocks().catch(err => this.log.error({ err, cid: this.id }));
@@ -4265,6 +4467,18 @@ class ImapFlow extends EventEmitter {
                     continue; // Process next lock in queue
                 }
 
+                // Both grant paths finish the same way. autoidle() is re-checked because a stale
+                // auto-IDLE timer may still be armed at this point: on the SELECT path run()
+                // re-arms auto-IDLE when the SELECT settles - a moment before currentLock is set -
+                // and the fast path can inherit a timer from an earlier command. Either way the
+                // timer must not fire inside the lock.
+                const grantLock = () => {
+                    this.currentLock = lock;
+                    armHeldTimer();
+                    this.autoidle();
+                    resolve({ path, release });
+                };
+
                 if (this.mailbox && this.mailbox.path === path && !!this.mailbox.readOnly === !!options.readOnly) {
                     // Fast path: mailbox is already selected with the right access mode
                     this.log.trace({
@@ -4274,9 +4488,7 @@ class ImapFlow extends EventEmitter {
                         idling: this.idling,
                         ...(options.description && { description: options.description })
                     });
-                    this.currentLock = lock;
-                    armHeldTimer();
-                    resolve({ path, release });
+                    grantLock();
                     break; // Stop processing; next lock waits for release()
                 }
 
@@ -4290,9 +4502,7 @@ class ImapFlow extends EventEmitter {
                         idling: this.idling,
                         ...(options.description && { description: options.description })
                     });
-                    this.currentLock = lock;
-                    armHeldTimer();
-                    resolve({ path, release });
+                    grantLock();
                     break; // Wait for this lock to be released
                 } catch (err) {
                     if (err.responseStatus === 'NO') {