imapflow 1.6.6 → 1.7.1

package/lib/imap-flow.js

@@ -38,6 +38,7 @@ const {
     AuthenticationFailure,
     getColorFlags,
     hasCapability,
+    logConnectionError,
     unrefTimer,
     parseUintValue,
     isUnsafeKey,
@@ -64,6 +65,91 @@ const MAX_THROTTLE_DELAY = 5 * 60 * 1000;
 // ImapFlow constructor option `maxLockHoldTime`. Set to 0 or false to disable.
 const HELD_LOCK_WARN_MS = 30 * 60 * 1000;
 
+// How long the connection has to stay inactive before auto-IDLE starts. Long enough that a caller
+// running a sequence of commands is not interrupted by an IDLE it immediately has to break.
+// Configurable via the ImapFlow constructor option `autoIdleDelay`.
+const AUTO_IDLE_DELAY = 15 * 1000;
+
+// Headroom kept between the auto-IDLE delay and the socket inactivity watchdog, so IDLE reaches
+// the wire before the watchdog can fire. See normalizeAutoIdleDelay().
+const AUTO_IDLE_SOCKET_MARGIN = 1000;
+
+// Commands whose client frames carry credentials; the raw traffic log withholds frame content
+// while one of these is in flight. See the logRaw branch in write().
+const RAW_SENSITIVE_COMMANDS = new Set(['LOGIN', 'AUTHENTICATE']);
+
+// Stand-in payload for a withheld raw client frame. Fixed width, so the entry says nothing
+// about the length of what it replaced.
+const RAW_HIDDEN_PLACEHOLDER = Buffer.from('(* value hidden *)\r\n').toString('base64');
+
+// Whether any attribute of a command is marked as a secret. Recurses into nested lists because
+// the command compiler honors `sensitive` at any depth, and the two must agree on what counts.
+function hasSensitiveAttribute(attributes) {
+    return [].concat(attributes || []).some(node => (Array.isArray(node) ? hasSensitiveAttribute(node) : !!node && node.sensitive));
+}
+
+// How deep flattenLoggedError() follows a chain of errors. Bounded because the chain comes from
+// whatever failed, not from this library: a cause chain can be arbitrarily long, and the cycle
+// check below only catches errors that repeat.
+const MAX_ERROR_FLATTEN_DEPTH = 4;
+
+// Recognizes an Error without instanceof, which fails for an error that crossed a realm boundary
+// (worker thread, vm context) even though it serializes exactly the same way.
+function isErrorLike(value) {
+    return value instanceof Error || (!!value && typeof value === 'object' && typeof value.message === 'string' && typeof value.stack === 'string');
+}
+
+// An Error carries `message` and `stack` on its prototype rather than as own enumerable
+// properties, so JSON.stringify() renders one as `{}` and both logger fallback paths (the console
+// fallback and emitLogs) would drop everything identifying it. Flattening happens here for both,
+// so their shapes cannot drift apart.
+//
+// Nested errors are flattened too, because the top level is often not where the answer is: this
+// library attaches the underlying failure as an enumerable `_err` (proxy setup, response
+// processing, normalized connection deadlines), and Node reports a multi-address connect failure
+// as an AggregateError whose members hold the per-address causes.
+function flattenLoggedError(value, depth = 0, seen = new Set()) {
+    if (depth >= MAX_ERROR_FLATTEN_DEPTH) {
+        return isErrorLike(value) ? value.message : value;
+    }
+
+    if (Array.isArray(value)) {
+        return value.map(entry => flattenLoggedError(entry, depth + 1, seen));
+    }
+
+    if (!isErrorLike(value)) {
+        // Anything else is left alone: exploding a Buffer would produce one key per byte, and a
+        // Date would become a pair of undefined fields.
+        return value;
+    }
+
+    // A repeat renders as its message alone, so a chain that loops back does not restate a full
+    // stack for every level down to the depth cap
+    if (seen.has(value)) {
+        return value.message;
+    }
+    seen.add(value);
+
+    let flatErr = {
+        message: value.message,
+        stack: value.stack
+    };
+
+    // `cause` (passed through the Error options argument) and the AggregateError members are own
+    // properties but not enumerable, so Object.keys does not list them
+    for (let key of new Set([...Object.keys(value), 'cause', 'errors'])) {
+        if (key in value) {
+            flatErr[key] = flattenLoggedError(value[key], depth + 1, seen);
+        }
+    }
+
+    return flatErr;
+}
+
+// The largest delay setTimeout can honor (2^31 - 1 ms). Anything above fires after 1 ms instead,
+// so the auto-IDLE delay cap has to stay inside this range even when socketTimeout is not.
+const MAX_TIMER_DELAY = 2 ** 31 - 1;
+
 const states = {
     NOT_AUTHENTICATED: 0x01,
     AUTHENTICATED: 0x02,
@@ -71,6 +157,49 @@ const states = {
     LOGOUT: 0x04
 };
 
+/**
+ * Normalizes the configured auto-IDLE delay into a value `setTimeout` can honor. Anything Node
+ * would silently turn into a 1ms timer - NaN, a negative number, a value above the 32-bit range -
+ * falls back to the default instead, because a 1ms delay means an IDLE/DONE round trip around
+ * every single command. The delay is also capped below `socketTimeout`, see AUTO_IDLE_SOCKET_MARGIN.
+ *
+ * @param {*} value - The configured `autoIdleDelay` option.
+ * @param {Number} socketTimeout - The normalized socket inactivity timeout.
+ * @param {Object} log - Logger, used to report a value that could not be used as given.
+ * @param {String} cid - Connection id for the log entry.
+ * @returns {Number} Delay in milliseconds.
+ */
+const normalizeAutoIdleDelay = (value, socketTimeout, log, cid) => {
+    const maxDelay = Math.max(0, Math.min(socketTimeout, MAX_TIMER_DELAY) - AUTO_IDLE_SOCKET_MARGIN);
+    const configured = value !== undefined && value !== null;
+
+    // Numeric strings are accepted, because configuration usually arrives from an environment
+    // variable or a JSON file. Booleans and blank strings are not: Number() would read them as 0,
+    // i.e. "IDLE around every command", the opposite of the "off" they suggest.
+    let delay = typeof value === 'number' || (typeof value === 'string' && value.trim()) ? Number(value) : NaN;
+    let reason = null;
+
+    if (!Number.isFinite(delay) || delay < 0) {
+        reason = 'not a non-negative finite number';
+        delay = AUTO_IDLE_DELAY;
+    }
+
+    if (delay > maxDelay) {
+        // An invalid value keeps its own reason: the cap then applies to the fallback default,
+        // not to anything the caller asked for.
+        reason = reason || `above socketTimeout (${socketTimeout} ms)`;
+        delay = maxDelay;
+    }
+
+    // Only an explicitly configured value is worth warning about. Capping the default because the
+    // caller picked a short socketTimeout is expected behavior, not a misconfiguration.
+    if (configured && reason) {
+        log.warn({ msg: 'Adjusted unusable autoIdleDelay option', requested: value, autoIdleDelay: delay, reason, cid });
+    }
+
+    return Math.floor(delay);
+};
+
 /**
  * @typedef {Object} MailboxObject
  * @global
@@ -188,6 +317,15 @@ class ImapFlow extends EventEmitter {
      * @property {Boolean} [disableAutoIdle=false]
      *     If `true`, do not start IDLE automatically. Useful when only specific operations are needed.
      *
+     * @property {Number} [autoIdleDelay=15000]
+     *     How long (in milliseconds) the connection has to be inactive before IDLE is started automatically.
+     *     Keep it above the pause your own code usually leaves between two commands, otherwise every command is
+     *     followed by an IDLE that the next command has to break, costing two extra round-trips per command.
+     *     To turn auto-IDLE off entirely use `disableAutoIdle` rather than a very large delay: the value is
+     *     capped below `socketTimeout`, because auto-IDLE has to start before the inactivity watchdog fires.
+     *     On servers without IDLE support this controls when the polling fallback starts, not how often it
+     *     polls - the poll interval is `maxIdleTime`, capped at 2 minutes.
+     *
      * @property {Object} [tls]
      *     Additional TLS options. For details, see [Node.js TLS connect](https://nodejs.org/api/tls.html#tls_tls_connect_options_callback).
      *
@@ -206,6 +344,7 @@ class ImapFlow extends EventEmitter {
      *
      * @property {Boolean} [logRaw=false]
      *     If `true`, logs all raw data (read and written) in base64 encoding. You can pipe such logs to [eerawlog](https://github.com/postalsys/eerawlog) command for readable output.
+     *     Client frames that carry credentials are replaced with a fixed placeholder and the entry is marked with `hidden: true`.
      *
      * @property {Boolean} [emitLogs=false]
      *     If `true`, emits `'log'` events with the same data passed to the logger.
@@ -373,6 +512,13 @@ class ImapFlow extends EventEmitter {
 
         this.commandParts = [];
 
+        // Whether the command currently being written carries credentials. send() sets this for
+        // every command before its first frame reaches the socket, and the raw traffic log reads
+        // it; every write belongs to the command send() dispatched last, because trySend() keeps
+        // one command in flight at a time. The initial value only covers a write before the
+        // first command, which no current path performs. See write().
+        this.rawSensitiveCommand = true;
+
         /**
          * Active IMAP capabilities. Value is either `true` for toggleable capabilities (eg. `UIDPLUS`)
          * or a number for capabilities with a value (eg. `APPENDLIMIT`)
@@ -438,6 +584,14 @@ class ImapFlow extends EventEmitter {
         this.idRequested = false;
 
         this.maxIdleTime = this.options.maxIdleTime || false;
+        this.autoIdleDelay = normalizeAutoIdleDelay(this.options.autoIdleDelay, this.socketTimeout, this.log, this.id);
+
+        // Wall-clock time of the last fallback poll, owned by lib/commands/idle.js
+        this._lastPollAt = 0;
+
+        // Download streams still fetching chunks. Counted, not a flag, so overlapping downloads
+        // cannot clear each other's suppression of auto-IDLE.
+        this._openDownloads = 0;
         this.missingIdleCommand = (this.options.missingIdleCommand || '').toString().toUpperCase().trim() || 'NOOP';
 
         this.disableBinary = !!this.options.disableBinary;
@@ -559,10 +713,17 @@ class ImapFlow extends EventEmitter {
         }
 
         if (this.logRaw) {
+            // Client frames of an authentication exchange carry credentials: the LOGIN
+            // arguments, and for AUTHENTICATE also the continuation writes (SASL PLAIN
+            // response, AUTH=LOGIN password, OAuth token payload) that bypass send(). The
+            // parsed command log masks these, so the raw log must withhold them too, but
+            // `data` still carries the placeholder rather than being dropped - the field is
+            // part of the documented log format and consumers decode it unconditionally.
             this.log.trace({
                 src: 'c',
                 msg: 'write to socket',
-                data: chunk.toString('base64'),
+                data: this.rawSensitiveCommand ? RAW_HIDDEN_PLACEHOLDER : chunk.toString('base64'),
+                ...(this.rawSensitiveCommand ? { hidden: true } : {}),
                 compress: !!this._deflate,
                 secure: !!this.secureConnection,
                 cid: this.id
@@ -618,6 +779,19 @@ class ImapFlow extends EventEmitter {
             return;
         }
 
+        // Classify before the first await. Every frame of this command - the command line and
+        // any continuation write that follows it - belongs to it until the next send(), because
+        // trySend() keeps one command in flight at a time. Reading currentRequest inside write()
+        // instead would be racy: rejectCurrentRequest() can clear it while the two compiler
+        // awaits below are pending, and the credential frame would then be logged in the clear.
+        // Uppercased because the wire protocol is case-insensitive and exec() passes the
+        // caller's spelling through unchanged. The command list covers the mechanisms whose
+        // secret arrives in a continuation frame, which carries no attributes of its own; the
+        // `sensitive` marker catches anything that instead puts a secret on the command line,
+        // so marking an attribute is enough to keep a new command out of the raw log too.
+        this.rawSensitiveCommand =
+            RAW_SENSITIVE_COMMANDS.has(typeof data.command === 'string' ? data.command.toUpperCase() : '') || hasSensitiveAttribute(data.attributes);
+
         // Compile with asArray=true: splits output into parts for literal handling.
         // First part is the command text up to the first literal, remaining parts
         // are stored in this.commandParts and sent after server "+" continuations.
@@ -715,7 +889,7 @@ class ImapFlow extends EventEmitter {
             // trySend() settles dispatch failures itself, by rejecting the affected
             // command through requestTagMap; this catch exists only so a throw from the
             // dispatch machinery itself can never surface as a floating rejection.
-            this.trySend().catch(err => this.log.warn({ err, cid: this.id }));
+            this.trySend().catch(err => logConnectionError(this, 'Failed to dispatch command', err));
         });
 
         // Prevent unhandled promise rejection if close() rejects this request
@@ -726,15 +900,14 @@ class ImapFlow extends EventEmitter {
         return promise;
     }
 
-    // Resolves the handler for an untagged server response. IMAP untagged responses
-    // come in two forms:
+    // Resolves an untagged server response to the keyword it is dispatched on. IMAP untagged
+    // responses come in two forms:
     //   * CAPABILITY ...       (keyword as command)
     //   * 42 FETCH (...)       (numeric prefix + keyword)
-    // For numeric-prefixed responses, we extract the keyword (FETCH, EXISTS, EXPUNGE, etc.)
-    // and look up the handler by that keyword instead.
-    // Handler priority: command-specific handlers (registered per exec() call) take
-    // precedence over global handlers (registered on the connection).
-    getUntaggedHandler(command, attributes) {
+    // For numeric-prefixed responses the keyword sits in the first attribute, because `command`
+    // holds the sequence number. Also used for logging, so a failure reports FETCH rather than
+    // the message number that happened to precede it.
+    normalizeUntaggedCommand(command, attributes) {
         if (/^[0-9]+$/.test(command)) {
             let type = attributes && attributes.length && typeof attributes[0].value === 'string' ? attributes[0].value.toUpperCase() : false;
             if (type) {
@@ -742,7 +915,13 @@ class ImapFlow extends EventEmitter {
             }
         }
 
-        command = command.toUpperCase().trim();
+        return command.toUpperCase().trim();
+    }
+
+    // Handler priority: command-specific handlers (registered per exec() call) take
+    // precedence over global handlers (registered on the connection).
+    getUntaggedHandler(command, attributes) {
+        command = this.normalizeUntaggedCommand(command, attributes);
         // Check command-specific handler first (registered in exec() options.untagged)
         if (this.currentRequest && this.currentRequest.options && this.currentRequest.options.untagged && this.currentRequest.options.untagged[command]) {
             return this.currentRequest.options.untagged[command];
@@ -920,7 +1099,7 @@ class ImapFlow extends EventEmitter {
         err.parserError = parserError;
         this.rejectCurrentRequest(err);
 
-        this.trySend().catch(sendErr => this.log.warn({ err: sendErr, cid: this.id }));
+        this.trySend().catch(sendErr => logConnectionError(this, 'Failed to dispatch command', sendErr));
     }
 
     /**
@@ -991,7 +1170,9 @@ class ImapFlow extends EventEmitter {
             try {
                 await this.currentRequest.options.onPlusTag(parsed);
             } catch (err) {
-                this.log.warn({ err, cid: this.id });
+                // The handler ran across an await and may have closed the connection, which
+                // clears currentRequest, so the command name is read defensively
+                this.log.warn({ msg: 'Failed to process continuation response', command: this.currentRequest?.command, err, cid: this.id });
             }
             return true;
         }
@@ -1005,7 +1186,7 @@ class ImapFlow extends EventEmitter {
                 this.write(content);
                 this.log.debug({ src: 'c', msg: `(* ${content.length}B continuation *)`, cid: this.id });
             } catch (err) {
-                this.log.warn({ err, cid: this.id });
+                logConnectionError(this, 'Failed to send literal continuation', err);
             }
             return true;
         }
@@ -1014,12 +1195,13 @@ class ImapFlow extends EventEmitter {
         // section[0] can be a parsed NIL (null), e.g. from a "[NIL]" response code - the
         // dereference must be guarded or one such line tears down the whole connection
         if (section && section.length && section[0] && section[0].type === 'ATOM' && typeof section[0].value === 'string') {
-            let sectionHandler = this.getSectionHandler(section[0].value.toUpperCase().trim());
+            let sectionKey = section[0].value.toUpperCase().trim();
+            let sectionHandler = this.getSectionHandler(sectionKey);
             if (sectionHandler) {
                 try {
                     await sectionHandler(section.slice(1));
                 } catch (err) {
-                    this.log.warn({ err, cid: this.id });
+                    this.log.warn({ msg: 'Failed to process response section', section: sectionKey, err, cid: this.id });
                 }
             }
         }
@@ -1030,7 +1212,14 @@ class ImapFlow extends EventEmitter {
                 try {
                     await untaggedHandler(parsed);
                 } catch (err) {
-                    this.log.warn({ err, cid: this.id });
+                    // Normalized only here: this runs for every untagged response, including
+                    // every FETCH, and the keyword is needed only to describe a failure
+                    this.log.warn({
+                        msg: 'Failed to process untagged response',
+                        command: this.normalizeUntaggedCommand(parsed.command, parsed.attributes),
+                        err,
+                        cid: this.id
+                    });
                     return true;
                 }
             }
@@ -1141,6 +1330,9 @@ class ImapFlow extends EventEmitter {
 
                     if (err.responseStatus === 'NO' && txt.includes('Some of the requested messages no longer exist')) {
                         // Treat as successful response
+                        // Kept at warn: the caller is handed fewer messages than it asked for and
+                        // is told nothing else about it, so this entry is the only record that
+                        // the response was truncated.
                         this.log.warn({ msg: 'Partial FETCH response', cid: this.id, err });
                         await new Promise(resolve => request.resolve({ response: parsed, next: resolve }));
                         break;
@@ -1252,10 +1444,22 @@ class ImapFlow extends EventEmitter {
         /**
          * Socket timeout event handler.
          *
-         * When a socket timeout occurs during IDLE, the handler attempts to recover the connection
-         * by sending a NOOP command and then returning to IDLE state.
+         * A quiet socket is only a dead connection when something was supposed to be talking. An
+         * idling session, a download whose consumer stopped draining, and a held mailbox lock
+         * whose owner is busy between commands are all expected to go quiet, so the handler keeps
+         * such a connection alive with a NOOP instead of tearing it down. An in-flight command is
+         * the opposite: its reply is overdue, a recovery NOOP would only queue up behind it and
+         * never reach the wire, so the timeout is reported as an error. The IDLE command itself is
+         * the one exception - it stays in flight for as long as idling lasts, and run() breaks it
+         * through preCheck() before the NOOP is dispatched.
          *
-         * @fires ImapFlow#error Emits error event unless the current command is IDLE
+         * IDLE is not restarted here: run() re-arms auto-IDLE once the NOOP settles, and
+         * autoidle() knows whether the connection is actually free for IDLE - an open download or
+         * a held lock keeps just the keepalive, and with disableAutoIdle nothing restarts at all.
+         * If the server is dead the NOOP never settles, and the next timeout fires with the NOOP
+         * as the stuck in-flight command, which lands in the error branch below.
+         *
+         * @fires ImapFlow#error Emits error event if the connection cannot be recovered
          */
         this._socketTimeout =
             this._socketTimeout ||
@@ -1263,26 +1467,21 @@ class ImapFlow extends EventEmitter {
                 const err = new Error('Socket timeout');
                 err.code = 'ETIMEOUT';
 
-                if (this.idling) {
+                const quietExpected = this.idling || this._openDownloads || this.currentLock;
+                const commandStuck = this.currentRequest && !(this.idling && this.currentRequest.command === 'IDLE');
+
+                if (quietExpected && !commandStuck) {
                     if (!this.usable || !this.socket || this.socket.destroyed) {
                         this.emitError(err);
                         return;
                     }
-                    // Attempt to recover IDLE connections. During true IDLE the NOOP cannot
-                    // reach the server until IDLE has been terminated: run() awaits preCheck(),
-                    // which sends DONE and only resolves once the server has completed the IDLE
-                    // command. Fallback polling has no such handshake - preCheck() there just
-                    // cancels the polling session.
-                    this.run('NOOP')
-                        .then(() => this.idle())
-                        .catch(err => {
-                            this.log.warn({ msg: 'IDLE recovery failed after timeout', err, cid: this.id });
-                            if (!this.isClosed) {
-                                this.close();
-                            }
-                        });
+                    this.run('NOOP').catch(err => {
+                        this.log.warn({ msg: 'Connection recovery failed after timeout', err, cid: this.id });
+                        if (!this.isClosed) {
+                            this.close();
+                        }
+                    });
                 } else {
-                    // Close immediately for non-IDLE operations
                     this.log.debug({ msg: 'Socket timeout', cid: this.id });
                     this.emitError(err);
                 }
@@ -1445,7 +1644,7 @@ class ImapFlow extends EventEmitter {
                 }
                 this.writeSocket.end();
             } catch (err) {
-                this.log.error({ err, info: 'Failed to destroy PassThrough socket', cid: this.id });
+                this.log.error({ err, msg: 'Failed to destroy PassThrough socket', cid: this.id });
                 throw err;
             }
         };
@@ -1951,6 +2150,22 @@ class ImapFlow extends EventEmitter {
         });
     }
 
+    // Reports one expunged message, either through the caller's expungeHandler or as an
+    // 'expunge' event. Shared by the EXPUNGE and VANISHED paths so the two cannot drift.
+    async notifyExpunge(payload) {
+        if (typeof this.options.expungeHandler !== 'function') {
+            this.emit('expunge', payload);
+            return;
+        }
+
+        try {
+            await this.options.expungeHandler(payload);
+        } catch (err) {
+            // The throw comes from the caller's own handler, not from this library
+            this.log.error({ msg: 'Failed to notify expunge event', payload, err, cid: this.id });
+        }
+    }
+
     async untaggedExpunge(untagged) {
         if (!this.mailbox) {
             // mailbox closed, ignore
@@ -1971,15 +2186,7 @@ class ImapFlow extends EventEmitter {
                 vanished: false
             };
 
-            if (typeof this.options.expungeHandler === 'function') {
-                try {
-                    await this.options.expungeHandler(payload);
-                } catch (err) {
-                    this.log.error({ msg: 'Failed to notify expunge event', payload, error: err, cid: this.id });
-                }
-            } else {
-                this.emit('expunge', payload);
-            }
+            await this.notifyExpunge(payload);
         }
     }
 
@@ -2018,15 +2225,7 @@ class ImapFlow extends EventEmitter {
                 earlier: tags.includes('EARLIER')
             };
 
-            if (typeof this.options.expungeHandler === 'function') {
-                try {
-                    await this.options.expungeHandler(payload);
-                } catch (err) {
-                    this.log.error({ msg: 'Failed to notify expunge event', payload, error: err, cid: this.id });
-                }
-            } else {
-                this.emit('expunge', payload);
-            }
+            await this.notifyExpunge(payload);
         }
     }
 
@@ -2120,6 +2319,16 @@ class ImapFlow extends EventEmitter {
         return range;
     }
 
+    // The single definition of "the connection is not free". A held or queued mailbox lock, a
+    // command in flight or queued, and an open download stream all mean a caller is
+    // mid-sequence: starting IDLE there injects an IDLE/DONE round trip - or, with
+    // `missingIdleCommand` set to SELECT or STATUS, a mailbox poll - between two of that
+    // caller's own commands. Every one of those states ends by calling autoidle() again, so
+    // declining while busy postpones IDLE, it never cancels it.
+    connectionBusy() {
+        return !!(this.currentLock || this.locks.length || this.currentRequest || this.requestQueue.length || this._openDownloads);
+    }
+
     // Timer process-liveness policy: connection establishment and greeting deadlines keep the
     // process alive, because a caller is waiting on connect() to settle. Background timers
     // (auto-IDLE, IDLE restart, fallback polling, throttle back-off, the held-lock diagnostic) are
@@ -2130,9 +2339,22 @@ class ImapFlow extends EventEmitter {
         if (this.options.disableAutoIdle || this.state !== this.states.SELECTED) {
             return;
         }
+
+        if (this.connectionBusy()) {
+            return;
+        }
+
         this.idleStartTimer = setTimeout(() => {
-            this.idle().catch(err => this.log.warn({ err, cid: this.id }));
-        }, 15 * 1000);
+            // Re-checked at fire time: paths that take ownership of the connection clear this
+            // timer, but the guard must not depend on every one of them doing so - a single
+            // missed clearTimeout would inject IDLE between a caller's own commands. Declining
+            // postpones rather than cancels: whatever made the connection busy calls autoidle()
+            // again when it finishes.
+            if (this.state !== this.states.SELECTED || this.connectionBusy()) {
+                return;
+            }
+            this.idle().catch(err => logConnectionError(this, 'Auto-IDLE failed', err));
+        }, this.autoIdleDelay);
         unrefTimer(this.idleStartTimer);
     }
 
@@ -2192,16 +2414,21 @@ class ImapFlow extends EventEmitter {
                     throw new Error('Failed to setup proxy connection');
                 }
             } catch (err) {
+                // Logged here rather than relying on proxy-connection.js, which only reports
+                // failures from inside the two connect helpers. An unsupported scheme, a proxy URL
+                // that will not parse and a deadline that expired before the connect started all
+                // reject before any logging happens there, so this is the one place that sees
+                // every way proxy setup can fail.
+                this.log.error({ msg: 'Failed to setup proxy connection', err, cid: this.id });
+
                 if (err.code === 'CONNECT_TIMEOUT') {
                     // The shared deadline expired during proxy setup. Report it as the documented
                     // connection timeout rather than as a generic proxy failure.
-                    this.log.error({ err, cid: this.id });
                     throw err;
                 }
                 let error = new Error('Failed to setup proxy connection');
                 error.code = err.code || 'ProxyError';
                 error._err = err;
-                this.log.error({ error, cid: this.id });
                 throw error;
             }
         }
@@ -2410,7 +2637,9 @@ class ImapFlow extends EventEmitter {
             }
 
             if (typeof this.preCheck === 'function') {
-                this.preCheck().catch(err => this.log.warn({ err, cid: this.id }));
+                // Runs while the connection is being torn down, so the rejection this sees is
+                // almost always the NoConnection close() is about to raise itself.
+                this.preCheck().catch(err => logConnectionError(this, 'Failed to break IDLE while closing', err));
             }
 
             // Session-only public state must not survive the connection it describes: callers read
@@ -2503,7 +2732,7 @@ class ImapFlow extends EventEmitter {
                     this._inflate.destroy();
                     this._inflate = null;
                 } catch (err) {
-                    this.log.error({ err, info: 'Failed to destroy inflate stream', cid: this.id });
+                    this.log.error({ err, msg: 'Failed to destroy inflate stream', cid: this.id });
                 }
             }
 
@@ -2513,7 +2742,7 @@ class ImapFlow extends EventEmitter {
                     this._deflate.destroy();
                     this._deflate = null;
                 } catch (err) {
-                    this.log.error({ err, info: 'Failed to destroy deflate stream', cid: this.id });
+                    this.log.error({ err, msg: 'Failed to destroy deflate stream', cid: this.id });
                 }
             }
 
@@ -2531,7 +2760,7 @@ class ImapFlow extends EventEmitter {
                         this.streamer.destroy();
                     }
                 } catch (err) {
-                    this.log.error({ err, info: 'Failed to cleanup streamer', cid: this.id });
+                    this.log.error({ err, msg: 'Failed to cleanup streamer', cid: this.id });
                 }
             }
 
@@ -2584,7 +2813,7 @@ class ImapFlow extends EventEmitter {
             this._socketEnd = null;
             this._socketTimeout = null;
 
-            this.log.trace({
+            this.log.debug({
                 msg: 'Connection closed',
                 cid: this.id,
                 ...(this._unknownTagCount ? { unknownTagCount: this._unknownTagCount } : {})
@@ -2600,7 +2829,7 @@ class ImapFlow extends EventEmitter {
             this.emit('close');
         } catch (ex) {
             // close failed
-            this.log.error(ex);
+            this.log.error({ err: ex, cid: this.id });
         }
     }
 
@@ -3963,6 +4192,23 @@ class ImapFlow extends EventEmitter {
             }
         };
 
+        // A download is a sequence of chunk FETCHes with a backpressure wait in between. Those
+        // gaps look exactly like an inactive connection, so without this auto-IDLE would start
+        // between chunks and the next chunk would have to break it again - two extra round
+        // trips per chunk, for as long as the consumer is slow. Counted before control returns
+        // to the event loop: the head chunk's own FETCH already armed the auto-IDLE timer, and
+        // with a very short autoIdleDelay that timer could otherwise fire before the deferred
+        // chunk loop below has marked the download open.
+        this._openDownloads++;
+        let downloadDone = false;
+        let finishDownload = () => {
+            if (!downloadDone) {
+                downloadDone = true;
+                this._openDownloads--;
+                this.autoidle();
+            }
+        };
+
         // Kick off the download pipeline asynchronously. The first chunk was
         // already fetched above (to get metadata); write it to the decoder
         // stream and then fetch remaining chunks via fetchAllParts().
@@ -3987,6 +4233,7 @@ class ImapFlow extends EventEmitter {
                     /* c8 ignore stop */
                 })
                 .finally(() => {
+                    finishDownload();
                     if (!fetchAborted && stream && !stream.destroyed) {
                         stream.end();
                     }
@@ -3999,6 +4246,7 @@ class ImapFlow extends EventEmitter {
                 writeResult = writeChunk(chunk);
             } catch (err) {
                 stream.emit('error', err);
+                finishDownload();
                 /* c8 ignore next 3 */ // emitting the error above triggers cleanup (fetchAborted=true), so this end() guard is already false here
                 if (!fetchAborted && stream && !stream.destroyed) {
                     stream.end();
@@ -4006,12 +4254,14 @@ class ImapFlow extends EventEmitter {
                 return;
             }
 
-            /* c8 ignore next 7 */ // `stream` is piped to the limiter before this runs, so the head write drains synchronously and always returns true (verified for chunkSize up to 8MB); the drain-wait branch is unreachable
+            /* c8 ignore next 9 */ // `stream` is piped to the limiter before this runs, so the head write drains synchronously and always returns true (verified for chunkSize up to 8MB); the drain-wait branch is unreachable
             if (!writeResult) {
                 // Initial chunk filled the buffer, wait for drain
                 stream.once('drain', () => {
                     if (!fetchAborted) {
                         runFetchAllParts();
+                    } else {
+                        finishDownload();
                     }
                 });
             } else {
@@ -4182,18 +4432,22 @@ class ImapFlow extends EventEmitter {
 
         clearTimeout(this.idleStartTimer);
 
-        if (typeof this.preCheck === 'function') {
-            await this.preCheck();
-        }
-
-        let result = await this.runInternal(command, ...args);
+        try {
+            // The preCheck (breaking an active IDLE) sits inside the try on purpose: the
+            // clearTimeout above is unconditional, so every exit - a failed command or a
+            // preCheck that rejects - must still reach the finally, or auto-IDLE would stay
+            // disarmed on an otherwise healthy connection until some later command succeeded.
+            if (typeof this.preCheck === 'function') {
+                await this.preCheck();
+            }
 
-        if (command !== 'IDLE') {
-            // do not autostart IDLE, if IDLE itself was stopped
-            this.autoidle();
+            return await this.runInternal(command, ...args);
+        } finally {
+            if (command !== 'IDLE') {
+                // do not autostart IDLE, if IDLE itself was stopped
+                this.autoidle();
+            }
         }
-
-        return result;
     }
 
     /**
@@ -4314,6 +4568,10 @@ class ImapFlow extends EventEmitter {
                             idling: this.idling
                         });
                         this.currentLock = false;
+                        // autoidle() will not arm while a lock is held, so the release is what
+                        // restarts it. It re-checks the queue itself, so a lock waiting behind
+                        // this one still keeps IDLE off.
+                        this.autoidle();
                         // Use setImmediate to avoid stack overflow
                         setImmediate(() => {
                             this.processLocks().catch(err => this.log.error({ err, cid: this.id }));
@@ -4335,6 +4593,18 @@ class ImapFlow extends EventEmitter {
                     continue; // Process next lock in queue
                 }
 
+                // Both grant paths finish the same way. autoidle() is re-checked because a stale
+                // auto-IDLE timer may still be armed at this point: on the SELECT path run()
+                // re-arms auto-IDLE when the SELECT settles - a moment before currentLock is set -
+                // and the fast path can inherit a timer from an earlier command. Either way the
+                // timer must not fire inside the lock.
+                const grantLock = () => {
+                    this.currentLock = lock;
+                    armHeldTimer();
+                    this.autoidle();
+                    resolve({ path, release });
+                };
+
                 if (this.mailbox && this.mailbox.path === path && !!this.mailbox.readOnly === !!options.readOnly) {
                     // Fast path: mailbox is already selected with the right access mode
                     this.log.trace({
@@ -4344,9 +4614,7 @@ class ImapFlow extends EventEmitter {
                         idling: this.idling,
                         ...(options.description && { description: options.description })
                     });
-                    this.currentLock = lock;
-                    armHeldTimer();
-                    resolve({ path, release });
+                    grantLock();
                     break; // Stop processing; next lock waits for release()
                 }
 
@@ -4360,9 +4628,7 @@ class ImapFlow extends EventEmitter {
                         idling: this.idling,
                         ...(options.description && { description: options.description })
                     });
-                    this.currentLock = lock;
-                    armHeldTimer();
-                    resolve({ path, release });
+                    grantLock();
                     break; // Wait for this lock to be released
                 } catch (err) {
                     if (err.responseStatus === 'NO') {
@@ -4497,7 +4763,19 @@ class ImapFlow extends EventEmitter {
                         // we are checking to make sure the level is supported.
                         // if it isn't supported but the level is error or fatal, log to console anyway.
                         if (level === 'fatal' || level === 'error') {
-                            console.log(JSON.stringify(...args));
+                            let entry = args[0];
+                            try {
+                                if (entry && typeof entry === 'object' && entry.err) {
+                                    entry = Object.assign({}, entry, { err: flattenLoggedError(entry.err) });
+                                }
+                                console.error(JSON.stringify(entry));
+                            } catch {
+                                // Serializing failed (a circular structure, a BigInt, a throwing
+                                // getter). This fallback exists so an error is never lost, so hand
+                                // the entry to console.error itself - it inspects rather than
+                                // serializes, and handles all three - instead of dropping it.
+                                console.error(entry);
+                            }
                         }
                     } else {
                         mainLogger[level](...args);
@@ -4505,18 +4783,21 @@ class ImapFlow extends EventEmitter {
                 }
 
                 if (this.emitLogs && args && args[0] && typeof args[0] === 'object') {
-                    let logEntry = Object.assign({ level, t: Date.now(), cid: this.id, lo: ++this.lo }, args[0]);
-                    if (logEntry.err && typeof logEntry.err === 'object') {
-                        let err = logEntry.err;
-                        logEntry.err = {
-                            stack: err.stack
-                        };
-                        // enumerable error fields
-                        Object.keys(err).forEach(key => {
-                            logEntry.err[key] = err[key];
-                        });
+                    // Guarded for the same reason as the console fallback above: a log call must
+                    // never throw. Most of these run inside catch blocks in the protocol
+                    // machinery, where a throw would escape the handler that was recovering from
+                    // something else and strand the connection. A throwing property getter on the
+                    // logged error and a throwing 'log' listener both end up here.
+                    try {
+                        let logEntry = Object.assign({ level, t: Date.now(), cid: this.id, lo: ++this.lo }, args[0]);
+                        if (logEntry.err) {
+                            logEntry.err = flattenLoggedError(logEntry.err);
+                        }
+                        this.emit('log', logEntry);
+                    } catch {
+                        // Nothing to do with it: reporting the failure would re-enter this
+                        // same path
                     }
-                    this.emit('log', logEntry);
                 }
             };
         }