imapflow 1.5.0 → 1.6.0

package/lib/imap-flow.js

@@ -24,6 +24,7 @@ const FlowedDecoder = require('@zone-eu/mailsplit/lib/flowed-decoder');
 const { PassThrough } = require('stream');
 
 const { proxyConnection, detachEarlyErrorHandler } = require('./proxy-connection');
+const { ConnectionDeadline } = require('./connection-deadline');
 
 const {
     comparePaths,
@@ -36,14 +37,14 @@ const {
     expandRange,
     AuthenticationFailure,
     getColorFlags,
-    hasCapability
+    hasCapability,
+    unrefTimer
 } = require('./tools');
 
 const imapCommands = require('./imap-commands.js');
 
 const noop = () => {};
 
-const CONNECT_TIMEOUT = 90 * 1000;
 const GREETING_TIMEOUT = 16 * 1000;
 const UPGRADE_TIMEOUT = 10 * 1000;
 
@@ -205,7 +206,21 @@ class ImapFlow extends EventEmitter {
      *     If `true`, disconnects after successful authentication without performing other actions.
      *
      * @property {String} [proxy]
-     *     Proxy URL. Supports HTTP CONNECT (`http://`, `https://`) and SOCKS (`socks://`, `socks4://`, `socks5://`).
+     *     Proxy URL. Supports HTTP CONNECT (`http://`, `https://`) and SOCKS (`socks://`, `socks4://`, `socks4a://`, `socks5://`).
+     *     IPv6 proxy endpoints use the URL form, e.g. `socks5://[2001:db8::1]:1080`.
+     *
+     *     DNS behaviour depends on the proxy protocol:
+     *       - `http`/`https`: the destination hostname is sent to the proxy unresolved.
+     *       - `socks4`: destination hostnames are resolved locally to IPv4, because SOCKS4 carries
+     *         only IPv4 destination addresses and a hostname would silently become a SOCKS4a
+     *         request. IPv6 destinations are rejected.
+     *       - `socks4a`: destination hostnames are sent to the proxy for remote DNS. IPv6
+     *         destinations are rejected.
+     *       - `socks`/`socks5`: destination hostnames are sent to the proxy for remote DNS, and
+     *         IPv4/IPv6 literals are passed through unchanged.
+     *
+     *     The proxy endpoint itself is never resolved by ImapFlow - a hostname endpoint is handed
+     *     to Node as-is, keeping its normal lookup and connection behaviour.
      *
      * @property {Boolean} [qresync=false]
      *     If `true`, enables QRESYNC support so that EXPUNGE notifications include `uid` instead of `seq`.
@@ -228,7 +243,9 @@ class ImapFlow extends EventEmitter {
      *     without losing the other auto-enabled extensions.
      *
      * @property {Number} [connectionTimeout=90000]
-     *     Maximum time (in milliseconds) to wait for the connection to establish. Defaults to 90 seconds.
+     *     Maximum time (in milliseconds) to wait for a usable transport. Covers DNS resolution,
+     *     proxy negotiation and the TCP/TLS handshake as a single budget, so an expiry in any of
+     *     those phases rejects with error code `CONNECT_TIMEOUT`. Defaults to 90 seconds.
      *
      * @property {Number} [greetingTimeout=16000]
      *     Maximum time (in milliseconds) to wait for the server greeting after a connection is established. Defaults to 16 seconds.
@@ -294,6 +311,11 @@ class ImapFlow extends EventEmitter {
             this.secureConnection = true;
         }
 
+        // Normalized once so direct TLS, cleartext, proxied and STARTTLS-upgraded transports
+        // cannot end up with different inactivity watchdogs. As documented, 0 (and any other
+        // falsy or invalid value) means "use the default", not "disable".
+        this.socketTimeout = Number(this.options.socketTimeout) || SOCKET_TIMEOUT;
+
         this.logRaw = this.options.logRaw;
         this.streamer = new ImapStream({
             logger: this.log,
@@ -330,6 +352,14 @@ class ImapFlow extends EventEmitter {
         this.requestQueue = [];
         this.currentRequest = false;
 
+        // Count of tagged responses whose tag was never issued by this connection. Tolerated
+        // (non-conforming servers do this) but tracked, so the compatibility decision in
+        // countUnknownTag() can be revisited with field data instead of guesses. Warnings are
+        // emitted at the milestones below (1, 2, 4, 8, ...) so the count stays exact without
+        // turning a spraying server into a log flood.
+        this._unknownTagCount = 0;
+        this._nextUnknownTagWarn = 1;
+
         this.writeBytesCounter = 0;
 
         this.commandParts = [];
@@ -436,19 +466,21 @@ class ImapFlow extends EventEmitter {
         }
         err._connId = err._connId || this.id;
 
-        // During a STARTTLS handshake the upgrade promise owns the single error path
-        // (tlsSocketErrorHandler -> reject). Route the error there so a streamer-originated
-        // failure is surfaced with its real code (instead of a generic ClosedAfterConnect*)
-        // and cannot hang a verifyOnly connect() waiting on a 'close' that never rejects.
-        // Fall back to closing if the upgrade has no pending rejector.
+        // During a STARTTLS handshake the upgrade owns the single error path (its settle()
+        // helper). Route the error there so a streamer-originated failure is surfaced with its
+        // real code (instead of a generic ClosedAfterConnect*) and cannot hang a verifyOnly
+        // connect() waiting on a 'close' that never rejects. Fall back to closing if the upgrade
+        // has no pending rejector.
         if (this.upgrading) {
-            this.upgrading = false;
-            this.closeAfter();
-            if (typeof this._upgradeReject === 'function') {
-                let reject = this._upgradeReject;
-                this._upgradeReject = null;
+            let reject = this._upgradeReject;
+            this._upgradeReject = null;
+            if (typeof reject === 'function') {
+                // settle() clears the upgrade timer and flags, and closes the connection
                 reject(err);
+                return;
             }
+            this.upgrading = false;
+            this.closeAfter();
             return;
         }
 
@@ -603,6 +635,13 @@ class ImapFlow extends EventEmitter {
         // send each remaining part from this.commandParts.
         this.write(this.commandParts.shift());
 
+        // The command is on the wire now. Tagged-response correlation requires this, so a server
+        // that guesses the next (sequential) tag cannot settle a command during the window between
+        // it becoming current and actually being written.
+        if (this.currentRequest && this.currentRequest.tag === data.tag) {
+            this.currentRequest.sent = true;
+        }
+
         if (typeof options.onSend === 'function') {
             options.onSend();
         }
@@ -694,228 +733,369 @@ class ImapFlow extends EventEmitter {
         }
     }
 
+    // Releases a readable stream item exactly once. The item's `next` callback is the parser's
+    // backpressure token: until it is called, ImapStream stops feeding the connection. Every
+    // path out of response handling - success, handled error, or unexpected throw - has to go
+    // through here, otherwise the parser stalls permanently.
+    releaseStreamData(data) {
+        if (!data || data.released) {
+            return;
+        }
+        data.released = true;
+        if (typeof data.next === 'function') {
+            data.next();
+        }
+    }
+
+    // Records a tagged response whose tag was never issued by this connection. ImapFlow talks
+    // to a wide range of non-conforming servers, so this is tolerated rather than terminal, but
+    // it must not pass silently. Warnings are emitted for the first occurrence and then at
+    // powers of two so a server spraying stray tagged lines cannot flood the log, while the
+    // counter itself stays exact and is reported when the connection closes.
+    countUnknownTag(tag) {
+        if (this.isClosed) {
+            // teardown crossover, not a server compatibility signal
+            return;
+        }
+
+        this._unknownTagCount++;
+        if (this._unknownTagCount === this._nextUnknownTagWarn) {
+            this._nextUnknownTagWarn *= 2;
+            this.log.warn({
+                msg: 'Tagged response for an unknown tag',
+                tag,
+                unknownTagCount: this._unknownTagCount,
+                cid: this.id
+            });
+        }
+    }
+
+    // Terminally fails the connection on a protocol violation: stop parsing, then report. Both
+    // steps are explicit here rather than destroying the parser *with* the error and relying on
+    // its error listener to report, so the reporting path does not depend on teardown ordering or
+    // on the streamer error handler's suppression list.
+    failProtocol(err) {
+        if (this.streamer && !this.streamer.destroyed) {
+            // Destroyed without an error: nothing after a protocol violation may reach
+            // application state, and emitError() below owns reporting.
+            this.streamer.destroy();
+        }
+        this.emitError(err);
+    }
+
+    // Rejects the in-flight request, if any, exactly once. Used when response handling fails in
+    // a way that leaves the command's outcome unknown.
+    rejectCurrentRequest(err) {
+        if (!this.currentRequest) {
+            return;
+        }
+        let tag = this.currentRequest.tag;
+        this.currentRequest = false;
+        let request = this.requestTagMap.get(tag);
+        if (request) {
+            this.requestTagMap.delete(tag);
+            request.reject(err);
+        }
+    }
+
     async reader() {
         let data;
         let processedCount = 0;
         while ((data = this.streamer.read()) !== null) {
-            let parsed;
+            let keepReading;
 
             try {
-                parsed = await parser(data.payload, { literals: data.literals });
-                if (parsed.tag && !['*', '+'].includes(parsed.tag) && parsed.command) {
-                    let payload = { response: parsed.command };
-
-                    if (
-                        parsed.attributes &&
-                        parsed.attributes[0] &&
-                        parsed.attributes[0].section &&
-                        parsed.attributes[0].section[0] &&
-                        parsed.attributes[0].section[0].type === 'ATOM'
-                    ) {
-                        payload.code = parsed.attributes[0].section[0].value;
-                    }
-                    this.emit('response', payload);
-                }
+                keepReading = await this.handleResponse(data);
             } catch (err) {
-                // can not make sense of this
-                this.log.error({ src: 's', msg: data.payload.toString(), err, cid: this.id });
-                data.next();
-                continue;
+                // Response handling past the parse step (log compilation, response shape
+                // assumptions, an untagged handler bug) must never throw out of this loop: the
+                // parser would keep waiting on its backpressure callback forever, which is a
+                // silent permanent hang. Fail closed instead.
+                keepReading = false;
+                let error = new Error('Failed to process server response');
+                error.code = 'ResponseProcessingFailed';
+                error._err = err;
+                this.log.error({ msg: 'Failed to process server response', err, cid: this.id });
+                this.rejectCurrentRequest(error);
+                this.failProtocol(error);
+            } finally {
+                this.releaseStreamData(data);
             }
 
-            let logCompiled = await compiler(parsed, {
-                isLogging: true
-            });
+            if (!keepReading) {
+                return;
+            }
 
-            if (/^\d+$/.test(parsed.command) && parsed.attributes && parsed.attributes[0] && parsed.attributes[0].value === 'FETCH') {
-                // too many FETCH responses, might want to filter these out
-                this.log.trace({ src: 's', msg: logCompiled.toString(), cid: this.id, nullBytesRemoved: parsed.nullBytesRemoved });
-            } else {
-                this.log.debug({ src: 's', msg: logCompiled.toString(), cid: this.id, nullBytesRemoved: parsed.nullBytesRemoved });
+            // Yield to event loop every 10 processed messages to prevent CPU blocking
+            processedCount++;
+            if (processedCount % 10 === 0) {
+                await new Promise(resolve => setImmediate(resolve));
             }
+        }
+    }
+
+    /**
+     * Handles a single parsed server response: telemetry, continuation requests, response-code
+     * section handlers, untagged handlers and tagged command completion.
+     *
+     * @param {Object} data - Readable item from the parser stream.
+     * @returns {Promise<Boolean>} `true` to keep reading, `false` to stop (connection is failing).
+     */
+    async handleResponse(data) {
+        let parsed;
+
+        try {
+            parsed = await parser(data.payload, { literals: data.literals });
+            if (parsed.tag && !['*', '+'].includes(parsed.tag) && parsed.command) {
+                let payload = { response: parsed.command };
+
+                if (
+                    parsed.attributes &&
+                    parsed.attributes[0] &&
+                    parsed.attributes[0].section &&
+                    parsed.attributes[0].section[0] &&
+                    parsed.attributes[0].section[0].type === 'ATOM'
+                ) {
+                    payload.code = parsed.attributes[0].section[0].value;
+                }
+                this.emit('response', payload);
+            }
+        } catch (err) {
+            // can not make sense of this
+            this.log.error({ src: 's', msg: data.payload.toString(), err, cid: this.id });
+            return true;
+        }
 
-            // IMAP "+" (continuation request) handling. The server sends "+" in two cases:
-            // 1. During IDLE or AUTHENTICATE, where a custom handler (onPlusTag) processes it
-            // 2. During literal data transfer, where we send the next queued literal chunk
-            if (parsed.tag === '+' && this.currentRequest && this.currentRequest.options && typeof this.currentRequest.options.onPlusTag === 'function') {
+        let logCompiled = await compiler(parsed, {
+            isLogging: true
+        });
+
+        if (/^\d+$/.test(parsed.command) && parsed.attributes && parsed.attributes[0] && parsed.attributes[0].value === 'FETCH') {
+            // too many FETCH responses, might want to filter these out
+            this.log.trace({ src: 's', msg: logCompiled.toString(), cid: this.id, nullBytesRemoved: parsed.nullBytesRemoved });
+        } else {
+            this.log.debug({ src: 's', msg: logCompiled.toString(), cid: this.id, nullBytesRemoved: parsed.nullBytesRemoved });
+        }
+
+        // IMAP "+" (continuation request) handling. The server sends "+" in two cases:
+        // 1. During IDLE or AUTHENTICATE, where a custom handler (onPlusTag) processes it
+        // 2. During literal data transfer, where we send the next queued literal chunk
+        if (parsed.tag === '+' && this.currentRequest && this.currentRequest.options && typeof this.currentRequest.options.onPlusTag === 'function') {
+            try {
+                await this.currentRequest.options.onPlusTag(parsed);
+            } catch (err) {
+                this.log.warn({ err, cid: this.id });
+            }
+            return true;
+        }
+
+        // Server acknowledged our literal size with "+", send the actual literal data
+        if (parsed.tag === '+' && this.commandParts.length) {
+            let content = this.commandParts.shift();
+            // A write() failure here (e.g. socket closed mid-command) must not fail the whole
+            // connection; the command's own tagged response or the close path reports it.
+            try {
+                this.write(content);
+                this.log.debug({ src: 'c', msg: `(* ${content.length}B continuation *)`, cid: this.id });
+            } catch (err) {
+                this.log.warn({ err, cid: this.id });
+            }
+            return true;
+        }
+
+        let section = parsed.attributes && parsed.attributes.length && parsed.attributes[0] && !parsed.attributes[0].value && parsed.attributes[0].section;
+        if (section && section.length && section[0].type === 'ATOM' && typeof section[0].value === 'string') {
+            let sectionHandler = this.getSectionHandler(section[0].value.toUpperCase().trim());
+            if (sectionHandler) {
                 try {
-                    await this.currentRequest.options.onPlusTag(parsed);
+                    await sectionHandler(section.slice(1));
                 } catch (err) {
                     this.log.warn({ err, cid: this.id });
                 }
-                data.next();
-                continue;
             }
+        }
 
-            // Server acknowledged our literal size with "+", send the actual literal data
-            if (parsed.tag === '+' && this.commandParts.length) {
-                let content = this.commandParts.shift();
-                // A write() failure here (e.g. socket closed mid-command) must not propagate
-                // out of the loop and skip data.next(), which would stall the parser stream.
+        if (parsed.tag === '*' && parsed.command) {
+            let untaggedHandler = this.getUntaggedHandler(parsed.command, parsed.attributes);
+            if (untaggedHandler) {
                 try {
-                    this.write(content);
-                    this.log.debug({ src: 'c', msg: `(* ${content.length}B continuation *)`, cid: this.id });
+                    await untaggedHandler(parsed);
                 } catch (err) {
                     this.log.warn({ err, cid: this.id });
+                    return true;
                 }
-                data.next();
-                continue;
             }
+        }
 
-            let section = parsed.attributes && parsed.attributes.length && parsed.attributes[0] && !parsed.attributes[0].value && parsed.attributes[0].section;
-            if (section && section.length && section[0].type === 'ATOM' && typeof section[0].value === 'string') {
-                let sectionHandler = this.getSectionHandler(section[0].value.toUpperCase().trim());
-                if (sectionHandler) {
-                    try {
-                        await sectionHandler(section.slice(1));
-                    } catch (err) {
-                        this.log.warn({ err, cid: this.id });
-                    }
-                }
-            }
+        // Tagged response correlation. A tagged response may only complete the command that was
+        // actually written to the socket (invariant 2), so the three cases below are kept apart:
+        // the active command completes, a command that has not been written yet is proof of
+        // desynchronization (queued behind another command, or current but not yet on the wire),
+        // and an entirely unknown tag is recorded but tolerated.
+        if (parsed.tag && !['*', '+'].includes(parsed.tag)) {
+            if (this.currentRequest && this.currentRequest.tag === parsed.tag && this.currentRequest.sent) {
+                let request = this.requestTagMap.get(parsed.tag);
+                this.requestTagMap.delete(parsed.tag);
+                this.currentRequest = false;
 
-            if (parsed.tag === '*' && parsed.command) {
-                let untaggedHandler = this.getUntaggedHandler(parsed.command, parsed.attributes);
-                if (untaggedHandler) {
-                    try {
-                        await untaggedHandler(parsed);
-                    } catch (err) {
-                        this.log.warn({ err, cid: this.id });
-                        data.next();
-                        continue;
-                    }
+                if (request) {
+                    await this.settleRequest(request, parsed, !!data.trailingAfterLine);
                 }
-            }
 
-            if (this.requestTagMap.has(parsed.tag)) {
+                // Send the next queued command only after the completed command's handler has
+                // applied its own state (e.g. select.js publishing the new mailbox), so the next
+                // command cannot reach the wire against half-updated state. A failure here must
+                // not propagate, or the whole connection would be failed over a send error that
+                // the command's own promise already reports.
+                // Note: on a rejected command the handler's catch block runs on its own microtask
+                // chain, so only the success path is fully ordered.
+                try {
+                    await this.trySend();
+                } catch (err) {
+                    this.log.warn({ err, cid: this.id });
+                }
+            } else if (this.requestTagMap.has(parsed.tag)) {
+                // The server answered a command that has not been written to the socket yet.
+                // Continuing would report unsent mutations as successful and leave every later
+                // response ambiguous, so reject this request and fail the connection closed.
                 let request = this.requestTagMap.get(parsed.tag);
                 this.requestTagMap.delete(parsed.tag);
 
-                if (this.currentRequest && this.currentRequest.tag === parsed.tag) {
-                    // send next pending command. A failure here must not propagate out of the
-                    // loop and skip data.next() below, which would stall the parser stream.
-                    this.currentRequest = false;
-                    try {
-                        await this.trySend();
-                    } catch (err) {
-                        this.log.warn({ err, cid: this.id });
-                    }
-                }
-
-                switch (parsed.command.toUpperCase()) {
-                    case 'OK':
-                    case 'BYE':
-                        // hasTrailingData is forwarded so STARTTLS can detect a plaintext
-                        // injection (data buffered after the tagged OK, before the handshake).
-                        await new Promise(resolve => request.resolve({ response: parsed, next: resolve, hasTrailingData: !!data.trailingAfterLine }));
-                        break;
+                let err = new Error('Server sent a tagged response for a command that was not in flight');
+                err.code = 'UnexpectedTag';
+                err.details = {
+                    received: parsed.tag,
+                    expected: this.currentRequest ? this.currentRequest.tag : null
+                };
 
-                    case 'NO':
-                    case 'BAD': {
-                        let txt =
-                            parsed.attributes &&
-                            parsed.attributes
-                                .filter(val => val.type === 'TEXT')
-                                .map(val => val.value.trim())
-                                .join(' ');
+                this.log.error({ msg: 'Protocol desynchronization', err, cid: this.id });
+                request.reject(err);
+                this.failProtocol(err);
+                return false;
+            } else {
+                this.countUnknownTag(parsed.tag);
+            }
+        }
 
-                        let err = new Error('Command failed');
-                        err.response = parsed;
-                        err.responseStatus = parsed.command.toUpperCase();
+        return true;
+    }
 
-                        try {
-                            err.executedCommand =
-                                parsed.tag +
-                                (
-                                    await compiler(request, {
-                                        isLogging: true
-                                    })
-                                ).toString();
-                        } catch {
-                            // ignore
-                        }
+    /**
+     * Settles a request with its tagged completion response.
+     *
+     * On success the returned promise stays pending until the command handler calls `next()` on
+     * the response, which is what orders state application before the next queued command is
+     * dispatched. A command handler must therefore always release its own response before
+     * awaiting another command on the same connection.
+     *
+     * @param {Object} request - Pending request entry (resolve/reject and the compiled command).
+     * @param {Object} parsed - Parsed tagged response.
+     * @param {Boolean} hasTrailingData - Whether more input was already buffered after this line.
+     * @returns {Promise<void>}
+     */
+    async settleRequest(request, parsed, hasTrailingData) {
+        switch ((parsed.command || '').toUpperCase()) {
+            case 'OK':
+            case 'BYE':
+                // hasTrailingData is forwarded so STARTTLS can detect a plaintext
+                // injection (data buffered after the tagged OK, before the handshake).
+                await new Promise(resolve => request.resolve({ response: parsed, next: resolve, hasTrailingData }));
+                break;
 
-                        if (txt) {
-                            err.responseText = txt;
+            case 'NO':
+            case 'BAD': {
+                let txt =
+                    parsed.attributes &&
+                    parsed.attributes
+                        .filter(val => val.type === 'TEXT')
+                        .map(val => val.value.trim())
+                        .join(' ');
 
-                            if (err.responseStatus === 'NO' && txt.includes('Some of the requested messages no longer exist')) {
-                                // Treat as successful response
-                                this.log.warn({ msg: 'Partial FETCH response', cid: this.id, err });
-                                await new Promise(resolve => request.resolve({ response: parsed, next: resolve }));
-                                break;
-                            }
+                let err = new Error('Command failed');
+                err.response = parsed;
+                err.responseStatus = parsed.command.toUpperCase();
 
-                            let throttleDelay = false;
+                try {
+                    err.executedCommand =
+                        parsed.tag +
+                        (
+                            await compiler(request, {
+                                isLogging: true
+                            })
+                        ).toString();
+                } catch {
+                    // ignore
+                }
 
-                            // MS365 throttling detection: Office 365 returns BAD with a human-readable
-                            // backoff time when rate limits are hit. Parse the delay from the response text.
-                            // Example: "tag BAD Request is throttled. Suggested Backoff Time: 92415 milliseconds"
-                            if (/Request is throttled/i.test(txt) && /Backoff Time/i.test(txt)) {
-                                let throttlingMatch = txt.match(/Backoff Time[:=\s]+(\d+)/i);
-                                if (throttlingMatch && throttlingMatch[1] && !isNaN(throttlingMatch[1])) {
-                                    throttleDelay = Number(throttlingMatch[1]);
-                                }
-                            }
+                if (txt) {
+                    err.responseText = txt;
 
-                            // Wait and return a throttling error
-                            if (throttleDelay) {
-                                err.code = 'ETHROTTLE';
-                                err.throttleReset = throttleDelay;
+                    if (err.responseStatus === 'NO' && txt.includes('Some of the requested messages no longer exist')) {
+                        // Treat as successful response
+                        this.log.warn({ msg: 'Partial FETCH response', cid: this.id, err });
+                        await new Promise(resolve => request.resolve({ response: parsed, next: resolve }));
+                        break;
+                    }
 
-                                let delayResponse = throttleDelay;
-                                if (delayResponse > 5 * 60 * 1000) {
-                                    // Cap wait at 5 minutes to avoid hanging connections indefinitely.
-                                    // The server-suggested delay can be very large.
-                                    delayResponse = 5 * 60 * 1000;
-                                }
+                    let throttleDelay = false;
 
-                                this.log.warn({ msg: 'Throttling detected', cid: this.id, throttleDelay, delayResponse, err });
-
-                                // Tracked, abortable wait. Storing the timer lets close() clear it so
-                                // the back-off never keeps the event loop alive, and storing the resolve
-                                // lets close() abort the wait promptly (aborted=true) instead of blocking
-                                // the reader for up to 5 minutes and rejecting long after the connection
-                                // is gone. Normal expiry resolves with aborted=false.
-                                let aborted = await new Promise(resolve => {
-                                    this._throttleAbort = resolve;
-                                    this._throttleTimer = setTimeout(() => resolve(false), delayResponse);
-                                    if (typeof this._throttleTimer.unref === 'function') {
-                                        this._throttleTimer.unref();
-                                    }
-                                });
-                                this._throttleTimer = null;
-                                this._throttleAbort = null;
-
-                                if (aborted) {
-                                    // Connection closed during back-off: reject promptly with a
-                                    // connection error (carrying any server BYE reason) instead of
-                                    // waiting out the throttle delay.
-                                    request.reject(this.createNoConnectionError(this.byeReason));
-                                    break;
-                                }
-                            }
+                    // MS365 throttling detection: Office 365 returns BAD with a human-readable
+                    // backoff time when rate limits are hit. Parse the delay from the response text.
+                    // Example: "tag BAD Request is throttled. Suggested Backoff Time: 92415 milliseconds"
+                    if (/Request is throttled/i.test(txt) && /Backoff Time/i.test(txt)) {
+                        let throttlingMatch = txt.match(/Backoff Time[:=\s]+(\d+)/i);
+                        if (throttlingMatch && throttlingMatch[1] && !isNaN(throttlingMatch[1])) {
+                            throttleDelay = Number(throttlingMatch[1]);
                         }
-
-                        request.reject(err);
-                        break;
                     }
 
-                    default: {
-                        let err = new Error('Invalid server response');
-                        err.code = 'InvalidResponse';
-                        err.response = parsed;
-                        request.reject(err);
-                        break;
+                    // Wait and return a throttling error
+                    if (throttleDelay) {
+                        err.code = 'ETHROTTLE';
+                        err.throttleReset = throttleDelay;
+
+                        let delayResponse = throttleDelay;
+                        if (delayResponse > 5 * 60 * 1000) {
+                            // Cap wait at 5 minutes to avoid hanging connections indefinitely.
+                            // The server-suggested delay can be very large.
+                            delayResponse = 5 * 60 * 1000;
+                        }
+
+                        this.log.warn({ msg: 'Throttling detected', cid: this.id, throttleDelay, delayResponse, err });
+
+                        // Tracked, abortable wait. Storing the timer lets close() clear it so
+                        // the back-off never keeps the event loop alive, and storing the resolve
+                        // lets close() abort the wait promptly (aborted=true) instead of blocking
+                        // the reader for up to 5 minutes and rejecting long after the connection
+                        // is gone. Normal expiry resolves with aborted=false.
+                        let aborted = await new Promise(resolve => {
+                            this._throttleAbort = resolve;
+                            this._throttleTimer = setTimeout(() => resolve(false), delayResponse);
+                            unrefTimer(this._throttleTimer);
+                        });
+                        this._throttleTimer = null;
+                        this._throttleAbort = null;
+
+                        if (aborted) {
+                            // Connection closed during back-off: reject promptly with a
+                            // connection error (carrying any server BYE reason) instead of
+                            // waiting out the throttle delay.
+                            request.reject(this.createNoConnectionError(this.byeReason));
+                            break;
+                        }
                     }
                 }
-            }
 
-            data.next();
+                request.reject(err);
+                break;
+            }
 
-            // Yield to event loop every 10 processed messages to prevent CPU blocking
-            processedCount++;
-            if (processedCount % 10 === 0) {
-                await new Promise(resolve => setImmediate(resolve));
+            default: {
+                let err = new Error('Invalid server response');
+                err.code = 'InvalidResponse';
+                err.response = parsed;
+                request.reject(err);
+                break;
             }
         }
     }
@@ -939,6 +1119,29 @@ class ImapFlow extends EventEmitter {
         this.streamer.on('readable', this.socketReadable);
     }
 
+    /**
+     * Applies the transport options every established application socket needs: TCP keepalive and
+     * the inactivity watchdog. Called for direct TLS, cleartext, proxied and STARTTLS-upgraded
+     * sockets, so the watchdog cannot silently differ between transports (a STARTTLS session used
+     * to end up with no armed timer at all).
+     *
+     * @param {Object} socket - The socket that now carries the IMAP session.
+     */
+    configureSocket(socket) {
+        /* c8 ignore next 3 */ // defensive: connect() only calls this with an established socket
+        if (!socket) {
+            return;
+        }
+
+        if (typeof socket.setKeepAlive === 'function') {
+            socket.setKeepAlive(true, 5 * 1000);
+        }
+
+        if (typeof socket.setTimeout === 'function') {
+            socket.setTimeout(this.socketTimeout);
+        }
+    }
+
     setSocketHandlers() {
         // Clear any existing handlers first to prevent duplicates
         this.clearSocketHandlers();
@@ -971,7 +1174,11 @@ class ImapFlow extends EventEmitter {
                         this.emitError(err);
                         return;
                     }
-                    // Attempt to recover IDLE connections
+                    // Attempt to recover IDLE connections. During true IDLE the NOOP cannot
+                    // reach the server until IDLE has been terminated: run() awaits preCheck(),
+                    // which sends DONE and only resolves once the server has completed the IDLE
+                    // command. Fallback polling has no such handshake - preCheck() there just
+                    // cancels the polling session.
                     this.run('NOOP')
                         .then(() => this.idle())
                         .catch(err => {
@@ -1150,9 +1357,10 @@ class ImapFlow extends EventEmitter {
         };
         /* c8 ignore stop */
 
-        Object.defineProperty(this.writeSocket, 'destroyed', {
-            get: () => !this.socket || this.socket.destroyed
-        });
+        // The PassThrough reports its own `destroyed` state. It used to proxy the raw socket's
+        // instead, which made close() skip destroying it and left the second raw-socket teardown
+        // branch unreachable. write() checks the raw socket separately, so nothing depends on the
+        // two states being conflated.
 
         // Manual pump loop: reads chunks from writeSocket, pushes them into
         // deflate, and flushes when the buffer is drained. This ensures each
@@ -1289,8 +1497,6 @@ class ImapFlow extends EventEmitter {
             throw failSTARTTLSInjection();
         }
         let upgraded = await new Promise((resolve, reject) => {
-            // Expose this rejector so emitError() can settle the upgrade with a streamer error.
-            this._upgradeReject = reject;
             let socketPlain = this.socket;
             let opts = Object.assign(
                 {
@@ -1302,71 +1508,73 @@ class ImapFlow extends EventEmitter {
             );
             this.clearSocketHandlers();
 
-            // Store error handler for cleanup after successful upgrade
-            /* c8 ignore start */ // plain-socket error during the TLS handshake window is timing-dependent (errors surface via the streamer/emitError path in tests)
-            const socketPlainErrorHandler = err => {
-                clearTimeout(this.connectTimeout);
-                clearTimeout(this.upgradeTimeout);
-                if (!this.upgrading) {
-                    // don't care anymore
+            let settled = false;
+
+            // Single settlement path for the upgrade. Every terminal outcome - handshake
+            // success, an error on the plain or the TLS socket, the upgrade timeout, an
+            // explicit close(), or a streamer error routed here by emitError() - goes through
+            // this helper exactly once. It owns clearing the upgrade timer, the exposed
+            // rejector, the `upgrading` flag and the temporary handshake handlers, so a late
+            // socket event cannot re-enter an already settled upgrade or leave state behind.
+            const settle = (err, result) => {
+                if (settled) {
                     return;
                 }
-                this.closeAfter();
+                settled = true;
+
+                clearTimeout(this.upgradeTimeout);
+                this.upgradeTimeout = null;
                 this.upgrading = false;
-                err.tlsFailed = true;
-                reject(err);
+                this._upgradeReject = null;
+
+                socketPlain.removeListener('error', settle);
+                if (this.socket && this.socket !== socketPlain) {
+                    this.socket.removeListener('error', settle);
+                }
+
+                if (err) {
+                    clearTimeout(this.connectTimeout);
+                    // Preserve the original error, marked as a TLS failure so callers can tell
+                    // an upgrade failure from an ordinary command failure.
+                    err.tlsFailed = true;
+                    this.closeAfter();
+                    return reject(err);
+                }
+
+                resolve(result);
             };
-            /* c8 ignore stop */
-            socketPlain.once('error', socketPlainErrorHandler);
+
+            // Exposed so emitError() and close() can settle the upgrade through the same path.
+            this._upgradeReject = settle;
+
+            // An error on either socket settles the upgrade, so settle() is the listener itself:
+            // one function, one settlement, and removeListener() in settle() needs no separate
+            // handler references. A TLS handshake failure (bad certificate, protocol mismatch)
+            // is emitted on the new TLS socket rather than on the plain one, so both are covered.
+            socketPlain.once('error', settle);
 
             /* c8 ignore start */ // UPGRADE_TIMEOUT is 10s; firing it deterministically would make the test suite hang
             this.upgradeTimeout = setTimeout(() => {
-                if (!this.upgrading) {
-                    return;
-                }
-                this.closeAfter();
                 let err = new Error('Failed to upgrade connection in required time');
-                err.tlsFailed = true;
                 err.code = 'UPGRADE_TIMEOUT';
-                reject(err);
+                settle(err);
             }, UPGRADE_TIMEOUT);
             /* c8 ignore stop */
 
-            // A TLS handshake failure (bad certificate, protocol mismatch, etc.) is emitted on the
-            // new TLS socket, not on the plain socket, so it must be handled here. Without this the
-            // upgrade promise would only settle via the timeout and connect() would reject with a
-            // generic "Unexpected close" instead of the actual TLS error.
-            const tlsSocketErrorHandler = err => {
-                clearTimeout(this.connectTimeout);
-                clearTimeout(this.upgradeTimeout);
-                /* c8 ignore start */ // the already-settled early return is a defensive double-fire guard, not separately exercised
-                if (!this.upgrading) {
-                    // already settled
-                    return;
-                }
-                /* c8 ignore stop */
-                this.upgrading = false;
-                err.tlsFailed = true;
-                this.clearSocketHandlers();
-                this.closeAfter();
-                reject(err);
-            };
-
             this.upgrading = true;
             this.socket = tls.connect(opts, () => {
                 try {
-                    clearTimeout(this.upgradeTimeout);
                     /* c8 ignore start */ // race: connection closed during the TLS handshake window
                     if (this.isClosed) {
-                        // not sure if this is possible?
-                        return this.close();
+                        let err = new Error('Connection closed during TLS upgrade');
+                        err.code = 'NoConnection';
+                        return settle(err);
                     }
                     /* c8 ignore stop */
 
                     // TLS handshake complete. Reconnect the now-encrypted socket
                     // to the IMAP parser stream and record the cipher details.
                     this.secureConnection = true;
-                    this.upgrading = false;
                     this.streamer.secureConnection = true;
                     this.socket.pipe(this.streamer);
                     /* c8 ignore next */ // an upgraded TLS socket always exposes getCipher(), so the false fallback is unreachable
@@ -1384,21 +1592,28 @@ class ImapFlow extends EventEmitter {
                         });
                     }
 
-                    // Clean up the error handlers after successful upgrade
-                    socketPlain.removeListener('error', socketPlainErrorHandler);
-                    this.socket.removeListener('error', tlsSocketErrorHandler);
+                    // The plain socket is now only the TLS transport: drop its superseded
+                    // inactivity timer so no armed timer is left behind without a listener.
+                    if (typeof socketPlain.setTimeout === 'function') {
+                        socketPlain.setTimeout(0);
+                    }
 
                     // Install the normal socket handlers only now that the handshake
-                    // succeeded. Doing this during the handshake would leave both
-                    // tlsSocketErrorHandler and the generic _socketError on the socket;
-                    // a handshake 'error' would then fire BOTH (EventEmitter clones its
-                    // listener array on emit), causing a duplicate error and a possible
-                    // unhandled 'error' crash. Keeping tlsSocketErrorHandler as the sole
+                    // succeeded. Doing this during the handshake would leave both settle() and
+                    // the generic _socketError on the socket; a handshake 'error' would then fire
+                    // BOTH (EventEmitter clones its listener array on emit), causing a duplicate
+                    // error and a possible unhandled 'error' crash. Keeping settle() as the sole
                     // listener until here guarantees a single error path for the upgrade.
                     this.setSocketHandlers();
 
-                    this._upgradeReject = null;
-                    return resolve(true);
+                    // Arm the inactivity watchdog on the socket that now carries the session.
+                    // Without this a STARTTLS-upgraded connection has no watchdog at all: the
+                    // timer was armed on the plain socket, while the timeout listener lives on
+                    // the TLS socket.
+                    this.configureSocket(this.socket);
+
+                    // settle() also removes the temporary handshake handlers
+                    settle(null, true);
                     /* c8 ignore next 3 */ // defensive: the success callback body does not throw under normal operation
                 } catch (ex) {
                     this.emitError(ex);
@@ -1408,8 +1623,8 @@ class ImapFlow extends EventEmitter {
             // Registered after tls.connect (the TLS socket now exists). This is the ONLY
             // error listener during the handshake window; the generic handlers are installed
             // by setSocketHandlers() inside the success callback above, so a handshake error
-            // has a single error path (tlsSocketErrorHandler -> reject).
-            this.socket.once('error', tlsSocketErrorHandler);
+            // has a single error path.
+            this.socket.once('error', settle);
 
             this.writeSocket = this.socket;
         });
@@ -1778,6 +1993,11 @@ class ImapFlow extends EventEmitter {
         return range;
     }
 
+    // Timer process-liveness policy: connection establishment and greeting deadlines keep the
+    // process alive, because a caller is waiting on connect() to settle. Background timers
+    // (auto-IDLE, IDLE restart, fallback polling, throttle back-off, the held-lock diagnostic) are
+    // unref'd, so an otherwise idle process is not held open by them. Every timer is still cleared
+    // explicitly on close().
     autoidle() {
         clearTimeout(this.idleStartTimer);
         if (this.options.disableAutoIdle || this.state !== this.states.SELECTED) {
@@ -1786,6 +2006,7 @@ class ImapFlow extends EventEmitter {
         this.idleStartTimer = setTimeout(() => {
             this.idle().catch(err => this.log.warn({ err, cid: this.id }));
         }, 15 * 1000);
+        unrefTimer(this.idleStartTimer);
     }
 
     // PUBLIC API METHODS
@@ -1806,6 +2027,11 @@ class ImapFlow extends EventEmitter {
         }
         this._connectCalled = true;
 
+        // One deadline for the whole attempt, started before anything is resolved or negotiated.
+        // Proxy DNS and proxy negotiation used to run entirely outside the timer, so a stalled
+        // proxy could hang far beyond the documented connectionTimeout.
+        let deadline = new ConnectionDeadline(this.options.connectionTimeout);
+
         let connector = this.secureConnection ? tls : net;
 
         let opts = Object.assign(
@@ -1834,11 +2060,17 @@ class ImapFlow extends EventEmitter {
         let socket = false;
         if (this.options.proxy) {
             try {
-                socket = await proxyConnection(this.log, this.options.proxy, this.host, this.port);
+                socket = await proxyConnection(this.log, this.options.proxy, this.host, this.port, { deadline });
                 if (!socket) {
                     throw new Error('Failed to setup proxy connection');
                 }
             } catch (err) {
+                if (err.code === 'CONNECT_TIMEOUT') {
+                    // The shared deadline expired during proxy setup. Report it as the documented
+                    // connection timeout rather than as a generic proxy failure.
+                    this.log.error({ err, cid: this.id });
+                    throw err;
+                }
                 let error = new Error('Failed to setup proxy connection');
                 error.code = err.code || 'ProxyError';
                 error._err = err;
@@ -1848,17 +2080,13 @@ class ImapFlow extends EventEmitter {
         }
 
         let connectPromise = new Promise((resolve, reject) => {
+            // Whatever the proxy phase already used is gone from the budget
             this.connectTimeout = setTimeout(() => {
-                let err = new Error('Failed to establish connection in required time');
-                err.code = 'CONNECT_TIMEOUT';
-                err.details = {
-                    /* c8 ignore next */ // firing the timeout with the default (large) value would hang the suite, so only the explicit-option path is tested
-                    connectionTimeout: this.options.connectionTimeout || CONNECT_TIMEOUT
-                };
+                let err = deadline.error();
                 this.log.error({ err, cid: this.id });
                 this.closeAfter();
                 reject(err);
-            }, this.options.connectionTimeout || CONNECT_TIMEOUT);
+            }, deadline.remaining());
 
             let onConnect = () => {
                 try {
@@ -1868,8 +2096,7 @@ class ImapFlow extends EventEmitter {
                     // (its "before connection setup" message no longer applies).
                     detachEarlyErrorHandler(socket);
 
-                    this.socket.setKeepAlive(true, 5 * 1000);
-                    this.socket.setTimeout(this.options.socketTimeout || SOCKET_TIMEOUT);
+                    this.configureSocket(this.socket);
 
                     this.greetingTimeout = setTimeout(() => {
                         let err = new Error(
@@ -2020,8 +2247,23 @@ class ImapFlow extends EventEmitter {
             }
 
             this.usable = false;
+            // close() takes over ownership of the idling state: dropping the session token means a
+            // poll or IDLE that unwinds after this point sees that it no longer owns the flag and
+            // leaves it alone (see claimIdling() in commands/idle.js).
+            this._idleSession = null;
             this.idling = false;
 
+            // An in-flight STARTTLS upgrade has to be settled through its own single settlement
+            // path, otherwise the upgrade promise (and the session it belongs to) stays pending
+            // for the lifetime of the process.
+            if (typeof this._upgradeReject === 'function') {
+                let reject = this._upgradeReject;
+                this._upgradeReject = null;
+                let err = new Error('Connection closed during TLS upgrade');
+                err.code = 'NoConnection';
+                reject(err);
+            }
+
             if (typeof this.initialReject === 'function' && !this.options.verifyOnly) {
                 clearTimeout(this.greetingTimeout);
                 let reject = this.initialReject;
@@ -2045,6 +2287,20 @@ class ImapFlow extends EventEmitter {
                 this.preCheck().catch(err => this.log.warn({ err, cid: this.id }));
             }
 
+            // Session-only public state must not survive the connection it describes: callers read
+            // these properties in reconnect logic and would otherwise mistake cached objects for
+            // live server state. Cleared during the first close only, so repeated close() calls
+            // stay idempotent and cannot emit an event twice.
+            // `byeReason` is deliberately kept: it explains why the session ended.
+            let closedMailbox = false;
+            if (!this.isClosed) {
+                closedMailbox = this.mailbox;
+                this.mailbox = false;
+                this.currentSelectCommand = false;
+                this.authenticated = false;
+                this.preCheck = false;
+            }
+
             // Collect all pending requests to reject
             let pendingRequests = [];
 
@@ -2156,22 +2412,16 @@ class ImapFlow extends EventEmitter {
             if (this.isClosed) {
                 return;
             }
-
-            // Socket teardown order matters when compression is active:
-            // writeSocket may be a PassThrough (compression) or the raw socket (no compression).
-            // Destroy the underlying socket first, then writeSocket (if different).
-            // The second socket.destroy() block handles the case where writeSocket.destroy()
-            // did not also destroy the underlying socket.
-            if (this.socket && !this.socket.destroyed && this.writeSocket !== this.socket) {
-                try {
-                    this.socket.destroy();
-                } catch (err) {
-                    this.log.error({ err, cid: this.id });
-                }
-            }
+            // Set before teardown so a socket event that re-enters close() during destruction
+            // cannot run this block a second time.
             this.isClosed = true;
 
-            if (this.writeSocket && !this.writeSocket.destroyed) {
+            // Socket teardown, in one documented order. Each stream owns and reports its own
+            // lifecycle, so each is destroyed exactly once:
+            //   1. the compression PassThrough (writeSocket), if compression replaced it
+            //   2. the raw socket, which is also writeSocket when compression is not active
+            // The compression streams themselves were destroyed above.
+            if (this.writeSocket && this.writeSocket !== this.socket && !this.writeSocket.destroyed) {
                 try {
                     this.writeSocket.destroy();
                 } catch (err) {
@@ -2179,7 +2429,7 @@ class ImapFlow extends EventEmitter {
                 }
             }
 
-            if (this.socket && !this.socket.destroyed && this.writeSocket !== this.socket) {
+            if (this.socket && !this.socket.destroyed) {
                 try {
                     this.socket.destroy();
                 } catch (err) {
@@ -2200,7 +2450,19 @@ class ImapFlow extends EventEmitter {
             this._socketEnd = null;
             this._socketTimeout = null;
 
-            this.log.trace({ msg: 'Connection closed', cid: this.id });
+            this.log.trace({
+                msg: 'Connection closed',
+                cid: this.id,
+                ...(this._unknownTagCount ? { unknownTagCount: this._unknownTagCount } : {})
+            });
+
+            // A mailbox that was still selected is now closed, so the transition is reported once,
+            // whether the session ended with a clean logout or a lost transport. Emitted before
+            // 'close' and only from the first close(), so no consumer sees it twice.
+            if (closedMailbox) {
+                this.emit('mailboxClose', closedMailbox);
+            }
+
             this.emit('close');
         } catch (ex) {
             // close failed
@@ -2246,6 +2508,7 @@ class ImapFlow extends EventEmitter {
      * @property {String} parentPath Same as `parent`, but as a complete string path (unicode string)
      * @property {Set<string>} flags a set of flags for this mailbox
      * @property {String} specialUse one of special-use flags (if applicable): "\All", "\Archive", "\Drafts", "\Flagged", "\Junk", "\Sent", "\Trash". Additionally INBOX has non-standard "\Inbox" flag set
+     * @property {String} [specialUseSource] how `specialUse` was determined: `"user"` (from `specialUseHints`), `"extension"` (SPECIAL-USE or XLIST flag reported by the server) or `"name"` (matched against known localized folder names)
      * @property {Boolean} listed `true` if mailbox was found from the output of LIST command
      * @property {Boolean} subscribed `true` if the mailbox is subscribed - reported by LSUB or by LIST RETURN (SUBSCRIBED) on LIST-EXTENDED/IMAP4rev2 servers
      * @property {StatusObject} [status] If `statusQuery` was used, then this value includes the status response
@@ -2268,6 +2531,7 @@ class ImapFlow extends EventEmitter {
      * @property {String} [specialUseHints.trash] Path to "Trash" folder
      * @property {String} [specialUseHints.junk] Path to "Junk Mail" folder
      * @property {String} [specialUseHints.drafts] Path to "Drafts" folder
+     * @property {String} [specialUseHints.archive] Path to "Archive" folder
      */
 
     /**
@@ -3752,9 +4016,7 @@ class ImapFlow extends EventEmitter {
         }
 
         if (!this.socket || this.socket.destroyed) {
-            const error = new Error('Connection not available');
-            error.code = 'NoConnection';
-            throw error;
+            throw this.createNoConnectionError();
         }
 
         clearTimeout(this.idleStartTimer);
@@ -3763,9 +4025,7 @@ class ImapFlow extends EventEmitter {
             await this.preCheck();
         }
 
-        let handler = this.commands.get(command);
-
-        let result = await handler(this, ...args);
+        let result = await this.runInternal(command, ...args);
 
         if (command !== 'IDLE') {
             // do not autostart IDLE, if IDLE itself was stopped
@@ -3775,6 +4035,33 @@ class ImapFlow extends EventEmitter {
         return result;
     }
 
+    /**
+     * Dispatches a command without the IDLE handshake that `run()` performs.
+     *
+     * Used by callers that already own the connection's idle state - fallback polling issues its
+     * commands through here, because `run()` would await `preCheck()`, and the preCheck it would
+     * await belongs to the very polling session making the call, so the session would cancel
+     * itself. Auto-IDLE is not restarted either, for the same reason: the caller is the idle loop.
+     *
+     * @param {String} command Command name, as registered in the command registry.
+     * @param {...*} args Arguments forwarded to the command implementation.
+     * @returns {Promise<*>} Whatever the command implementation returns, or `false` for an
+     *   unknown command.
+     */
+    async runInternal(command, ...args) {
+        command = command.toUpperCase();
+        if (!this.commands.has(command)) {
+            return false;
+        }
+
+        if (!this.socket || this.socket.destroyed) {
+            throw this.createNoConnectionError();
+        }
+
+        let handler = this.commands.get(command);
+        return await handler(this, ...args);
+    }
+
     // Mailbox lock queue processor. Implements a mutex pattern: only one lock
     // is active at a time. When the active lock is released, the next queued
     // lock is processed. The `processingLock` flag prevents concurrent runs
@@ -3832,6 +4119,7 @@ class ImapFlow extends EventEmitter {
                         return;
                     }
                     lock.heldAt = Date.now();
+                    // Background diagnostic: must not keep the process alive on its own
                     lock.heldWarnTimer = setTimeout(() => {
                         lock.heldWarnTimer = null;
                         this.log.warn({
@@ -3844,6 +4132,7 @@ class ImapFlow extends EventEmitter {
                             cid: this.id
                         });
                     }, threshold);
+                    unrefTimer(lock.heldWarnTimer);
                 };
 
                 // release() is captured per-lock. It must only clear this.currentLock
@@ -4199,6 +4488,11 @@ class ImapFlow extends EventEmitter {
 /**
  * Mailbox was closed
  *
+ * Emitted both when a selected mailbox is closed explicitly, by `mailboxClose()` or by
+ * selecting a different mailbox, and when the connection itself goes away while a mailbox
+ * was still selected, whether through a clean logout or a lost transport. The transition is
+ * reported once per selected mailbox, before the `close` event.
+ *
  * @event module:imapflow~ImapFlow#mailboxClose
  * @type {MailboxObject}
  * @example