imapflow 1.4.8 → 1.5.0

package/lib/commands/status.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const { encodePath, normalizePath } = require('../tools.js');
+const { encodePath, normalizePath, buildStatusQueryAttributes, isRev2Active } = require('../tools.js');
 
 /**
  * Requests status information about a mailbox.
@@ -24,33 +24,18 @@ module.exports = async (connection, path, query) => {
     // otherwise use unquoted ATOM. Same approach as in SELECT.
     let attributes = [{ type: encodedPath.indexOf('&') >= 0 ? 'STRING' : 'ATOM', value: encodedPath }];
 
-    // Build the list of STATUS data items the caller wants.
-    // HIGHESTMODSEQ requires the CONDSTORE extension to be available.
-    let queryAttributes = [];
-    Object.keys(query || {}).forEach(key => {
-        if (!query[key]) {
-            return;
-        }
+    // Build the list of STATUS data items the caller wants
+    let queryAttributes = buildStatusQueryAttributes(connection, query);
 
-        switch (key.toUpperCase()) {
-            case 'MESSAGES':
-            case 'RECENT':
-            case 'UIDNEXT':
-            case 'UIDVALIDITY':
-            case 'UNSEEN':
-                queryAttributes.push({ type: 'ATOM', value: key.toUpperCase() });
-                break;
-
-            case 'HIGHESTMODSEQ':
-                if (connection.capabilities.has('CONDSTORE')) {
-                    queryAttributes.push({ type: 'ATOM', value: key.toUpperCase() });
-                }
-                break;
-        }
-    });
+    // RECENT does not exist in IMAP4rev2 so it is never requested from a rev2
+    // session; its defined value there is always 0. Synthesizing it keeps the
+    // return shape identical to a rev1 session for the same query.
+    let syntheticRecent = query && query.recent && isRev2Active(connection);
 
     if (!queryAttributes.length) {
-        return false;
+        // A query that only contained items unavailable on this session - the
+        // caller still gets a status object if every such item has a defined value
+        return syntheticRecent ? { path, recent: 0 } : false;
     }
 
     attributes.push(queryAttributes);
@@ -101,7 +86,12 @@ module.exports = async (connection, path, query) => {
                             updateMailbox: (val, conn) => {
                                 conn.mailbox.highestModseq = val;
                             }
-                        }
+                        },
+                        // IMAP4rev2 additions (RFC 9051): total mailbox size in octets
+                        // (number64, exact as a JS number up to 2^53-1) and count of
+                        // messages with the \Deleted flag
+                        SIZE: { key: 'size', parser: Number },
+                        DELETED: { key: 'deleted', parser: Number }
                     };
 
                     let key;
@@ -134,6 +124,9 @@ module.exports = async (connection, path, query) => {
             }
         });
         response.next();
+        if (syntheticRecent) {
+            map.recent = 0;
+        }
         return map;
     } catch (err) {
         // A NO response usually means the mailbox doesn't exist. Verify by

package/lib/handler/imap-compiler.js

@@ -130,15 +130,18 @@ module.exports = async (response, options) => {
                 if (isLogging) {
                     resp.push(formatRespEntry('"(* ' + node.value.length + 'B literal *)"'));
                 } else {
-                    let literalLength = !node.value ? 0 : Math.max(node.value.length, 0);
-
-                    // canAppend: whether the literal data can be sent in the same buffer segment.
-                    // With LITERAL+ (RFC 7888) the client does not wait for a continuation response.
-                    // With LITERAL- (RFC 7888) the client can skip the wait only for literals <= 4096 bytes.
-                    // When asArray is false we always append inline (single-buffer mode).
-                    let canAppend = !asArray || literalPlus || (literalMinus && literalLength <= 4096);
-                    // Append '+' to the size marker when using LITERAL+ or LITERAL- (non-synchronizing)
-                    let usePlus = canAppend && (literalMinus || literalPlus);
+                    // The literal size marker counts octets - string values are written as
+                    // UTF-8, so their UTF-16 .length would undercount multi-byte characters
+                    let literalLength = !node.value ? 0 : Buffer.isBuffer(node.value) ? node.value.length : Buffer.byteLength(node.value.toString());
+
+                    // Append '+' to the size marker only when the extension actually permits a
+                    // non-synchronizing literal of this size (RFC 7888): LITERAL+ always,
+                    // LITERAL- only up to 4096 bytes
+                    let usePlus = literalPlus || (literalMinus && literalLength <= 4096);
+                    // canAppend: whether the literal data can be sent in the same buffer segment -
+                    // non-synchronizing literals always, and everything in single-buffer mode
+                    // (asArray false), which has no continuation flow
+                    let canAppend = !asArray || usePlus;
 
                     // Emit the literal header: optional '~' prefix for literal8, then {size[+]}\r\n
                     resp.push(formatRespEntry(`${node.isLiteral8 ? '~' : ''}{${literalLength}${usePlus ? '+' : ''}}\r\n`));

package/lib/handler/token-parser.js

@@ -580,6 +580,13 @@ class TokenParser {
                         if (!this.currentNode.literalLength) {
                             // special case where literal content length is 0
                             // close the node right away, do not wait for additional input
+                            if (this.options.literals && this.options.literals.length) {
+                                // ImapStream queues a Buffer for every literal marker it
+                                // extracts, including {0} - consume the queue entry so
+                                // subsequent literals in the same response stay aligned
+                                // with their markers instead of shifting by one
+                                this.currentNode.value = this.options.literals.shift();
+                            }
                             this.currentNode.endPos = this.pos + i;
                             this.currentNode.isClosed = true;
                             this.currentNode = this.currentNode.parentNode;

package/lib/imap-flow.d.ts

@@ -54,6 +54,8 @@ export interface ImapFlowOptions {
     disableBinary?: boolean;
     /** If true, do not enable supported extensions */
     disableAutoEnable?: boolean;
+    /** If true, do not enable IMAP4rev2 mode even if the server supports it */
+    disableIMAP4rev2?: boolean;
     /** How long to wait for the connection to be established. Defaults to 90 seconds */
     connectionTimeout?: number;
     /** How long to wait for the greeting. Defaults to 16 seconds */
@@ -103,7 +105,7 @@ export interface MailboxObject {
     specialUse?: string;
     /** True if mailbox was found from the output of LIST command */
     listed?: boolean;
-    /** True if mailbox was found from the output of LSUB command */
+    /** True if the mailbox is subscribed - reported by LSUB or by LIST RETURN (SUBSCRIBED) on LIST-EXTENDED/IMAP4rev2 servers */
     subscribed?: boolean;
     /** A Set of flags available to use in this mailbox. If it is not set or includes special flag "\*" then any flag can be used */
     permanentFlags?: Set<string>;
@@ -184,7 +186,7 @@ export interface ListResponse {
     specialUse?: string;
     /** True if mailbox was found from the output of LIST command */
     listed: boolean;
-    /** True if mailbox was found from the output of LSUB command */
+    /** True if the mailbox is subscribed - reported by LSUB or by LIST RETURN (SUBSCRIBED) on LIST-EXTENDED/IMAP4rev2 servers */
     subscribed: boolean;
     /** If statusQuery was used, then this value includes the status response */
     status?: StatusObject;
@@ -205,6 +207,10 @@ export interface ListOptions {
         unseen?: boolean;
         /** If true request last known modseq value */
         highestModseq?: boolean;
+        /** If true request total mailbox size in octets (requires STATUS=SIZE or IMAP4rev2) */
+        size?: boolean;
+        /** If true request count of messages with \Deleted flag (requires IMAP4rev2) */
+        deleted?: boolean;
     };
     /** Set specific paths as special use folders */
     specialUseHints?: {
@@ -234,7 +240,7 @@ export interface ListTreeResponse {
     specialUse?: string;
     /** True if mailbox was found from the output of LIST command */
     listed?: boolean;
-    /** True if mailbox was found from the output of LSUB command */
+    /** True if the mailbox is subscribed - reported by LSUB or by LIST RETURN (SUBSCRIBED) on LIST-EXTENDED/IMAP4rev2 servers */
     subscribed?: boolean;
     /** If true then this mailbox can not be selected in the UI */
     disabled?: boolean;
@@ -280,6 +286,10 @@ export interface StatusObject {
     unseen?: number;
     /** Last known modseq value (if CONDSTORE extension is enabled) */
     highestModseq?: bigint;
+    /** Total size of the mailbox in octets (only if requested and the server supports STATUS=SIZE or IMAP4rev2) */
+    size?: number;
+    /** Count of messages with \Deleted flag (only if requested and IMAP4rev2 is active) */
+    deleted?: number;
 }
 
 export type SequenceString = string | number | bigint;
@@ -491,6 +501,8 @@ export interface FetchMessageObject {
     internalDate?: Date | string;
     /** A Map of message body parts where key is requested part identifier and value is a Buffer */
     bodyParts?: Map<string, Buffer>;
+    /** Part identifiers from bodyParts that arrived via FETCH BINARY, i.e. with the content-transfer-encoding already decoded by the server */
+    binaryParts?: Set<string>;
     /** Requested header lines as Buffer */
     headers?: Buffer;
     /** Account unique ID for this email */
@@ -753,6 +765,10 @@ export class ImapFlow extends EventEmitter {
             uidValidity?: boolean;
             unseen?: boolean;
             highestModseq?: boolean;
+            /** Requires STATUS=SIZE or IMAP4rev2 */
+            size?: boolean;
+            /** Requires IMAP4rev2 */
+            deleted?: boolean;
         }
     ): Promise<StatusObject>;
 

package/lib/imap-flow.js

@@ -35,7 +35,8 @@ const {
     normalizePath,
     expandRange,
     AuthenticationFailure,
-    getColorFlags
+    getColorFlags,
+    hasCapability
 } = require('./tools');
 
 const imapCommands = require('./imap-commands.js');
@@ -69,7 +70,7 @@ const states = {
  * @property {Set<string>} flags list of flags for this mailbox
  * @property {String} [specialUse] one of special-use flags (if applicable): "\All", "\Archive", "\Drafts", "\Flagged", "\Junk", "\Sent", "\Trash". Additionally INBOX has non-standard "\Inbox" flag set
  * @property {Boolean} listed `true` if mailbox was found from the output of LIST command
- * @property {Boolean} subscribed `true` if mailbox was found from the output of LSUB command
+ * @property {Boolean} subscribed `true` if the mailbox is subscribed - reported by LSUB or by LIST RETURN (SUBSCRIBED) on LIST-EXTENDED/IMAP4rev2 servers
  * @property {Set<string>} permanentFlags A Set of flags available to use in this mailbox. If it is not set or includes special flag "\\\*" then any flag can be used.
  * @property {String} [mailboxId] unique mailbox ID if server has `OBJECTID` extension enabled
  * @property {BigInt} [highestModseq] latest known modseq value if server has CONDSTORE or XYMHIGHESTMODSEQ enabled
@@ -221,6 +222,11 @@ class ImapFlow extends EventEmitter {
      * @property {Boolean} [disableAutoEnable=false]
      *     If `true`, do not automatically enable supported IMAP extensions.
      *
+     * @property {Boolean} [disableIMAP4rev2=false]
+     *     If `true`, do not enable IMAP4rev2 mode even if the server supports it.
+     *     Use as a targeted opt-out for servers with broken IMAP4rev2 implementations
+     *     without losing the other auto-enabled extensions.
+     *
      * @property {Number} [connectionTimeout=90000]
      *     Maximum time (in milliseconds) to wait for the connection to establish. Defaults to 90 seconds.
      *
@@ -397,6 +403,14 @@ class ImapFlow extends EventEmitter {
 
         this.disableBinary = !!this.options.disableBinary;
 
+        // Set when the server rejects a LIST RETURN option group, the auxiliary
+        // SPECIAL-USE/CHILDREN return options, or the LSUB command, so later
+        // listings on this connection skip what the server does not support
+        this.skipListSubscribedArg = false;
+        this.skipListStatusArgs = false;
+        this.skipListAuxArgs = false;
+        this.skipLsub = false;
+
         // Named error handler for proper cleanup. Certain error codes represent
         // expected socket/network issues (buffer exhaustion, connection reset, broken pipe,
         // timeout, unreachable host) that just need a silent connection close rather
@@ -568,7 +582,8 @@ class ImapFlow extends EventEmitter {
         // are stored in this.commandParts and sent after server "+" continuations.
         let compiled = await compiler(data, {
             asArray: true,
-            literalMinus: this.capabilities.has('LITERAL-') || this.capabilities.has('LITERAL+')
+            // LITERAL- is part of base IMAP4rev2
+            literalMinus: hasCapability(this, 'LITERAL-') || this.capabilities.has('LITERAL+')
         });
         this.commandParts = compiled;
 
@@ -1053,13 +1068,29 @@ class ImapFlow extends EventEmitter {
         }
 
         if (!this.options.disableAutoEnable) {
-            // enable extensions if possible
-            await this.run('ENABLE', ['CONDSTORE', 'UTF8=ACCEPT'].concat(this.options.qresync ? 'QRESYNC' : []));
+            await this.autoEnable();
         }
 
         this.usable = true;
     }
 
+    // Enable extensions if possible. IMAP4rev2 must be enabled explicitly on
+    // servers that advertise both rev1 and rev2 (RFC 9051 Appendix A); a single
+    // ENABLE call is used so the enabled set is built in one round trip.
+    async autoEnable() {
+        let enableList = ['CONDSTORE', 'UTF8=ACCEPT'].concat(this.options.qresync ? 'QRESYNC' : []).concat(this.options.disableIMAP4rev2 ? [] : 'IMAP4rev2');
+        let enableResult = await this.run('ENABLE', enableList);
+        if (enableResult === false && enableList.includes('IMAP4rev2')) {
+            // RFC 5161 requires servers to ignore unknown ENABLE arguments, but a
+            // broken implementation may reject the whole command over IMAP4rev2 -
+            // retry without it so CONDSTORE/QRESYNC are not lost as collateral
+            await this.run(
+                'ENABLE',
+                enableList.filter(extension => extension !== 'IMAP4rev2')
+            );
+        }
+    }
+
     async compress() {
         if (!(await this.run('COMPRESS'))) {
             return; // was not able to negotiate compression
@@ -1510,6 +1541,9 @@ class ImapFlow extends EventEmitter {
             return;
         }
         this.state = this.states.AUTHENTICATED;
+        // documented contract for the `authenticated` property: `true` when the
+        // connection was authenticated by a PREAUTH greeting (no credentials known)
+        this.authenticated = true;
         this.beginSession(err => {
             this.log.error({ err, cid: this.id });
             this.closeAfter();
@@ -2213,7 +2247,7 @@ class ImapFlow extends EventEmitter {
      * @property {Set<string>} flags a set of flags for this mailbox
      * @property {String} specialUse one of special-use flags (if applicable): "\All", "\Archive", "\Drafts", "\Flagged", "\Junk", "\Sent", "\Trash". Additionally INBOX has non-standard "\Inbox" flag set
      * @property {Boolean} listed `true` if mailbox was found from the output of LIST command
-     * @property {Boolean} subscribed `true` if mailbox was found from the output of LSUB command
+     * @property {Boolean} subscribed `true` if the mailbox is subscribed - reported by LSUB or by LIST RETURN (SUBSCRIBED) on LIST-EXTENDED/IMAP4rev2 servers
      * @property {StatusObject} [status] If `statusQuery` was used, then this value includes the status response
      */
 
@@ -2227,6 +2261,8 @@ class ImapFlow extends EventEmitter {
      * @property {Boolean} [statusQuery.uidValidity] if `true` request mailbox `UIDVALIDITY` value
      * @property {Boolean} [statusQuery.unseen] if `true` request count of unseen messages
      * @property {Boolean} [statusQuery.highestModseq] if `true` request last known modseq value
+     * @property {Boolean} [statusQuery.size] if `true` request total mailbox size in octets (requires STATUS=SIZE or IMAP4rev2)
+     * @property {Boolean} [statusQuery.deleted] if `true` request count of messages with \\Deleted flag (requires IMAP4rev2)
      * @property {Object} [specialUseHints] set specific paths as special use folders, this would override special use flags provided from the server
      * @property {String} [specialUseHints.sent] Path to "Sent Mail" folder
      * @property {String} [specialUseHints.trash] Path to "Trash" folder
@@ -2261,9 +2297,10 @@ class ImapFlow extends EventEmitter {
      * @property {Set<string>} flags list of flags for this mailbox
      * @property {String} specialUse one of special-use flags (if applicable): "\All", "\Archive", "\Drafts", "\Flagged", "\Junk", "\Sent", "\Trash". Additionally INBOX has non-standard "\Inbox" flag set
      * @property {Boolean} listed `true` if mailbox was found from the output of LIST command
-     * @property {Boolean} subscribed `true` if mailbox was found from the output of LSUB command
+     * @property {Boolean} subscribed `true` if the mailbox is subscribed - reported by LSUB or by LIST RETURN (SUBSCRIBED) on LIST-EXTENDED/IMAP4rev2 servers
      * @property {Boolean} disabled If `true` then this mailbox can not be selected in the UI
      * @property {ListTreeResponse[]} folders An array of subfolders
+     * @property {StatusObject} [status] If `statusQuery` was used, then this value includes the status response
      */
 
     /**
@@ -2430,6 +2467,8 @@ class ImapFlow extends EventEmitter {
      * @property {BigInt} [uidValidity] Mailbox `UIDVALIDITY` value
      * @property {Number} [unseen] Count of unseen messages
      * @property {BigInt} [highestModseq] Last known modseq value (if CONDSTORE extension is enabled)
+     * @property {Number} [size] Total size of the mailbox in octets (only if requested and the server supports STATUS=SIZE or IMAP4rev2)
+     * @property {Number} [deleted] Count of messages with \\Deleted flag (only if requested and IMAP4rev2 is active)
      */
 
     /**
@@ -2443,6 +2482,8 @@ class ImapFlow extends EventEmitter {
      * @param {Boolean} query.uidValidity if `true` request mailbox `UIDVALIDITY` value
      * @param {Boolean} query.unseen if `true` request count of unseen messages
      * @param {Boolean} query.highestModseq if `true` request last known modseq value
+     * @param {Boolean} query.size if `true` request total mailbox size in octets (requires STATUS=SIZE or IMAP4rev2)
+     * @param {Boolean} query.deleted if `true` request count of messages with \\Deleted flag (requires IMAP4rev2)
      * @returns {Promise<StatusObject>} status of the indicated mailbox
      *
      * @example
@@ -2937,6 +2978,7 @@ class ImapFlow extends EventEmitter {
      * @property {MessageStructureObject} [bodyStructure] message body structure
      * @property {Date} [internalDate] message internal date
      * @property {Map<string, Buffer>} [bodyParts] a Map of message body parts where key is requested part identifier and value is a Buffer
+     * @property {Set<string>} [binaryParts] part identifiers from `bodyParts` that arrived via FETCH BINARY, i.e. with the content-transfer-encoding already decoded by the server
      * @property {Buffer} [headers] Requested header lines as Buffer
      */
 
@@ -3371,7 +3413,11 @@ class ImapFlow extends EventEmitter {
         //   4. Byte limiter (enforces maxBytes cap)
         // `stream` is the head of the pipeline (where raw chunks are written),
         // `output` is the tail (what the caller reads from).
-        switch (meta.encoding) {
+        // Parts that arrived via FETCH BINARY (response.binaryParts) are already
+        // decoded by the server - decoding again would corrupt the data, so stage 1
+        // is skipped for them.
+        let clientEncoding = response.binaryParts && response.binaryParts.has(part) ? false : meta.encoding;
+        switch (clientEncoding) {
             case 'base64':
                 output = stream = new libbase64.Decoder();
                 break;
@@ -3681,7 +3727,10 @@ class ImapFlow extends EventEmitter {
         for (let part of Object.keys(data)) {
             let meta = data[part].meta;
 
-            switch (meta.encoding) {
+            // parts that arrived via FETCH BINARY (response.binaryParts) are already
+            // decoded by the server - decoding again would corrupt the data
+            let clientEncoding = response.binaryParts && response.binaryParts.has(part) ? false : meta.encoding;
+            switch (clientEncoding) {
                 case 'base64':
                     data[part].content = data[part].content ? libbase64.decode(data[part].content.toString()) : null;
                     break;

package/lib/search-compiler.js

@@ -2,7 +2,7 @@
 
 'use strict';
 
-const { formatDate, formatFlag, canUseFlag, isDate } = require('./tools.js');
+const { formatDate, formatFlag, canUseFlag, isDate, isRev2Active } = require('./tools.js');
 
 /**
  * Sets a boolean flag in the IMAP search attributes.
@@ -186,10 +186,24 @@ module.exports.searchCompiler = (connection, query) => {
 
                 // Simple boolean flags without UN- support
                 case 'ALL':
+                    if (params[term]) {
+                        setBoolOpt(attributes, term, true);
+                    }
+                    break;
+
                 case 'NEW':
                 case 'OLD':
                 case 'RECENT':
                     if (params[term]) {
+                        // The \Recent flag and the NEW/OLD/RECENT search keys were
+                        // removed in IMAP4rev2 (RFC 9051) - a rev2 session would
+                        // reject the whole search with a tagged BAD, so fail with a
+                        // descriptive error instead
+                        if (isRev2Active(connection)) {
+                            let error = new Error(`The "${term.toLowerCase()}" search key does not exist in IMAP4rev2`);
+                            error.code = 'MissingServerExtension';
+                            throw error;
+                        }
                         setBoolOpt(attributes, term, true);
                     }
                     break;