imapflow 2.5.8 → 2.6.0

package/dist/esm/download.js

@@ -229,9 +229,9 @@ export async function downloadMessage(client, range, part, options) {
         };
         let response = await fetchExpected(client, range, query, downloadOptions, expected);
         if (!response) {
-            // An empty answer also comes from a connection that closed while the consumer was
-            // paused between chunks (fetchOne() finds no mailbox then), and that is not a
-            // missing message: a caller that retries would skip it
+            // An empty answer from a connection that closed while the consumer was paused
+            // between chunks is not a missing message: a caller that retries would skip it
+            // (fetchOne() itself rejects with NoConnection when it finds no mailbox then)
             if (!client.usable) {
                 throw client.createNoConnectionError(false, { rejectedFrom: 'download' });
             }

package/dist/esm/errors.d.ts

@@ -35,6 +35,7 @@ export declare const ImapFlowErrorCode: {
     readonly LineTooLarge: "LineTooLarge";
     readonly LiteralTooLarge: "LiteralTooLarge";
     readonly ResponseTooLarge: "ResponseTooLarge";
+    readonly TooManyTokens: "TooManyTokens";
     readonly TooManyResponses: "TooManyResponses";
     readonly InvalidStringValue: "InvalidStringValue";
     readonly InvalidTokenValue: "InvalidTokenValue";
@@ -50,6 +51,7 @@ export declare const ImapFlowErrorCode: {
     readonly ERR_INVALID_URL: "ERR_INVALID_URL";
     readonly UnsupportedTransportOption: "UnsupportedTransportOption";
     readonly InstanceReused: "InstanceReused";
+    readonly NoMailboxSelected: "NoMailboxSelected";
 };
 /** One of the {@link ImapFlowErrorCode} values */
 export type ImapFlowErrorCode = (typeof ImapFlowErrorCode)[keyof typeof ImapFlowErrorCode];
@@ -117,7 +119,7 @@ export interface ImapFlowError extends Error {
     lineLength?: number | undefined;
     /** The size of a rejected response */
     responseSize?: number | undefined;
-    /** The bound that was exceeded */
+    /** The bound that was exceeded (bytes, or tokens for TooManyTokens) */
     maxSize?: number | undefined;
     /** OAuth error details from the server, for XOAUTH2 authentication failures */
     oauthError?: any;

package/dist/esm/errors.js

@@ -39,6 +39,7 @@ export const ImapFlowErrorCode = {
     LineTooLarge: 'LineTooLarge',
     LiteralTooLarge: 'LiteralTooLarge',
     ResponseTooLarge: 'ResponseTooLarge',
+    TooManyTokens: 'TooManyTokens',
     // one command received more untagged responses than are kept, see MAX_COLLECTED_RESPONSES
     TooManyResponses: 'TooManyResponses',
     // invalid values in a command
@@ -59,7 +60,9 @@ export const ImapFlowErrorCode = {
     // the transport option
     UnsupportedTransportOption: 'UnsupportedTransportOption',
     // API misuse
-    InstanceReused: 'InstanceReused'
+    InstanceReused: 'InstanceReused',
+    // a message command (FETCH) was called while no mailbox was selected
+    NoMailboxSelected: 'NoMailboxSelected'
 };
 /**
  * Error subclass thrown when IMAP authentication fails.

package/dist/esm/handler/imap-parser.js

@@ -3,8 +3,8 @@ import { ParserInstance } from './parser-instance.js';
 import { createImapError } from '../errors.js';
 import { boundedInput } from './limits.js';
 // The codes the parser raises on purpose: the numbered ParserErrorN codes, ParserErrorExchange,
-// the nesting limit and the inline literal size limit
-const PARSER_ERROR_CODES = /^(?:ParserError\d+|ParserErrorExchange|MAX_IMAP_NESTING_REACHED|LiteralTooLarge)$/;
+// the nesting limit, the inline literal size limit and the token limit
+const PARSER_ERROR_CODES = /^(?:ParserError\d+|ParserErrorExchange|MAX_IMAP_NESTING_REACHED|LiteralTooLarge|TooManyTokens)$/;
 /**
  * Anything the parser throws other than one of its own coded errors - a TypeError from a parser
  * bug, a RangeError from runaway recursion or a bad Buffer offset (which carries a Node code), a
@@ -75,6 +75,39 @@ function readSearchNumbers(str) {
     numbers.push(value);
     return numbers;
 }
+// The CONDSTORE list that ends a SEARCH answer to a MODSEQ query (RFC 7162 section 3.1.5)
+const RE_SEARCH_MODSEQ = /^ \(MODSEQ (\d{1,19})\) *$/i;
+/**
+ * Reads an untagged SEARCH response without the tokenizer, see readSearchNumbers(): the numbers,
+ * and the MODSEQ value of a trailing CONDSTORE list. A MODSEQ query on a large mailbox matches as
+ * many messages as any other, so its answer must not fall back to the tokenizer either.
+ *
+ * @param str - The rest of the line after the command
+ * @returns The numbers, with the MODSEQ list as the attributes the tokenizer would have made of
+ *   it, or null when the line is left to the tokenizer
+ */
+function readSearchResponse(str) {
+    // the last character tells whether there is a list to look for, without scanning the line
+    if (!str.trimEnd().endsWith(')')) {
+        const numbers = readSearchNumbers(str);
+        return numbers && { numbers };
+    }
+    const open = str.lastIndexOf(' (');
+    const match = open < 0 ? null : RE_SEARCH_MODSEQ.exec(str.slice(open));
+    if (!match) {
+        return null;
+    }
+    const numbers = readSearchNumbers(str.slice(0, open));
+    return (numbers && {
+        numbers,
+        attributes: [
+            [
+                { type: 'ATOM', value: 'MODSEQ' },
+                { type: 'ATOM', value: match[1] }
+            ]
+        ]
+    });
+}
 /**
  * Parses a raw IMAP command or response buffer into a structured object.
  * Handles edge cases such as null-byte-padded responses from buggy servers and
@@ -130,10 +163,9 @@ export default async function parser(command, options) {
             response.command += ' ' + (await parserInstance.getElement(imapFormalSyntax.command()));
         }
         if (response.tag === '*' && (response.command || '').toUpperCase() === 'SEARCH') {
-            const numbers = readSearchNumbers(parserInstance.remainder);
-            if (numbers) {
-                response.numbers = numbers;
-                return response;
+            const search = readSearchResponse(parserInstance.remainder);
+            if (search) {
+                return Object.assign(response, search);
             }
         }
         if (parserInstance.remainder.trim().length) {

package/dist/esm/handler/limits.d.ts

@@ -2,6 +2,7 @@ import { type ImapFlowError } from '../errors.js';
 export declare const MAX_LITERAL_SIZE: number;
 export declare const MAX_LINE_SIZE: number;
 export declare const MAX_RESPONSE_SIZE: number;
+export declare const MAX_RESPONSE_TOKENS = 1000000;
 export declare const ERROR_CONTEXT_LENGTH = 1024;
 /**
  * The input a parse error carries: a bounded prefix with the full length.

package/dist/esm/handler/limits.js

@@ -18,6 +18,13 @@ export const MAX_LINE_SIZE = MAX_LITERAL_SIZE;
 // of exactly the maximum permitted size impossible to receive. Configuring both limits calls for
 // the same headroom - set maxResponseSize above maxLiteralSize, not equal to it.
 export const MAX_RESPONSE_SIZE = 2 * MAX_LITERAL_SIZE;
+// Default maximum number of tokens (atoms, strings, literals, lists and the other parse tree
+// nodes) in a single response. The byte limits bound what a server sends, not what parsing it
+// costs: every token becomes a tree node and an attribute object, around a hundred bytes of heap
+// for a two-byte "1 ", so a line of many small tokens costs fifty times its size. A legitimate
+// response stays far below this (the long one, a SEARCH answer of plain numbers, is read without
+// the tokenizer, see parser()), so the bound only stops a server that floods one response.
+export const MAX_RESPONSE_TOKENS = 1000000;
 // How much of the offending input a parse error carries along. The error travels whole into log
 // entries (pino copies every property of a logged error) and into the rejection of the command
 // the line belonged to, and the line can be as long as the configured line cap.

package/dist/esm/handler/token-parser.d.ts

@@ -7,7 +7,8 @@ import type { ParserInstance } from './parser-instance.js';
  * (ATOM, string, LITERAL, SEQUENCE, LIST, SECTION, PARTIAL).
  */
 export interface TokenNode {
-    childNodes: TokenNode[];
+    /** Created with the first child, a leaf token does not carry an empty array */
+    childNodes?: TokenNode[] | undefined;
     type: string | false;
     value: string | Buffer;
     isClosed: boolean;
@@ -44,6 +45,9 @@ export declare class TokenParser {
     options: ParserOptions;
     parent: TokenParserParent | ParserInstance;
     maxLiteralSize: number;
+    maxResponseTokens: number;
+    /** Nodes created so far, checked against maxResponseTokens */
+    tokenCount: number;
     tree: TokenNode;
     currentNode: TokenNode;
     pos: number;
@@ -62,6 +66,7 @@ export declare class TokenParser {
      * @param options.literals - Pre-parsed literal values from the input stream.
      * @param options.maxLiteralSize - Maximum size (in bytes) of a literal parsed inline
      *   from the input, i.e. when no pre-parsed literal buffers were supplied. Defaults to 1GB.
+     * @param options.maxResponseTokens - Maximum number of parse tree nodes, see MAX_RESPONSE_TOKENS.
      */
     constructor(parent: TokenParserParent | ParserInstance, startPos?: number | undefined, str?: string | null | undefined, options?: ParserOptions | undefined);
     /**
@@ -93,7 +98,8 @@ export declare class TokenParser {
      * @param parentNode - The parent node to attach this node to. If omitted, creates a root node.
      * @param startPos - The starting position of this node in the original input string.
      * @returns The newly created node with childNodes, type, value, and isClosed properties.
-     * @throws {Error} If the nesting depth exceeds MAX_NODE_DEPTH outside a FETCH data item.
+     * @throws {Error} If the nesting depth exceeds MAX_NODE_DEPTH outside a FETCH data item, or
+     *   the response has more than maxResponseTokens nodes.
      */
     createNode(parentNode?: TokenNode | undefined, startPos?: number | undefined): TokenNode;
     /**

package/dist/esm/handler/token-parser.js

@@ -1,6 +1,6 @@
 /* eslint new-cap: 0 */
 import imapFormalSyntax from './imap-formal-syntax.js';
-import { MAX_LITERAL_SIZE, ERROR_CONTEXT_LENGTH, boundedInput, normalizeLimit, createLiteralTooLargeError } from './limits.js';
+import { MAX_LITERAL_SIZE, MAX_RESPONSE_TOKENS, ERROR_CONTEXT_LENGTH, boundedInput, normalizeLimit, createLiteralTooLargeError } from './limits.js';
 import { createImapError } from '../errors.js';
 const STATE_ATOM = 0x001;
 const STATE_LITERAL = 0x002;
@@ -39,6 +39,7 @@ export class TokenParser {
      * @param options.literals - Pre-parsed literal values from the input stream.
      * @param options.maxLiteralSize - Maximum size (in bytes) of a literal parsed inline
      *   from the input, i.e. when no pre-parsed literal buffers were supplied. Defaults to 1GB.
+     * @param options.maxResponseTokens - Maximum number of parse tree nodes, see MAX_RESPONSE_TOKENS.
      */
     constructor(parent, startPos, str, options) {
         this.str = (str || '').toString();
@@ -47,6 +48,8 @@ export class TokenParser {
         // Same normalization and default as the streaming parser, so a direct user of this parser
         // gets the same bound (an explicit 0 means "reject any non-empty inline literal").
         this.maxLiteralSize = normalizeLimit(this.options.maxLiteralSize, MAX_LITERAL_SIZE);
+        this.maxResponseTokens = normalizeLimit(this.options.maxResponseTokens, MAX_RESPONSE_TOKENS);
+        this.tokenCount = 0;
         this.truncatedAttributes = [];
         this.tree = this.currentNode = this.createNode();
         this.pos = startPos || 0;
@@ -83,7 +86,9 @@ export class TokenParser {
         await this.processString();
         const attributes = [];
         let branch = attributes;
-        let walk = async (node) => {
+        // Synchronous on purpose: the recursion is bounded by MAX_NODE_DEPTH, and an async walk
+        // allocated a promise per token, which a line of many small tokens multiplies
+        let walk = (node) => {
             let curBranch = branch;
             let elm;
             let partial;
@@ -140,12 +145,16 @@ export class TokenParser {
                     branch[branch.length - 1].partial = partial;
                     break;
             }
-            for (let childNode of node.childNodes) {
-                await walk(childNode);
+            // Each child is released once it is converted, so the tree and the attributes built
+            // from it are not both held whole at the peak of a line of many tokens
+            let childNodes = node.childNodes;
+            for (let i = 0; childNodes && i < childNodes.length; i++) {
+                walk(childNodes[i]);
+                childNodes[i] = null;
             }
             branch = curBranch;
         };
-        await walk(this.tree);
+        walk(this.tree);
         return attributes;
     }
     /**
@@ -156,11 +165,20 @@ export class TokenParser {
      * @param parentNode - The parent node to attach this node to. If omitted, creates a root node.
      * @param startPos - The starting position of this node in the original input string.
      * @returns The newly created node with childNodes, type, value, and isClosed properties.
-     * @throws {Error} If the nesting depth exceeds MAX_NODE_DEPTH outside a FETCH data item.
+     * @throws {Error} If the nesting depth exceeds MAX_NODE_DEPTH outside a FETCH data item, or
+     *   the response has more than maxResponseTokens nodes.
      */
     createNode(parentNode, startPos) {
+        // The root node is not a token of the response
+        if (parentNode && ++this.tokenCount > this.maxResponseTokens) {
+            throw createImapError(`Response has more than ${this.maxResponseTokens} tokens`, 'TooManyTokens', {
+                maxSize: this.maxResponseTokens,
+                parserContext: { ...boundedInput(this.str), pos: startPos }
+            });
+        }
         let node = {
-            childNodes: [],
+            // declared up front, so a node keeps one shape when its first child arrives
+            childNodes: undefined,
             type: false,
             value: '',
             isClosed: true,
@@ -193,7 +211,7 @@ export class TokenParser {
             node.startPos = startPos;
         }
         if (parentNode) {
-            parentNode.childNodes.push(node);
+            (parentNode.childNodes ||= []).push(node);
         }
         return node;
     }
@@ -205,7 +223,7 @@ export class TokenParser {
      */
     isFetchResponse() {
         let command = this.parent.command;
-        let first = this.tree.childNodes[0];
+        let first = this.tree.childNodes?.[0];
         return (typeof command === 'string' &&
             /^\d+$/.test(command) &&
             !!first &&
@@ -231,7 +249,7 @@ export class TokenParser {
         // The item is the last child of the list while it is being parsed, so its key is the
         // child before it. A key is an atom (BODYSTRUCTURE, ENVELOPE, BODY); anything else means
         // the row itself is malformed, and the key is reported as unknown.
-        let siblings = item.parentNode ? item.parentNode.childNodes : [];
+        let siblings = item.parentNode?.childNodes || [];
         let key = siblings[siblings.length - 2];
         this.truncatedAttributes.push(key && key.type === 'ATOM' && typeof key.value === 'string' ? key.value.toUpperCase() : '?');
     }
@@ -259,13 +277,26 @@ export class TokenParser {
         // character, which is quadratic in the token length, and the server decides that length:
         // a 600 KB ESEARCH result took seconds of event loop time
         let tokenLength = 0;
+        // Where the token starts in the input, and whether its value still is that run of the
+        // input. A token is read as a run of characters, so its value is sliced from the input
+        // instead of grown with +=: each += of a long token adds a cons string of some twenty
+        // bytes per character, which the server multiplies with a long token
+        let tokenStart = 0;
+        let tokenSliced = true;
+        // Where the quoted string being read starts, and whether it holds an escape
+        let stringStart = 0;
+        let stringEscaped = false;
         let tokenLast = '';
         let tokenPrev = '';
         let tokenDigitsOnly = true;
         let tokenHasStar = false;
         let tokenHasDot = false;
         const appendToToken = (c) => {
-            this.currentNode.value += c;
+            if (!tokenLength) {
+                tokenStart = i;
+            }
+            tokenSliced = tokenSliced && i === tokenStart + tokenLength && this.str.charAt(i) === c;
+            this.currentNode.value = tokenSliced ? this.str.slice(tokenStart, i + 1) : this.currentNode.value + c;
             tokenLength++;
             tokenPrev = tokenLast;
             tokenLast = c;
@@ -289,6 +320,7 @@ export class TokenParser {
         const startToken = (first) => {
             this.currentNode.value = '';
             tokenLength = 0;
+            tokenSliced = true;
             tokenLast = tokenPrev = '';
             tokenDigitsOnly = true;
             tokenHasStar = tokenHasDot = false;
@@ -299,11 +331,14 @@ export class TokenParser {
         // ImapStream supplies one buffer per literal marker it framed, {0} included. A marker
         // without a buffer means the line and its literals do not match up, so fail with a parser
         // error instead of reading a missing buffer or shifting every later literal by one
+        // Read by index: shift() moves every remaining buffer, which made a response of many
+        // literals quadratic to parse
+        let literalIndex = 0;
         const takeLiteral = (literals) => {
-            if (!literals.length) {
+            if (literalIndex >= literals.length) {
                 throw this.parserError(`Literal without data at position ${this.pos + i} [E35]`, 'ParserError35', this.pos + i, chr);
             }
-            return literals.shift();
+            return literals[literalIndex++];
         };
         // Any ATOM supported char starts a new Atom sequence, otherwise throw an error
         // Allow \ as the first char for atom to support system flags
@@ -330,6 +365,8 @@ export class TokenParser {
                             this.currentNode = this.createNode(this.currentNode, this.pos + i);
                             this.currentNode.type = 'string';
                             this.state = STATE_STRING;
+                            stringStart = i + 1;
+                            stringEscaped = false;
                             this.currentNode.isClosed = false;
                             break;
                         // ( starts a new list
@@ -560,8 +597,11 @@ export class TokenParser {
                     appendToToken(chr);
                     break;
                 case STATE_STRING:
-                    // DQUOTE ends the string sequence
+                    // DQUOTE ends the string sequence. Its value is taken from the input in one
+                    // piece, for the same reason tokens are sliced, see appendToToken()
                     if (chr === '"') {
+                        let value = this.str.slice(stringStart, i);
+                        this.currentNode.value = stringEscaped ? value.replace(/\\([\s\S])/g, '$1') : value;
                         this.currentNode.endPos = this.pos + i;
                         this.currentNode.isClosed = true;
                         this.currentNode = this.currentNode.parentNode;
@@ -569,15 +609,14 @@ export class TokenParser {
                         checkSP();
                         break;
                     }
-                    // \ Escapes the following char
+                    // \ Escapes the following char, which is skipped here so it can not end the string
                     if (chr === '\\') {
                         i++;
                         if (i >= len) {
                             throw this.parserError(`Unexpected end of input at position ${this.pos + i} [E18]`, 'ParserError18', this.pos + i);
                         }
-                        chr = this.str.charAt(i);
+                        stringEscaped = true;
                     }
-                    this.currentNode.value += chr;
                     break;
                 case STATE_PARTIAL:
                     if (chr === '>') {

package/dist/esm/handler/types.d.ts

@@ -38,8 +38,9 @@ export interface ImapResponse {
     /** Parsed attributes of the response */
     attributes?: ImapAttributeList | undefined;
     /**
-     * The numbers of an untagged SEARCH response (`* SEARCH 2 5 9`), set instead of `attributes`
-     * when the line holds nothing else, see parser()
+     * The numbers of an untagged SEARCH response (`* SEARCH 2 5 9`), set when the line holds
+     * nothing else but a CONDSTORE `(MODSEQ n)` list, which is then the only entry of
+     * `attributes`, see parser()
      */
     numbers?: number[] | undefined;
     /** Number of leading NUL bytes stripped from the line, if any */
@@ -79,6 +80,8 @@ export interface ParserOptions {
     literals?: Buffer[] | undefined;
     /** Maximum size in bytes of a literal parsed inline from the input */
     maxLiteralSize?: number | undefined;
+    /** Maximum number of tokens in the response, see MAX_RESPONSE_TOKENS */
+    maxResponseTokens?: number | undefined;
 }
 /**
  * One assembled response as pushed by ImapStream: the line payload, its literals and the

package/dist/esm/imap-flow.d.ts

@@ -134,6 +134,23 @@ export declare class ImapFlow extends EventEmitter {
      * Is current mailbox idling (`true`) or not (`false`)
      */
     idling: boolean;
+    /**
+     * Data of the application's own, kept with the client and never read by ImapFlow. Starts as
+     * the `context` option; cast it to the type the application stores there
+     */
+    context: unknown;
+    /**
+     * When the TCP (or TLS) connection to the server was established, in milliseconds since the
+     * epoch, `null` before that. Kept after the connection closed
+     */
+    connectedAt: number | null;
+    /**
+     * When the client last wrote to the server (a command, a literal, the DONE that ends IDLE, an
+     * authentication response), in milliseconds since the epoch, `null` before the first write.
+     * Servers drop sessions that have not sent anything for a while, the IDLE ones included
+     * (RFC 9051 allows 30 minutes), and this is what their timer measures
+     */
+    lastClientWrite: number | null;
     /** Whether log entries are also emitted as 'log' events, see the `emitLogs` option */
     emitLogs: boolean;
     /** Hash algorithm for the fallback message id, see `ImapFlowOptions.idHashAlgorithm` */
@@ -148,6 +165,16 @@ export declare class ImapFlow extends EventEmitter {
     byeReason: string | undefined;
     /** Negotiated TLS session details, `false` for a cleartext connection */
     tls: TlsInfo | false | undefined;
+    /**
+     * The mailboxes a `verifyOnly` connection with `includeMailboxes` listed before it logged out,
+     * `undefined` otherwise
+     *
+     * @example
+     * let client = new ImapFlow({ ..., verifyOnly: true, includeMailboxes: true });
+     * await client.connect();
+     * console.log(client.mailboxList.map(mailbox => mailbox.path));
+     */
+    get mailboxList(): ListResponse[] | undefined;
     /**
      * Creates a client for one IMAP connection. Nothing is sent before `connect()` is called
      */
@@ -344,6 +371,9 @@ export declare class ImapFlow extends EventEmitter {
      * otherwise IDLE is started by default on connection inactivity. NB! If `idle()` is called manually then it does not
      * return until IDLE is finished which means you would have to call some other command out of scope.
      *
+     * Unlike auto-IDLE, a direct call does not wait for the connection to be free: a mailbox lock
+     * holder may idle inside its own lock, and idle() can not tell it from another caller.
+     *
      * @returns `false` if IDLE failed, `undefined` otherwise
      *
      * @example
@@ -505,6 +535,9 @@ export declare class ImapFlow extends EventEmitter {
      * @param query Fetch query
      * @param options Fetch options
      * @yields Message data object
+     * @throws An error with the code `NoMailboxSelected` when no mailbox is selected (for example
+     *   after `mailboxClose()` or a failed `mailboxOpen()`), or `NoConnection` once the connection
+     *   is gone. Earlier releases yielded nothing in that case, as for an empty mailbox
      *
      * @example
      * let mailbox = await client.mailboxOpen('INBOX');
@@ -526,6 +559,7 @@ export declare class ImapFlow extends EventEmitter {
      * @param query Fetch query
      * @param options Fetch options
      * @returns Array of Message data object
+     * @throws An error with the code `NoMailboxSelected` when no mailbox is selected, see fetch()
      *
      * @example
      * let mailbox = await client.mailboxOpen('INBOX');
@@ -542,7 +576,9 @@ export declare class ImapFlow extends EventEmitter {
      * @param seq Single UID or sequence number of the message to fetch for
      * @param query Fetch query
      * @param options Fetch options
-     * @returns Message data object
+     * @returns Message data object, `false` when the message was not found
+     * @throws An error with the code `NoMailboxSelected` when no mailbox is selected, see fetch().
+     *   Earlier releases resolved `undefined` then
      *
      * @example
      * let mailbox = await client.mailboxOpen('INBOX');
@@ -550,7 +586,7 @@ export declare class ImapFlow extends EventEmitter {
      * let lastMsg = await client.fetchOne('*', {uid: true})
      * console.log(lastMsg.uid);
      */
-    fetchOne(seq: SequenceString, query: FetchQueryObject, options?: FetchOptions | undefined): Promise<FetchMessageObject | false | undefined>;
+    fetchOne(seq: SequenceString, query: FetchQueryObject, options?: FetchOptions | undefined): Promise<FetchMessageObject | false>;
     /**
      * Download either full rfc822 formatted message or a specific bodystructure part as a Stream.
      * Bodystructure parts are decoded so the resulting stream is a binary file. Text content