imapflow 2.4.3 → 2.5.1

package/dist/cjs/streams-transport.js

@@ -0,0 +1,281 @@
+"use strict";
+// The `transport` option over a socket API built on web streams: the socket opens with a promise
+// that resolves to a ReadableStream and a WritableStream of bytes. `cloudflare:sockets` (celld,
+// Cloudflare Workers) is one, see sockets-transport.ts; Chrome's Direct Sockets and in-memory
+// connections fit the same shape. The streams are wrapped into a Duplex that behaves like the
+// net.Socket ImapFlow drives.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createStreamsTransport = createStreamsTransport;
+const node_stream_1 = require("node:stream");
+const tools_js_1 = require("./tools.js");
+function closeQuietly(socket) {
+    try {
+        Promise.resolve(socket.close()).catch(tools_js_1.noop);
+    }
+    catch {
+        // already closed
+    }
+}
+class StreamsDuplex extends node_stream_1.Duplex {
+    constructor(api, socket, onConnect, plain) {
+        super({ allowHalfOpen: false });
+        // Taken once the socket is open: locking the streams of a socket returned by startTls()
+        // before its handshake completes stalls the handshake on celld
+        this._reader = null;
+        this._writer = null;
+        this._pumping = false;
+        this._readRequested = false;
+        // A runtime may only start TLS on a socket without a read in flight (with one, celld never
+        // completes the handshake), while the read loop always keeps one. So once the client
+        // announced a STARTTLS (holdAfterStarttls), reading stops after the tagged line that answers
+        // it, the only command in flight then, and resumes only when that line is not an OK.
+        this._awaitingStarttls = false;
+        this._lineTail = '';
+        // Set when the partial line got too long to be the one looked for and was dropped: the rest
+        // of it, up to the next line break, is skipped too
+        this._tailDropped = false;
+        this._readsHeld = false;
+        // Set once startTls() handed the connection to a TLS socket: the read that ends then is not
+        // the end of the connection, and closing this socket would close the TLS one
+        this._upgraded = false;
+        this._timeout = 0;
+        this._timer = null;
+        // Traffic only records the time, the one timer re-arms itself for the rest of the idle period
+        this._lastActivity = 0;
+        this._api = api;
+        this._socket = socket;
+        this._plain = plain || null;
+        socket.opened.then(info => {
+            if (this.destroyed) {
+                return;
+            }
+            try {
+                this._reader = info.readable.getReader();
+                this._writer = info.writable.getWriter();
+            }
+            catch (err) {
+                // streams already locked by someone else
+                this.destroy(err);
+                return;
+            }
+            this.remoteAddress = info.remoteAddress;
+            this.remotePort = info.remotePort;
+            this.localAddress = info.localAddress;
+            this.localPort = info.localPort;
+            const tls = info.tls;
+            if (tls) {
+                this.authorized = tls.authorized;
+                this.getCipher = () => ({ name: tls.name || '', standardName: tls.standardName || tls.name || '', version: tls.version || '' });
+            }
+            this._touch();
+            onConnect();
+            if (this._readRequested) {
+                void this._pump();
+            }
+        }, (err) => this.destroy(err));
+        socket.closed?.catch((err) => {
+            if (!this._upgraded) {
+                this.destroy(err);
+            }
+        });
+    }
+    _read() {
+        if (this._readsHeld) {
+            return;
+        }
+        if (!this._reader) {
+            this._readRequested = true;
+        }
+        else if (!this._pumping) {
+            void this._pump();
+        }
+    }
+    async _pump() {
+        const reader = this._reader;
+        this._pumping = true;
+        try {
+            for (;;) {
+                const { done, value } = await reader.read();
+                if (this._upgraded || this.destroyed) {
+                    return;
+                }
+                if (done || !value) {
+                    this.push(null);
+                    return;
+                }
+                this._touch();
+                const chunk = Buffer.from(value.buffer, value.byteOffset, value.byteLength);
+                const pushed = this.push(chunk);
+                if (this._awaitingStarttls && this._starttlsCompleted(chunk)) {
+                    this._readsHeld = true;
+                    return;
+                }
+                if (!pushed) {
+                    return;
+                }
+            }
+        }
+        catch (err) {
+            if (!this._upgraded) {
+                this.destroy(err);
+            }
+        }
+        finally {
+            this._pumping = false;
+        }
+    }
+    // Whether the chunk holds the tagged response of the STARTTLS command and it is an OK. Lines
+    // end with LF, CR is optional, as for the parser
+    _starttlsCompleted(chunk) {
+        const skipFirst = this._tailDropped;
+        const lines = (this._lineTail + chunk.toString('latin1')).split('\n');
+        const tail = lines.pop();
+        if (skipFirst && !lines.length) {
+            // still inside the dropped line
+            return false;
+        }
+        this._tailDropped = tail.length > 1024;
+        this._lineTail = this._tailDropped ? '' : tail;
+        for (let i = skipFirst ? 1 : 0; i < lines.length; i++) {
+            const tagged = /^(?![*+] )\S+ (OK|NO|BAD)\b/i.exec(lines[i]);
+            if (tagged) {
+                this._awaitingStarttls = false;
+                return tagged[1].toUpperCase() === 'OK';
+            }
+        }
+        return false;
+    }
+    /** Stops reading after the tagged response of the STARTTLS command about to be sent, if it is an OK */
+    holdAfterStarttls() {
+        this._awaitingStarttls = true;
+        this._lineTail = '';
+        this._tailDropped = false;
+    }
+    _write(chunk, _encoding, callback) {
+        if (!this._writer) {
+            // ImapFlow writes only after onConnect, this is a misuse
+            callback(new Error('Socket is not open yet'));
+            return;
+        }
+        this._touch();
+        this._writer.write(chunk).then(() => callback(), callback);
+    }
+    _destroy(err, callback) {
+        this._clearTimer();
+        if (!this._upgraded) {
+            this._close();
+        }
+        if (this._plain) {
+            // the plain connection went with the TLS one, unless the API keeps it open
+            this._plain._close();
+            this._plain.destroy();
+        }
+        callback(err);
+    }
+    // Closes the connection. An API that refuses to close while its streams are locked (Direct
+    // Sockets) gets a second attempt with the locks released; releasing them right away would
+    // fail the read in flight, which ends the request on celld
+    _close() {
+        const retry = () => {
+            for (const stream of [this._reader, this._writer]) {
+                try {
+                    stream?.releaseLock();
+                }
+                catch {
+                    // an older streams implementation refuses with a read in flight
+                }
+            }
+            closeQuietly(this._socket);
+        };
+        try {
+            const closing = this._socket.close();
+            if (typeof closing?.then === 'function') {
+                closing.then(undefined, retry);
+            }
+        }
+        catch {
+            retry();
+        }
+    }
+    /** Like net.Socket#setTimeout(): emits 'timeout' after `ms` without traffic, 0 disables */
+    setTimeout(ms) {
+        this._timeout = ms;
+        this._clearTimer();
+        this._touch();
+        return this;
+    }
+    setKeepAlive() {
+        return this;
+    }
+    /** Hands the connection over to a TLS socket, this one only stays as its transport */
+    startTls(options, onSecure) {
+        this._upgraded = true;
+        this._clearTimer();
+        // workerd refuses to start TLS while the streams are locked, nothing is in flight on them
+        this._reader?.releaseLock();
+        this._writer?.releaseLock();
+        let tlsSocket;
+        try {
+            tlsSocket = this._api.upgrade(this._socket, options);
+        }
+        catch (err) {
+            // refused before any handshake, the connection is closed with this socket
+            this._upgraded = false;
+            throw err;
+        }
+        return new StreamsDuplex(this._api, tlsSocket, onSecure, this);
+    }
+    _touch() {
+        this._lastActivity = Date.now();
+        if (!this._timer) {
+            this._arm(this._timeout);
+        }
+    }
+    _arm(ms) {
+        if (this._timeout > 0 && !this.destroyed && !this._upgraded) {
+            this._timer = (0, tools_js_1.unrefTimer)(setTimeout(() => {
+                this._timer = null;
+                const idle = Date.now() - this._lastActivity;
+                if (idle >= this._timeout) {
+                    this.emit('timeout');
+                }
+                else {
+                    this._arm(this._timeout - idle);
+                }
+            }, ms));
+        }
+    }
+    _clearTimer() {
+        (0, tools_js_1.clearTimer)(this._timer);
+        this._timer = null;
+    }
+}
+/**
+ * Builds a transport for the `transport` option over a socket API built on web streams. A new
+ * API needs only `open()`, and `upgrade()` if it can start TLS on an open connection:
+ *
+ * ```js
+ * const transport = createStreamsTransport({
+ *     open: ({ host, port, secure }) => {
+ *         const socket = openSomeSocket(host, port, secure);
+ *         return { opened: socket.ready.then(() => ({ readable: socket.readable, writable: socket.writable })), close: () => socket.close() };
+ *     }
+ * });
+ * ```
+ *
+ * The transport stops reading once the server accepted a STARTTLS, so the API never has to start
+ * TLS with a read in flight, and releases the stream locks before `upgrade()` is called.
+ *
+ * @param api - Opens (and upgrades) connections
+ * @returns A transport for the `transport` option
+ */
+function createStreamsTransport(api) {
+    const transport = {
+        connect: (options, onConnect) => new StreamsDuplex(api, api.open(options), onConnect)
+    };
+    if (api.upgrade) {
+        transport.prepareUpgrade = socket => socket.holdAfterStarttls();
+        transport.upgrade = (socket, options, onSecure) => socket.startTls(options, onSecure);
+    }
+    return transport;
+}

package/dist/cjs/tools.d.ts

@@ -5,8 +5,18 @@ import type { ImapAttributeList, ImapAttributeNode, ImapResponse } from './handl
 import type { FetchMessageObject, ListResponse, ListTreeResponse, MailboxObject, MessageEnvelopeObject, MessageStructureObject, ImapFlowEvents, StatusQuery } from './types.js';
 export { AuthenticationFailure } from './errors.js';
 export declare const EXPANDED_RANGE_LIMIT = 16777216;
+export declare const MAX_COLLECTED_RESPONSES = 1000000;
 export declare const MAX_UINT32_DIGITS = 10;
 export declare const noop: () => void;
+/**
+ * Tells an IP literal from a hostname without `node:net`, which some runtimes do not implement.
+ * Only used to decide whether a host can be a TLS servername: a dotted quad, or anything with a
+ * colon (an IPv6 literal, a hostname never contains one).
+ *
+ * @param host - Hostname or IP address
+ * @returns true for an IP literal
+ */
+export declare function isIpLiteral(host: string): boolean;
 /**
  * A stream decoder returned by getDecoder(). The Japanese decoder reports the `limited`
  * flag once it has buffered all it will accept; a streaming iconv decoder never sets it.
@@ -462,6 +472,26 @@ export declare function parseBigIntValue(value: unknown, maxDigits?: number): bi
  * @returns The parsed value, or false when it is not usable.
  */
 export declare function parseUintValue(value: unknown, maxDigits?: number): number | false;
+/**
+ * Walks an IMAP sequence range string (e.g. "1:3,5,7:9") one number at a time, without
+ * materializing it. A single "1:16777216" is a few bytes on the wire but sixteen million
+ * entries expanded, so a consumer that only needs to visit each number (the VANISHED handler)
+ * iterates instead of calling expandRange().
+ *
+ * The same bound as expandRange() applies: at most EXPANDED_RANGE_LIMIT numbers in total are
+ * produced, over all entries of the set.
+ *
+ * @param range - IMAP sequence range string
+ * @returns The sequence numbers, in the order the set lists them
+ */
+export declare function iterateRange(range: unknown): Generator<number, void, undefined>;
+/**
+ * Counts the numbers iterateRange() produces for a sequence range string, without walking it.
+ *
+ * @param range - IMAP sequence range string
+ * @returns The number of entries, capped at EXPANDED_RANGE_LIMIT like the expansion itself
+ */
+export declare function countRange(range: unknown): number;
 /**
  * Expands an IMAP sequence range string (e.g. "1:3,5,7:9") into an array of numbers.
  *

package/dist/cjs/tools.js

@@ -4,7 +4,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.noop = exports.MAX_UINT32_DIGITS = exports.EXPANDED_RANGE_LIMIT = exports.AuthenticationFailure = void 0;
+exports.noop = exports.MAX_UINT32_DIGITS = exports.MAX_COLLECTED_RESPONSES = exports.EXPANDED_RANGE_LIMIT = exports.AuthenticationFailure = void 0;
+exports.isIpLiteral = isIpLiteral;
 exports.buildConnectionError = buildConnectionError;
 exports.restampConnectionError = restampConnectionError;
 exports.guardedPromise = guardedPromise;
@@ -52,6 +53,8 @@ exports.getStringList = getStringList;
 exports.getStringValue = getStringValue;
 exports.parseBigIntValue = parseBigIntValue;
 exports.parseUintValue = parseUintValue;
+exports.iterateRange = iterateRange;
+exports.countRange = countRange;
 exports.expandRange = expandRange;
 exports.getDecoder = getDecoder;
 exports.packMessageRange = packMessageRange;
@@ -76,6 +79,15 @@ const CONNECTION_GONE_CODES = new Set([errors_js_1.ImapFlowErrorCode.NoConnectio
 // expand server sequences (e.g. ESEARCH ALL in commands/search.ts) can apply the same absolute
 // ceiling instead of inventing their own.
 exports.EXPANDED_RANGE_LIMIT = 0x1000000;
+// Upper bound for the untagged responses one command may accumulate in memory: the FETCH rows
+// fetchAll() collects and the mailboxes list() collects. The response size limits bound one
+// response at a time, not how many of them a server sends in reply to one command, so a hostile
+// server answering a single-message FETCH with an endless stream of unsolicited FETCH rows (or a
+// LIST with endless mailboxes) would otherwise grow the process without limit. Generous on
+// purpose: a million rows is far beyond any legitimate listing, and a mailbox that large is read
+// through the streaming fetch() generator, which keeps nothing. Reaching it therefore fails the
+// command (code TooManyResponses) instead of silently truncating its result.
+exports.MAX_COLLECTED_RESPONSES = 1000000;
 // Digit bounds for untrusted numeric values in server responses. UIDs, UIDVALIDITY and
 // message counts are 32-bit unsigned (nz-number in the RFC 9051 grammar, so at most 10
 // digits); MODSEQ and number64 values are 63-bit unsigned (RFC 7162, RFC 9051), at most
@@ -122,6 +134,17 @@ const IMAP4REV2_FOLDED_CAPABILITIES = new Set([
 // Deliberate no-op, used as the observer a guarded promise attaches to its own rejection.
 const noop = () => { };
 exports.noop = noop;
+/**
+ * Tells an IP literal from a hostname without `node:net`, which some runtimes do not implement.
+ * Only used to decide whether a host can be a TLS servername: a dotted quad, or anything with a
+ * colon (an IPv6 literal, a hostname never contains one).
+ *
+ * @param host - Hostname or IP address
+ * @returns true for an IP literal
+ */
+function isIpLiteral(host) {
+    return /^\d{1,3}(\.\d{1,3}){3}$/.test(host) || host.includes(':');
+}
 // The fields buildConnectionError() stamps to say *where* a connection error was rejected, as
 // opposed to what went wrong. restampConnectionError() clears them, because they belong to the
 // site that built the error rather than to the failure it describes.
@@ -594,13 +617,30 @@ function getFolderTree(folders) {
         root: true,
         folders: []
     };
+    // Children of every node indexed by name. Looking a child up with folders.find() made the
+    // tree quadratic in the number of siblings: a listing of 50k mailboxes at one level (a size
+    // the server chooses) took seconds of event loop time. A name is unique among its siblings,
+    // as the loop below updates an existing entry rather than adding a second one.
+    let childIndex = new Map();
+    let childByName = (node, name) => {
+        let index = childIndex.get(node);
+        return index ? index.get(name) : undefined;
+    };
+    let indexChild = (node, child) => {
+        let index = childIndex.get(node);
+        if (!index) {
+            index = new Map();
+            childIndex.set(node, index);
+        }
+        index.set(child.name, child);
+    };
     let getTreeNode = (parents) => {
         let node = tree;
         if (!parents || !parents.length) {
             return node;
         }
         for (let parent of parents) {
-            let cur = node.folders && node.folders.find(folder => folder.name === parent);
+            let cur = childByName(node, parent);
             if (cur) {
                 node = cur;
             }
@@ -615,7 +655,7 @@ function getFolderTree(folders) {
     for (let folder of byDepth) {
         let parent = getTreeNode(folder.parent);
         // see if entry already exists
-        let existing = parent.folders && parent.folders.find(existing => existing.name === folder.name);
+        let existing = childByName(parent, folder.name);
         if (existing) {
             // update values
             existing.name = folder.name;
@@ -660,6 +700,7 @@ function getFolderTree(folders) {
                 parent.folders = [];
             }
             parent.folders.push(data);
+            indexChild(parent, data);
         }
     }
     return tree;
@@ -1554,67 +1595,103 @@ function parseUintValue(value, maxDigits) {
     return Number.isSafeInteger(num) ? num : false;
 }
 /**
- * Expands an IMAP sequence range string (e.g. "1:3,5,7:9") into an array of numbers.
- *
- * Entries with endpoints that are not valid nz-numbers are skipped - the input
- * may come from an untrusted server, and 'Infinity' or similar garbage would
- * otherwise loop without bound. The whole set is expanded to at most
- * EXPANDED_RANGE_LIMIT entries in total: legitimate responses never reach the limit
- * (the mailbox would need that many messages), while hostile input is cut off
- * instead of exhausting memory. The total is capped, not just each range -
- * otherwise "1:16777216,1:16777216,..." would multiply the per-range bound by an
- * unbounded number of ranges.
+ * Parses an IMAP sequence set string (e.g. "1:3,5,9:7") into its entries, each as the pair of
+ * endpoints in the order written: a reversed "9:7" stays reversed, and iterateRange() walks it
+ * downwards. Entries with an endpoint that is not a valid nz-number are skipped - the input may
+ * come from an untrusted server, and 'Infinity' or similar garbage would otherwise loop without
+ * bound.
  *
- * @param range - IMAP sequence range string
- * @returns Array of expanded sequence numbers
+ * @param range - IMAP sequence set string, or whatever the response parser produced for it
+ * @returns The validated entries
  */
-function expandRange(range) {
-    let result = [];
+function parseRangeEntries(range) {
+    let entries = [];
     // Callers pass whatever the response parser produced for the sequence set, and a
     // malformed response can leave that as `false` (e.g. a VANISHED response carrying
     // only the (EARLIER) tag). Nothing to expand then, and throwing here would abort
     // the handler for the rest of the response.
     if (typeof range !== 'string') {
-        return result;
+        return entries;
     }
     for (let entry of range.split(',')) {
-        if (result.length >= exports.EXPANDED_RANGE_LIMIT) {
-            break;
-        }
         entry = entry.trim();
         let colon = entry.indexOf(':');
         if (colon < 0) {
             let value = Number(entry);
             if (isValidSequenceValue(value)) {
-                result.push(value);
+                entries.push([value, value]);
             }
             continue;
         }
         let first = Number(entry.substring(0, colon));
         let second = Number(entry.slice(colon + 1));
-        if (!isValidSequenceValue(first) || !isValidSequenceValue(second)) {
-            continue;
-        }
-        if (first === second) {
-            result.push(first);
-            continue;
+        if (isValidSequenceValue(first) && isValidSequenceValue(second)) {
+            entries.push([first, second]);
         }
-        // Remaining total budget doubles as the per-range bound
-        let remaining = exports.EXPANDED_RANGE_LIMIT - result.length;
-        if (first < second) {
-            let last = Math.min(second, first + remaining - 1);
-            for (let i = first; i <= last; i++) {
-                result.push(i);
+    }
+    return entries;
+}
+/**
+ * Walks an IMAP sequence range string (e.g. "1:3,5,7:9") one number at a time, without
+ * materializing it. A single "1:16777216" is a few bytes on the wire but sixteen million
+ * entries expanded, so a consumer that only needs to visit each number (the VANISHED handler)
+ * iterates instead of calling expandRange().
+ *
+ * The same bound as expandRange() applies: at most EXPANDED_RANGE_LIMIT numbers in total are
+ * produced, over all entries of the set.
+ *
+ * @param range - IMAP sequence range string
+ * @returns The sequence numbers, in the order the set lists them
+ */
+function* iterateRange(range) {
+    let count = 0;
+    for (let [first, second] of parseRangeEntries(range)) {
+        let step = first <= second ? 1 : -1;
+        for (let i = first; count < exports.EXPANDED_RANGE_LIMIT; i += step) {
+            yield i;
+            count++;
+            if (i === second) {
+                break;
             }
         }
-        else {
-            let last = Math.max(second, first - remaining + 1);
-            for (let i = first; i >= last; i--) {
-                result.push(i);
-            }
+        if (count >= exports.EXPANDED_RANGE_LIMIT) {
+            break;
         }
     }
-    return result;
+}
+/**
+ * Counts the numbers iterateRange() produces for a sequence range string, without walking it.
+ *
+ * @param range - IMAP sequence range string
+ * @returns The number of entries, capped at EXPANDED_RANGE_LIMIT like the expansion itself
+ */
+function countRange(range) {
+    let count = 0;
+    for (let [first, second] of parseRangeEntries(range)) {
+        count += Math.abs(second - first) + 1;
+        if (count >= exports.EXPANDED_RANGE_LIMIT) {
+            return exports.EXPANDED_RANGE_LIMIT;
+        }
+    }
+    return count;
+}
+/**
+ * Expands an IMAP sequence range string (e.g. "1:3,5,7:9") into an array of numbers.
+ *
+ * Entries with endpoints that are not valid nz-numbers are skipped - the input
+ * may come from an untrusted server, and 'Infinity' or similar garbage would
+ * otherwise loop without bound. The whole set is expanded to at most
+ * EXPANDED_RANGE_LIMIT entries in total: legitimate responses never reach the limit
+ * (the mailbox would need that many messages), while hostile input is cut off
+ * instead of exhausting memory. The total is capped, not just each range -
+ * otherwise "1:16777216,1:16777216,..." would multiply the per-range bound by an
+ * unbounded number of ranges.
+ *
+ * @param range - IMAP sequence range string
+ * @returns Array of expanded sequence numbers
+ */
+function expandRange(range) {
+    return Array.from(iterateRange(range));
 }
 /**
  * Returns a stream decoder for the given charset. Uses a special Japanese
@@ -1656,20 +1733,21 @@ function packMessageRange(list) {
     // Deduplicate before sorting so that repeated values do not produce
     // overlapping/non-canonical tokens (e.g. [1,1,2,3] -> "1:3", not "1,1:3").
     items = Array.from(new Set(items)).sort((a, b) => a - b);
-    let last = items[items.length - 1];
-    let result = [[last]];
-    for (let i = items.length - 2; i >= 0; i--) {
-        if (items[i] === items[i + 1] - 1) {
-            result[0].unshift(items[i]);
+    // One forward pass: a run of consecutive values is written out as soon as it ends. The runs
+    // used to be built with unshift(), which copies the array on every call, so packing a large
+    // SEARCH result (a server-chosen size) took seconds for 200k UIDs and about a minute for 1M,
+    // all of it on the event loop.
+    let parts = [];
+    let start = items[0];
+    let prev = start;
+    for (let i = 1; i <= items.length; i++) {
+        let value = items[i];
+        if (i < items.length && value === prev + 1) {
+            prev = value;
             continue;
         }
-        result.unshift([items[i]]);
+        parts.push(start === prev ? String(start) : start + ':' + prev);
+        start = prev = value;
     }
-    let parts = result.map(item => {
-        if (item.length === 1) {
-            return item[0];
-        }
-        return item.shift() + ':' + item.pop();
-    });
     return parts.join(',');
 }