imapflow 1.0.144 → 1.0.146

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## [1.0.146](https://github.com/postalsys/imapflow/compare/v1.0.145...v1.0.146) (2023-10-26)
+
+
+### Bug Fixes
+
+* **throttling:** automatically retry throttled FETCH commands a few times ([07a9aea](https://github.com/postalsys/imapflow/commit/07a9aea37ae55ed885ccc4c0e3732b367ad82cd3))
+
+## [1.0.145](https://github.com/postalsys/imapflow/compare/v1.0.144...v1.0.145) (2023-10-26)
+
+
+### Bug Fixes
+
+* **docs:** Fixed mailbox property name in AppendResponseObject ([ca6d789](https://github.com/postalsys/imapflow/commit/ca6d789b761d3117f733ed7b026122098f4d9142))
+* **special-use:** support custom special use flags for the Archive folder ([17aa6a8](https://github.com/postalsys/imapflow/commit/17aa6a8baf180a7349fe30f897184eb8e359a498))
+
 ## [1.0.144](https://github.com/postalsys/imapflow/compare/v1.0.143...v1.0.144) (2023-09-13)
 
 

package/lib/commands/fetch.js

@@ -13,180 +13,197 @@ module.exports = async (connection, range, query, options) => {
 
     let mailbox = connection.mailbox;
 
-    let messages = {
-        count: 0,
-        list: []
-    };
-
     const commandKey = connection.capabilities.has('BINARY') && options.binary && !connection.disableBinary ? 'BINARY' : 'BODY';
 
-    let response;
-    try {
-        let attributes = [{ type: 'SEQUENCE', value: (range || '*').toString() }];
+    let retryCount = 0;
+    while (retryCount < 4) {
+        let messages = {
+            count: 0,
+            list: []
+        };
 
-        let queryStructure = [];
+        let response;
+        try {
+            let attributes = [{ type: 'SEQUENCE', value: (range || '*').toString() }];
 
-        let setBodyPeek = (attributes, partial) => {
-            let bodyPeek = {
-                type: 'ATOM',
-                value: `${commandKey}.PEEK`,
-                section: [],
-                partial
-            };
+            let queryStructure = [];
 
-            if (Array.isArray(attributes)) {
-                attributes.forEach(attribute => {
-                    bodyPeek.section.push(attribute);
-                });
-            } else if (attributes) {
-                bodyPeek.section.push(attributes);
-            }
+            let setBodyPeek = (attributes, partial) => {
+                let bodyPeek = {
+                    type: 'ATOM',
+                    value: `${commandKey}.PEEK`,
+                    section: [],
+                    partial
+                };
+
+                if (Array.isArray(attributes)) {
+                    attributes.forEach(attribute => {
+                        bodyPeek.section.push(attribute);
+                    });
+                } else if (attributes) {
+                    bodyPeek.section.push(attributes);
+                }
 
-            queryStructure.push(bodyPeek);
-        };
+                queryStructure.push(bodyPeek);
+            };
 
-        ['all', 'fast', 'full', 'uid', 'flags', 'bodyStructure', 'envelope', 'internalDate'].forEach(key => {
-            if (query[key]) {
-                queryStructure.push({ type: 'ATOM', value: key.toUpperCase() });
-            }
-        });
+            ['all', 'fast', 'full', 'uid', 'flags', 'bodyStructure', 'envelope', 'internalDate'].forEach(key => {
+                if (query[key]) {
+                    queryStructure.push({ type: 'ATOM', value: key.toUpperCase() });
+                }
+            });
 
-        if (query.size) {
-            queryStructure.push({ type: 'ATOM', value: 'RFC822.SIZE' });
-        }
+            if (query.size) {
+                queryStructure.push({ type: 'ATOM', value: 'RFC822.SIZE' });
+            }
 
-        if (query.source) {
-            let partial;
-            if (typeof query.source === 'object' && (query.source.start || query.source.maxLength)) {
-                partial = [Number(query.source.start) || 0];
-                if (query.source.maxLength && !isNaN(query.source.maxLength)) {
-                    partial.push(Number(query.source.maxLength));
+            if (query.source) {
+                let partial;
+                if (typeof query.source === 'object' && (query.source.start || query.source.maxLength)) {
+                    partial = [Number(query.source.start) || 0];
+                    if (query.source.maxLength && !isNaN(query.source.maxLength)) {
+                        partial.push(Number(query.source.maxLength));
+                    }
                 }
+                queryStructure.push({ type: 'ATOM', value: `${commandKey}.PEEK`, section: [], partial });
             }
-            queryStructure.push({ type: 'ATOM', value: `${commandKey}.PEEK`, section: [], partial });
-        }
 
-        // if possible, always request for unique email id
-        if (connection.capabilities.has('OBJECTID')) {
-            queryStructure.push({ type: 'ATOM', value: 'EMAILID' });
-        } else if (connection.capabilities.has('X-GM-EXT-1')) {
-            queryStructure.push({ type: 'ATOM', value: 'X-GM-MSGID' });
-        }
-
-        if (query.threadId) {
+            // if possible, always request for unique email id
             if (connection.capabilities.has('OBJECTID')) {
-                queryStructure.push({ type: 'ATOM', value: 'THREADID' });
+                queryStructure.push({ type: 'ATOM', value: 'EMAILID' });
             } else if (connection.capabilities.has('X-GM-EXT-1')) {
-                queryStructure.push({ type: 'ATOM', value: 'X-GM-THRID' });
+                queryStructure.push({ type: 'ATOM', value: 'X-GM-MSGID' });
             }
-        }
 
-        if (query.labels) {
-            if (connection.capabilities.has('X-GM-EXT-1')) {
-                queryStructure.push({ type: 'ATOM', value: 'X-GM-LABELS' });
+            if (query.threadId) {
+                if (connection.capabilities.has('OBJECTID')) {
+                    queryStructure.push({ type: 'ATOM', value: 'THREADID' });
+                } else if (connection.capabilities.has('X-GM-EXT-1')) {
+                    queryStructure.push({ type: 'ATOM', value: 'X-GM-THRID' });
+                }
             }
-        }
 
-        // always ask for modseq if possible
-        if (connection.enabled.has('CONDSTORE') && !mailbox.noModseq) {
-            queryStructure.push({ type: 'ATOM', value: 'MODSEQ' });
-        }
+            if (query.labels) {
+                if (connection.capabilities.has('X-GM-EXT-1')) {
+                    queryStructure.push({ type: 'ATOM', value: 'X-GM-LABELS' });
+                }
+            }
 
-        // always make sure to include UID in the request as well even though server might auto-add it itself
-        if (!query.uid) {
-            queryStructure.push({ type: 'ATOM', value: 'UID' });
-        }
+            // always ask for modseq if possible
+            if (connection.enabled.has('CONDSTORE') && !mailbox.noModseq) {
+                queryStructure.push({ type: 'ATOM', value: 'MODSEQ' });
+            }
 
-        if (query.headers) {
-            if (Array.isArray(query.headers)) {
-                setBodyPeek([{ type: 'ATOM', value: 'HEADER.FIELDS' }, query.headers.map(header => ({ type: 'ATOM', value: header }))]);
-            } else {
-                setBodyPeek({ type: 'ATOM', value: 'HEADER' });
+            // always make sure to include UID in the request as well even though server might auto-add it itself
+            if (!query.uid) {
+                queryStructure.push({ type: 'ATOM', value: 'UID' });
             }
-        }
 
-        if (query.bodyParts && query.bodyParts.length) {
-            query.bodyParts.forEach(part => {
-                if (!part) {
-                    return;
+            if (query.headers) {
+                if (Array.isArray(query.headers)) {
+                    setBodyPeek([{ type: 'ATOM', value: 'HEADER.FIELDS' }, query.headers.map(header => ({ type: 'ATOM', value: header }))]);
+                } else {
+                    setBodyPeek({ type: 'ATOM', value: 'HEADER' });
                 }
-                let key;
-                let partial;
-                if (typeof part === 'object') {
-                    if (!part.key || typeof part.key !== 'string') {
+            }
+
+            if (query.bodyParts && query.bodyParts.length) {
+                query.bodyParts.forEach(part => {
+                    if (!part) {
                         return;
                     }
-                    key = part.key.toUpperCase();
-                    if (part.start || part.maxLength) {
-                        partial = [Number(part.start) || 0];
-                        if (part.maxLength && !isNaN(part.maxLength)) {
-                            partial.push(Number(part.maxLength));
+                    let key;
+                    let partial;
+                    if (typeof part === 'object') {
+                        if (!part.key || typeof part.key !== 'string') {
+                            return;
                         }
+                        key = part.key.toUpperCase();
+                        if (part.start || part.maxLength) {
+                            partial = [Number(part.start) || 0];
+                            if (part.maxLength && !isNaN(part.maxLength)) {
+                                partial.push(Number(part.maxLength));
+                            }
+                        }
+                    } else if (typeof part === 'string') {
+                        key = part.toUpperCase();
+                    } else {
+                        return;
                     }
-                } else if (typeof part === 'string') {
-                    key = part.toUpperCase();
-                } else {
-                    return;
-                }
 
-                setBodyPeek({ type: 'ATOM', value: key }, partial);
-            });
-        }
+                    setBodyPeek({ type: 'ATOM', value: key }, partial);
+                });
+            }
 
-        if (queryStructure.length === 1) {
-            queryStructure = queryStructure.pop();
-        }
+            if (queryStructure.length === 1) {
+                queryStructure = queryStructure.pop();
+            }
 
-        attributes.push(queryStructure);
+            attributes.push(queryStructure);
+
+            if (options.changedSince && connection.enabled.has('CONDSTORE') && !mailbox.noModseq) {
+                let changedSinceArgs = [
+                    {
+                        type: 'ATOM',
+                        value: 'CHANGEDSINCE'
+                    },
+                    {
+                        type: 'ATOM',
+                        value: options.changedSince.toString()
+                    }
+                ];
 
-        if (options.changedSince && connection.enabled.has('CONDSTORE') && !mailbox.noModseq) {
-            let changedSinceArgs = [
-                {
-                    type: 'ATOM',
-                    value: 'CHANGEDSINCE'
-                },
-                {
-                    type: 'ATOM',
-                    value: options.changedSince.toString()
+                if (options.uid && connection.enabled.has('QRESYNC')) {
+                    changedSinceArgs.push({
+                        type: 'ATOM',
+                        value: 'VANISHED'
+                    });
                 }
-            ];
 
-            if (options.uid && connection.enabled.has('QRESYNC')) {
-                changedSinceArgs.push({
-                    type: 'ATOM',
-                    value: 'VANISHED'
-                });
+                attributes.push(changedSinceArgs);
             }
 
-            attributes.push(changedSinceArgs);
-        }
-
-        response = await connection.exec(options.uid ? 'UID FETCH' : 'FETCH', attributes, {
-            untagged: {
-                FETCH: async untagged => {
-                    messages.count++;
-                    let formatted = await formatMessageResponse(untagged, mailbox);
-                    if (typeof options.onUntaggedFetch === 'function') {
-                        await new Promise((resolve, reject) => {
-                            options.onUntaggedFetch(formatted, err => {
-                                if (err) {
-                                    reject(err);
-                                } else {
-                                    resolve();
-                                }
+            response = await connection.exec(options.uid ? 'UID FETCH' : 'FETCH', attributes, {
+                untagged: {
+                    FETCH: async untagged => {
+                        messages.count++;
+                        let formatted = await formatMessageResponse(untagged, mailbox);
+                        if (typeof options.onUntaggedFetch === 'function') {
+                            await new Promise((resolve, reject) => {
+                                options.onUntaggedFetch(formatted, err => {
+                                    if (err) {
+                                        reject(err);
+                                    } else {
+                                        resolve();
+                                    }
+                                });
                             });
-                        });
-                    } else {
-                        messages.list.push(formatted);
+                        } else {
+                            messages.list.push(formatted);
+                        }
                     }
                 }
+            });
+
+            response.next();
+            return messages;
+        } catch (err) {
+            if (err.code === 'ETHROTTLE') {
+                // retrying
+                connection.log.warn({
+                    msg: 'Retrying throttled request',
+                    cid: connection.id,
+                    code: err.code,
+                    response: err.responseText,
+                    throttleReset: err.throttleReset,
+                    retryCount
+                });
+                retryCount++;
+                continue;
             }
-        });
 
-        response.next();
-        return messages;
-    } catch (err) {
-        connection.log.warn({ err, cid: connection.id });
-        return false;
+            connection.log.warn({ err, cid: connection.id });
+            return false;
+        }
     }
 };

package/lib/commands/idle.js

@@ -110,6 +110,8 @@ module.exports = async (connection, maxIdleTime) => {
         return;
     }
 
+    maxIdleTime = 3000;
+
     if (connection.capabilities.has('IDLE')) {
         let idleTimer;
         let stillIdling = false;

package/lib/commands/list.js

@@ -62,7 +62,11 @@ module.exports = async (connection, reference, mailbox, options) => {
         let specialUseHints = {};
         if (options.specialUseHints && typeof options.specialUseHints === 'object') {
             for (let type of Object.keys(options.specialUseHints)) {
-                if (['sent', 'junk', 'trash', 'drafts'].includes(type) && options.specialUseHints[type] && typeof options.specialUseHints[type] === 'string') {
+                if (
+                    ['sent', 'junk', 'trash', 'drafts', 'archive'].includes(type) &&
+                    options.specialUseHints[type] &&
+                    typeof options.specialUseHints[type] === 'string'
+                ) {
                     specialUseHints[normalizePath(connection, options.specialUseHints[type])] = `\\${type.replace(/^./, c => c.toUpperCase())}`;
                 }
             }

package/lib/imap-flow.js

@@ -562,25 +562,31 @@ class ImapFlow extends EventEmitter {
                         if (txt) {
                             err.responseText = txt;
 
+                            let throttleDelay = false;
+
                             // MS365 throttling
                             // tag BAD Request is throttled. Suggested Backoff Time: 92415 milliseconds
                             if (/Request is throttled/i.test(txt) && /Backoff Time/i.test(txt)) {
                                 let throttlingMatch = txt.match(/Backoff Time[:=\s]+(\d+)/i);
                                 if (throttlingMatch && throttlingMatch[1] && !isNaN(throttlingMatch[1])) {
-                                    err.code = 'ETHROTTLE';
-                                    err.throttleReset = Number(throttlingMatch[1]);
-
-                                    let delayResponse = err.throttleReset;
-                                    if (delayResponse > 5 * 60 * 1000) {
-                                        // max delay cap
-                                        delayResponse = 5 * 60 * 1000;
-                                    }
-                                    if (delayResponse) {
-                                        this.log.warn({ msg: 'Throttling detected', err, cid: this.id, delayResponse });
-                                        await new Promise(r => setTimeout(r, delayResponse));
-                                    }
+                                    throttleDelay = Number(throttlingMatch[1]);
                                 }
                             }
+
+                            // Wait and return a throttling error
+                            if (throttleDelay) {
+                                err.code = 'ETHROTTLE';
+                                err.throttleReset = throttleDelay;
+
+                                let delayResponse = throttleDelay;
+                                if (delayResponse > 5 * 60 * 1000) {
+                                    // max delay cap
+                                    delayResponse = 5 * 60 * 1000;
+                                }
+
+                                this.log.warn({ msg: 'Throttling detected', err, cid: this.id, throttleDelay, delayResponse });
+                                await new Promise(r => setTimeout(r, delayResponse));
+                            }
                         }
 
                         request.reject(err);
@@ -1939,7 +1945,7 @@ class ImapFlow extends EventEmitter {
     /**
      * @typedef {Object} AppendResponseObject
      * @global
-     * @property {String} path full mailbox path where the message was uploaded to
+     * @property {String} destination full mailbox path where the message was uploaded to
      * @property {BigInt} [uidValidity] mailbox `UIDVALIDITY` if server has `UIDPLUS` extension enabled
      * @property {Number} [uid] UID of the uploaded message if server has `UIDPLUS` extension enabled
      * @property {Number} [seq] sequence number of the uploaded message if path is currently selected mailbox
@@ -2736,7 +2742,10 @@ class ImapFlow extends EventEmitter {
 
         let result = await handler(this, ...args);
 
-        this.autoidle();
+        if (command !== 'IDLE') {
+            // do not autostart IDLE, if IDLE itself was stopped
+            this.autoidle();
+        }
 
         return result;
     }

package/lib/special-use.js

@@ -182,7 +182,7 @@ module.exports = {
             'الرسائل العشوائية',
             'هرزنامه',
             'สแปม',
-            '‎垃圾郵件',
+            '垃圾郵件',
             '垃圾邮件',
             '垃圾電郵'
         ],
@@ -275,7 +275,9 @@ module.exports = {
             '草稿',
             '草稿',
             '임시 보관함'
-        ]
+        ],
+
+        '\\Archive': ['archive']
     },
 
     specialUse(hasSpecialUseExtension, folder) {
@@ -289,7 +291,11 @@ module.exports = {
         }
 
         if (!result || !result.flag) {
-            let name = folder.name.toLowerCase().trim();
+            let name = folder.name
+                .toLowerCase()
+                .replace(/\u200e/g, '')
+                .trim();
+
             result = {
                 flag: Object.keys(module.exports.names).find(flag => module.exports.names[flag].includes(name)),
                 source: 'name'

package/lib/types.d.ts

@@ -805,13 +805,13 @@ declare type SearchObject = {
 };
 
 /**
- * @property path - full mailbox path where the message was uploaded to
+ * @property destination - full mailbox path where the message was uploaded to
  * @property [uidValidity] - mailbox `UIDVALIDITY` if server has `UIDPLUS` extension enabled
  * @property [uid] - UID of the uploaded message if server has `UIDPLUS` extension enabled
  * @property [seq] - sequence number of the uploaded message if path is currently selected mailbox
  */
 declare type AppendResponseObject = {
-    path: string;
+    destination: string;
     uidValidity?: bigint;
     uid?: number;
     seq?: number;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "imapflow",
-    "version": "1.0.144",
+    "version": "1.0.146",
     "description": "IMAP Client for Node",
     "main": "./lib/imap-flow.js",
     "scripts": {
@@ -9,7 +9,8 @@
         "docs": "rm -rf docs && mkdir -p docs && jsdoc lib/imap-flow.js -c jsdoc.json -R README.md --destination docs/ && cp assets/favicon.ico docs",
         "dst": "node types.js",
         "build": "npm run docs && npm run dst",
-        "st": "npm run docs && st -d docs -i index.html"
+        "st": "npm run docs && st -d docs -i index.html",
+        "update": "rm -rf node_modules package-lock.json && ncu -u && npm install"
     },
     "repository": {
         "type": "git",
@@ -30,10 +31,10 @@
         "@babel/eslint-parser": "7.22.15",
         "@babel/eslint-plugin": "7.22.10",
         "@babel/plugin-syntax-class-properties": "7.12.13",
-        "@babel/preset-env": "7.22.15",
-        "@types/node": "20.6.0",
+        "@babel/preset-env": "7.23.2",
+        "@types/node": "20.8.9",
         "braintree-jsdoc-template": "3.3.0",
-        "eslint": "8.49.0",
+        "eslint": "8.52.0",
         "eslint-config-nodemailer": "1.2.0",
         "eslint-config-prettier": "9.0.0",
         "grunt": "1.6.1",
@@ -51,8 +52,8 @@
         "libmime": "5.2.1",
         "libqp": "2.0.1",
         "mailsplit": "5.4.0",
-        "nodemailer": "6.9.5",
-        "pino": "8.15.1",
+        "nodemailer": "6.9.7",
+        "pino": "8.16.1",
         "socks": "2.7.1"
     }
 }