imapflow 1.0.73 → 1.0.77

package/.eslintrc

@@ -10,7 +10,8 @@
     "extends": ["nodemailer", "prettier"],
     "parser": "@babel/eslint-parser",
     "parserOptions": {
-        "ecmaVersion": 2018
+        "ecmaVersion": 2018,
+        "sourceType": "script"
     },
     "plugins": ["@babel"]
 }

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # CHANGELOG
 
+-   1.0.77 - 2021-11-25
+
+    -   Testing out proxy connections
+    -   Add X-GM-RAW support for search (jhi721 #68)
+
+-   1.0.75 - 2021-10-15
+
+    -   Fixed an exception with failing IDLE (ramiroaisen #60)
+    -   Do not use 1 and 1.MIME for single node messages, fallback to TEXT and HEADERS
+
 -   1.0.71 - 2021-09-28
 
     -   IDLE precheck changes

package/lib/commands/idle.js

@@ -12,50 +12,100 @@ module.exports = async connection => {
     if (connection.capabilities.has('IDLE')) {
         let response;
 
+        let preCheckWaitQueue = [];
         try {
             connection.idling = true;
 
-            let idleSent = false;
+            //let idleSent = false;
             let doneRequested = false;
             let doneSent = false;
+            let canEnd = false;
 
-            let preCheck = async () => {
+            let preCheck = () => {
                 doneRequested = true;
-                if (connection.idling && idleSent && !doneSent) {
-                    connection.log.debug({ src: 'c', msg: `DONE`, comment: `breaking IDLE` });
+                if (canEnd && !doneSent) {
+                    connection.log.debug({
+                        src: 'c',
+                        msg: `DONE`,
+                        comment: `breaking IDLE`,
+                        lockId: connection.currentLockId,
+                        path: connection.mailbox && connection.mailbox.path
+                    });
                     connection.write('DONE');
                     doneSent = true;
+
+                    connection.idling = false;
+                    connection.preCheck = false; // unset itself
+
+                    while (preCheckWaitQueue.length) {
+                        let { resolve } = preCheckWaitQueue.shift();
+                        resolve();
+                    }
                 }
             };
 
-            connection.preCheck = async () => {
-                await preCheck();
-                connection.idling = false;
-                connection.preCheck = false; // unset itself
+            connection.preCheck = () => {
+                let handler = new Promise((resolve, reject) => {
+                    preCheckWaitQueue.push({ resolve, reject });
+                });
+
+                connection.log.trace({
+                    msg: 'Requesting IDLE break',
+                    lockId: connection.currentLockId,
+                    path: connection.mailbox && connection.mailbox.path,
+                    queued: preCheckWaitQueue.length,
+                    doneRequested,
+                    canEnd,
+                    doneSent
+                });
+
+                preCheck();
+
+                return handler;
             };
 
             response = await connection.exec('IDLE', false, {
                 onPlusTag: async () => {
-                    connection.log.debug({ msg: `Initiated IDLE, waiting for server input` });
-                },
-                onSend: () => {
-                    idleSent = true;
+                    connection.log.debug({ msg: `Initiated IDLE, waiting for server input`, doneRequested });
+                    canEnd = true;
                     if (doneRequested) {
-                        preCheck().catch(err => {
-                            connection.log.debug({ msg: `preCheck error`, err });
-                        });
+                        preCheck();
                     }
+                },
+                onSend: () => {
+                    //idleSent = true;
                 }
             });
 
             // unset before response.next()
-            connection.preCheck = false;
+            if (typeof connection.preCheck === 'function') {
+                connection.log.trace({
+                    msg: 'Clearing pre-check function',
+                    lockId: connection.currentLockId,
+                    path: connection.mailbox && connection.mailbox.path,
+                    queued: preCheckWaitQueue.length,
+                    doneRequested,
+                    canEnd,
+                    doneSent
+                });
+                connection.preCheck = false;
+                while (preCheckWaitQueue.length) {
+                    let { resolve } = preCheckWaitQueue.shift();
+                    resolve();
+                }
+            }
 
             response.next();
             return;
         } catch (err) {
             connection.preCheck = false;
+            connection.idling = false;
+
             connection.log.warn({ err, cid: connection.id });
+            while (preCheckWaitQueue.length) {
+                let { reject } = preCheckWaitQueue.shift();
+                reject(err);
+            }
             return false;
         }
     }

package/lib/imap-flow.js

@@ -27,6 +27,8 @@ const libbase64 = require('libbase64');
 const FlowedDecoder = require('mailsplit/lib/flowed-decoder');
 const { PassThrough } = require('stream');
 
+const { proxyConnection } = require('./proxy-connection');
+
 const { comparePaths, updateCapabilities, getFolderTree, formatMessageResponse, getDecoder, packMessageRange, normalizePath } = require('./tools');
 
 const imapCommands = require('./imap-commands.js');
@@ -1031,7 +1033,7 @@ class ImapFlow extends EventEmitter {
      * await client.connect();
      */
     async connect() {
-        let connector = this.options.secure ? tls : net;
+        let connector = this.secureConnection ? tls : net;
 
         let opts = Object.assign(
             {
@@ -1054,6 +1056,22 @@ class ImapFlow extends EventEmitter {
         this.untaggedHandlers.EXISTS = (...args) => this.untaggedExists(...args);
         this.untaggedHandlers.EXPUNGE = (...args) => this.untaggedExpunge(...args);
 
+        let socket = false;
+        if (this.options.proxy) {
+            try {
+                socket = await proxyConnection(this.log, this.options.proxy, this.host, this.port);
+                if (!socket) {
+                    throw new Error('Failed to setup proxy connection');
+                }
+            } catch (err) {
+                let error = new Error('Failed to setup proxy connection');
+                error.code = err.code || 'ProxyError';
+                error._err = err;
+                this.log.error({ error, cid: this.id });
+                throw error;
+            }
+        }
+
         await new Promise((resolve, reject) => {
             this.connectTimeout = setTimeout(() => {
                 let err = new Error('Failed to established connection in required time');
@@ -1063,7 +1081,7 @@ class ImapFlow extends EventEmitter {
                 reject(err);
             }, CONNECT_TIMEOUT);
 
-            this.socket = connector.connect(opts, () => {
+            let onConnect = () => {
                 clearTimeout(this.connectTimeout);
                 this.socket.setKeepAlive(true, 5 * 1000);
                 this.socket.setTimeout(SOCKET_TIMEOUT);
@@ -1105,7 +1123,23 @@ class ImapFlow extends EventEmitter {
                 // executed by initial "* OK"
                 this.initialResolve = resolve;
                 this.initialReject = reject;
-            });
+            };
+
+            if (socket) {
+                // socket is already establised via proxy
+                if (this.secureConnection) {
+                    // TLS socket requires a handshake
+                    opts.socket = socket;
+                    this.socket = connector.connect(opts, onConnect);
+                } else {
+                    // cleartext socket is already usable
+                    this.socket = socket;
+                    setImmediate(onConnect);
+                }
+            } else {
+                this.socket = connector.connect(opts, onConnect);
+            }
+
             this.writeSocket = this.socket;
 
             this.socket.once('error', err => {
@@ -2025,7 +2059,13 @@ class ImapFlow extends EventEmitter {
             return {};
         }
 
-        options = options || {};
+        options = Object.assign(
+            {
+                chunkSize: 64 * 1024,
+                maxBytes: Infinity
+            },
+            options || {}
+        );
 
         let hasMore = true;
         let processed = 0;
@@ -2035,6 +2075,28 @@ class ImapFlow extends EventEmitter {
 
         let uid = false;
 
+        if (part === '1') {
+            // First part has special conditions for single node emails as
+            // the mime parts for root node are not 1 and 1.MIME but TEXT and HEADERS
+            let response = await this.fetchOne(range, { uid: true, bodyStructure: true }, options);
+
+            if (!response) {
+                return { response: false, chunk: false };
+            }
+
+            if (!uid && response.uid) {
+                uid = response.uid;
+                // force UID from now on even if first range was a sequence number
+                range = uid;
+                options.uid = true;
+            }
+
+            if (!response.bodyStructure.childNodes) {
+                // single text message
+                part = 'TEXT';
+            }
+        }
+
         let getNextPart = async query => {
             query = query || {};
 
@@ -2052,10 +2114,15 @@ class ImapFlow extends EventEmitter {
                     query.bodyParts = [];
                 }
 
-                if (query.size && /^[\d.]+$/.test(part)) {
-                    // fetch meta as well
-                    mimeKey = part + '.mime';
-                    query.bodyParts.push(mimeKey);
+                if (query.size) {
+                    if (/^[\d.]+$/.test(part)) {
+                        // fetch meta as well
+                        mimeKey = part + '.mime';
+                        query.bodyParts.push(mimeKey);
+                    } else if (part === 'text') {
+                        mimeKey = 'header';
+                        query.bodyParts.push(mimeKey);
+                    }
                 }
 
                 query.bodyParts.push({
@@ -2092,15 +2159,10 @@ class ImapFlow extends EventEmitter {
             }
 
             if (query.bodyParts) {
-                result.mime = response.bodyParts.get(mimeKey);
-
-                if (part === '1' && !result.mime) {
-                    // Outlook does not support requesting 1.MIME for text/* messages, so we issue an additional command
-                    let mimeResponse = await this.fetchOne(range, { bodyParts: ['header'] }, options);
-                    if (!mimeResponse) {
-                        return { response: false, chunk: false };
-                    }
-                    result.mime = mimeResponse.headers;
+                if (mimeKey === 'header') {
+                    result.mime = response.headers;
+                } else {
+                    result.mime = response.bodyParts.get(mimeKey);
                 }
             }
 

package/lib/proxy-connection.js

@@ -0,0 +1,120 @@
+'use strict';
+
+const httpProxyClient = require('nodemailer/lib/smtp-connection/http-proxy-client');
+const { SocksClient } = require('socks');
+const util = require('util');
+const httpProxyClientAsync = util.promisify(httpProxyClient);
+const dns = require('dns').promises;
+const net = require('net');
+
+const proxyConnection = async (logger, connectionUrl, host, port) => {
+    let proxyUrl = new URL(connectionUrl);
+
+    let protocol = proxyUrl.protocol.replace(/:$/, '').toLowerCase();
+
+    if (!net.isIP(host)) {
+        let resolveResult = await dns.resolve(host);
+        if (resolveResult && resolveResult.length) {
+            host = resolveResult[0];
+        }
+    }
+
+    switch (protocol) {
+        // Connect using a HTTP CONNECT method
+        case 'http':
+        case 'https': {
+            try {
+                let socket = await httpProxyClientAsync(proxyUrl.href, port, host);
+                if (socket) {
+                    if (proxyUrl.password) {
+                        proxyUrl.password = '(hidden)';
+                    }
+                    logger.info({
+                        msg: 'Established a socket via HTTP proxy',
+                        proxyUrl: proxyUrl.href,
+                        port,
+                        host
+                    });
+                }
+                return socket;
+            } catch (err) {
+                if (proxyUrl.password) {
+                    proxyUrl.password = '(hidden)';
+                }
+                logger.error({
+                    msg: 'Failed to establish a socket via HTTP proxy',
+                    proxyUrl: proxyUrl.href,
+                    port,
+                    host,
+                    err
+                });
+                throw err;
+            }
+        }
+
+        // SOCKS proxy
+        case 'socks':
+        case 'socks5':
+        case 'socks4':
+        case 'socks4a': {
+            let proxyType = Number(protocol.replace(/\D/g, '')) || 5;
+
+            let targetHost = proxyUrl.hostname;
+            if (!net.isIP(targetHost)) {
+                let resolveResult = await dns.resolve(targetHost);
+                if (resolveResult && resolveResult.length) {
+                    targetHost = resolveResult[0];
+                }
+            }
+
+            let connectionOpts = {
+                proxy: {
+                    host: targetHost,
+                    port: Number(proxyUrl.port) || 1080,
+                    type: proxyType
+                },
+                destination: {
+                    host,
+                    port
+                },
+                command: 'connect',
+                set_tcp_nodelay: true
+            };
+
+            if (proxyUrl.username || proxyUrl.password) {
+                connectionOpts.proxy.userId = proxyUrl.username;
+                connectionOpts.proxy.password = proxyUrl.password;
+            }
+
+            try {
+                const info = await SocksClient.createConnection(connectionOpts);
+                if (info && info.socket) {
+                    if (proxyUrl.password) {
+                        proxyUrl.password = '(hidden)';
+                    }
+                    logger.info({
+                        msg: 'Established a socket via SOCKS proxy',
+                        proxyUrl: proxyUrl.href,
+                        port,
+                        host
+                    });
+                }
+                return info.socket;
+            } catch (err) {
+                if (proxyUrl.password) {
+                    proxyUrl.password = '(hidden)';
+                }
+                logger.error({
+                    msg: 'Failed to establish a socket via SOCKS proxy',
+                    proxyUrl: proxyUrl.href,
+                    port,
+                    host,
+                    err
+                });
+                throw err;
+            }
+        }
+    }
+};
+
+module.exports = { proxyConnection };

package/lib/search-compiler.js

@@ -117,6 +117,11 @@ module.exports.searchCompiler = (connection, query) => {
                         setOpt(attributes, 'X-GM-THRID', params[term]);
                     }
                     break;
+                case 'GMRAW':
+                    if (connection.capabilities.has('X-GM-EXT-1')) {
+                        setOpt(attributes, 'X-GM-RAW', params[term]);
+                    }
+                    break;
 
                 case 'BEFORE':
                 case 'ON':

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "imapflow",
-    "version": "1.0.73",
+    "version": "1.0.77",
     "description": "IMAP Client for Node",
     "main": "./lib/imap-flow.js",
     "scripts": {
@@ -26,19 +26,19 @@
     },
     "homepage": "https://imapflow.com/",
     "devDependencies": {
-        "@babel/eslint-parser": "^7.15.7",
-        "@babel/eslint-plugin": "^7.14.5",
-        "@babel/plugin-syntax-class-properties": "^7.12.13",
-        "@babel/preset-env": "^7.15.6",
-        "@types/node": "16.10.1",
+        "@babel/eslint-parser": "7.16.3",
+        "@babel/eslint-plugin": "7.14.5",
+        "@babel/plugin-syntax-class-properties": "7.12.13",
+        "@babel/preset-env": "7.16.4",
+        "@types/node": "16.11.10",
         "braintree-jsdoc-template": "3.3.0",
-        "eslint": "7.32.0",
+        "eslint": "8.3.0",
         "eslint-config-nodemailer": "1.2.0",
         "eslint-config-prettier": "8.3.0",
         "grunt": "1.4.1",
         "grunt-cli": "1.4.3",
-        "grunt-contrib-nodeunit": "3.0.0",
-        "grunt-eslint": "23.0.0",
+        "grunt-contrib-nodeunit": "4.0.0",
+        "grunt-eslint": "24.0.0",
         "jsdoc": "3.6.7",
         "tsd-jsdoc": "2.5.0"
     },
@@ -48,7 +48,9 @@
         "libbase64": "1.2.1",
         "libmime": "5.0.0",
         "libqp": "1.1.0",
-        "mailsplit": "5.2.0",
-        "pino": "6.13.3"
+        "mailsplit": "5.3.1",
+        "nodemailer": "6.7.1",
+        "pino": "7.4.1",
+        "socks": "2.6.1"
     }
 }