imapflow 1.0.76 → 1.0.80

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # CHANGELOG
 
+-   1.0.80 - 2022-02-07
+
+    -   Added support for the missing NOMODSEQ mailbox modifier
+
+-   1.0.79 - 2021-12-29
+
+    -   Added property `greeting` that contains that first response from the server
+
+-   1.0.78 - 2021-11-28
+
+    -   Proxy support. Use configuration options `proxy: "url"` to use proxies
+
+-   1.0.77 - 2021-11-25
+
+    -   Testing out proxy connections
+    -   Add X-GM-RAW support for search (jhi721 #68)
+
 -   1.0.75 - 2021-10-15
 
     -   Fixed an exception with failing IDLE (ramiroaisen #60)

package/README.md

@@ -1,10 +1,14 @@
 # ImapFlow
 
-_Node.js IMAP Client library built for for [EmailEngine Email API](https://emailengine.app/)._
+---
 
-The focus for ImapFlow is to provide easy to use API over IMAP. Using ImapFlow does not expect knowledge about specific IMAP details, general understanding is good enough.
+> Managing IMAP connections is cool, but if you are only looking for an easy way to integrate email accounts, then ImapFlow was built for [EmailEngine Email API](https://emailengine.app/). It's a self-hosted software that converts all IMAP accounts to easy-to-use REST interfaces.
 
-IMAP extensions are handled in the background, so for example you can always request `labels` value from a {@link FetchQueryObject|fetch()} call but if the IMAP server does not support `X-GM-EXT1` extension, then `labels` value is not included in the response.
+---
+
+The focus for ImapFlow is to provide easy to use API over IMAP. Using ImapFlow does not expect knowledge about specific IMAP details. A general understanding is good enough.
+
+IMAP extensions are handled in the background, so for example, you can always request `labels` value from a {@link FetchQueryObject|fetch()} call, but if the IMAP server does not support `X-GM-EXT1` extension, then `labels` value is not included in the response.
 
 ## Source
 

package/lib/commands/fetch.js

@@ -86,7 +86,7 @@ module.exports = async (connection, range, query, options) => {
         }
 
         // always ask for modseq if possible
-        if (connection.enabled.has('CONDSTORE')) {
+        if (connection.enabled.has('CONDSTORE') && !mailbox.noModseq) {
             queryStructure.push({ type: 'ATOM', value: 'MODSEQ' });
         }
 
@@ -137,7 +137,7 @@ module.exports = async (connection, range, query, options) => {
 
         attributes.push(queryStructure);
 
-        if (options.changedSince && connection.enabled.has('CONDSTORE')) {
+        if (options.changedSince && connection.enabled.has('CONDSTORE') && !mailbox.noModseq) {
             attributes.push([
                 {
                     type: 'ATOM',

package/lib/commands/idle.js

@@ -66,7 +66,7 @@ module.exports = async connection => {
 
             response = await connection.exec('IDLE', false, {
                 onPlusTag: async () => {
-                    connection.log.debug({ msg: `Initiated IDLE, waiting for server input` });
+                    connection.log.debug({ msg: `Initiated IDLE, waiting for server input`, doneRequested });
                     canEnd = true;
                     if (doneRequested) {
                         preCheck();

package/lib/commands/select.js

@@ -85,6 +85,16 @@ module.exports = async (connection, path, options) => {
 
                         map[key] = value;
                     }
+
+                    if (section && section.length === 1 && section[0].type === 'ATOM' && typeof section[0].value === 'string') {
+                        let key = section[0].value.toLowerCase();
+                        switch (key) {
+                            case 'nomodseq':
+                                key = 'noModseq';
+                                map[key] = true;
+                                break;
+                        }
+                    }
                 },
                 FLAGS: async untagged => {
                     if (!untagged.attributes || (!untagged.attributes.length && Array.isArray(untagged.attributes[0]))) {

package/lib/commands/store.js

@@ -53,7 +53,7 @@ module.exports = async (connection, range, flags, options) => {
 
     let attributes = [{ type: 'SEQUENCE', value: range }, { type: 'ATOM', value: operation }, flags.map(flag => ({ type: 'ATOM', value: flag }))];
 
-    if (options.unchangedSince && connection.enabled.has('CONDSTORE')) {
+    if (options.unchangedSince && connection.enabled.has('CONDSTORE') && !connection.mailbox.noModseq) {
         attributes.push([
             {
                 type: 'ATOM',

package/lib/handler/imap-compiler.js

@@ -117,11 +117,13 @@ module.exports = async (response, options) => {
             case 'SECTION':
                 val = (node.value || '').toString('binary');
 
-                if (node.value === '' || imapFormalSyntax.verify(val.charAt(0) === '\\' ? val.substr(1) : val, imapFormalSyntax['ATOM-CHAR']()) >= 0) {
-                    val = JSON.stringify(val);
-                }
+                if (!node.section || val) {
+                    if (node.value === '' || imapFormalSyntax.verify(val.charAt(0) === '\\' ? val.substr(1) : val, imapFormalSyntax['ATOM-CHAR']()) >= 0) {
+                        val = JSON.stringify(val);
+                    }
 
-                resp += val;
+                    resp += val;
+                }
 
                 if (node.section) {
                     resp += '[';

package/lib/imap-flow.js

@@ -27,6 +27,8 @@ const libbase64 = require('libbase64');
 const FlowedDecoder = require('mailsplit/lib/flowed-decoder');
 const { PassThrough } = require('stream');
 
+const { proxyConnection } = require('./proxy-connection');
+
 const { comparePaths, updateCapabilities, getFolderTree, formatMessageResponse, getDecoder, packMessageRange, normalizePath } = require('./tools');
 
 const imapCommands = require('./imap-commands.js');
@@ -56,6 +58,7 @@ const states = {
  * @property {Set<string>} permanentFlags A Set of flags available to use in this mailbox. If it is not set or includes special flag "\\\*" then any flag can be used.
  * @property {String} [mailboxId] unique mailbox ID if server has `OBJECTID` extension enabled
  * @property {BigInt} [highestModseq] latest known modseq value if server has CONDSTORE or XYMHIGHESTMODSEQ enabled
+ * @property {String} [noModseq] if true then the server doesn't support the persistent storage of mod-sequences for the mailbox
  * @property {BigInt} uidValidity Mailbox `UIDVALIDITY` value
  * @property {Number} uidNext Next predicted UID
  * @property {Number} exists Messages in this folder
@@ -124,6 +127,7 @@ class ImapFlow extends EventEmitter {
      * @param {Boolean} [options.logRaw=false] If true then log data read from and written to socket encoded in base64
      * @param {Boolean} [options.emitLogs=false] If `true` then in addition of sending data to logger, ImapFlow emits 'log' events with the same data
      * @param {Boolean} [options.verifyOnly=false] If `true` then logs out automatically after successful authentication
+     * @param {Boolean} [options.proxy] Optional proxy URL. Supports HTTP CONNECT (`http://`, `https://`) and SOCKS (`socks://`, `socks4://`, `socks5://`) proxies
      */
     constructor(options) {
         super();
@@ -805,7 +809,13 @@ class ImapFlow extends EventEmitter {
         return false;
     }
 
-    async initialOK() {
+    async initialOK(message) {
+        this.greeting = (message.attributes || [])
+            .filter(entry => entry.type === 'TEXT')
+            .map(entry => entry.value)
+            .filter(entry => entry)
+            .join('');
+
         clearTimeout(this.greetingTimeout);
         this.untaggedHandlers.OK = null;
         this.untaggedHandlers.PREAUTH = null;
@@ -1054,6 +1064,22 @@ class ImapFlow extends EventEmitter {
         this.untaggedHandlers.EXISTS = (...args) => this.untaggedExists(...args);
         this.untaggedHandlers.EXPUNGE = (...args) => this.untaggedExpunge(...args);
 
+        let socket = false;
+        if (this.options.proxy) {
+            try {
+                socket = await proxyConnection(this.log, this.options.proxy, this.host, this.port);
+                if (!socket) {
+                    throw new Error('Failed to setup proxy connection');
+                }
+            } catch (err) {
+                let error = new Error('Failed to setup proxy connection');
+                error.code = err.code || 'ProxyError';
+                error._err = err;
+                this.log.error({ error, cid: this.id });
+                throw error;
+            }
+        }
+
         await new Promise((resolve, reject) => {
             this.connectTimeout = setTimeout(() => {
                 let err = new Error('Failed to established connection in required time');
@@ -1063,7 +1089,7 @@ class ImapFlow extends EventEmitter {
                 reject(err);
             }, CONNECT_TIMEOUT);
 
-            this.socket = connector.connect(opts, () => {
+            let onConnect = () => {
                 clearTimeout(this.connectTimeout);
                 this.socket.setKeepAlive(true, 5 * 1000);
                 this.socket.setTimeout(SOCKET_TIMEOUT);
@@ -1105,7 +1131,23 @@ class ImapFlow extends EventEmitter {
                 // executed by initial "* OK"
                 this.initialResolve = resolve;
                 this.initialReject = reject;
-            });
+            };
+
+            if (socket) {
+                // socket is already establised via proxy
+                if (this.secureConnection) {
+                    // TLS socket requires a handshake
+                    opts.socket = socket;
+                    this.socket = connector.connect(opts, onConnect);
+                } else {
+                    // cleartext socket is already usable
+                    this.socket = socket;
+                    setImmediate(onConnect);
+                }
+            } else {
+                this.socket = connector.connect(opts, onConnect);
+            }
+
             this.writeSocket = this.socket;
 
             this.socket.once('error', err => {

package/lib/proxy-connection.js

@@ -0,0 +1,120 @@
+'use strict';
+
+const httpProxyClient = require('nodemailer/lib/smtp-connection/http-proxy-client');
+const { SocksClient } = require('socks');
+const util = require('util');
+const httpProxyClientAsync = util.promisify(httpProxyClient);
+const dns = require('dns').promises;
+const net = require('net');
+
+const proxyConnection = async (logger, connectionUrl, host, port) => {
+    let proxyUrl = new URL(connectionUrl);
+
+    let protocol = proxyUrl.protocol.replace(/:$/, '').toLowerCase();
+
+    if (!net.isIP(host)) {
+        let resolveResult = await dns.resolve(host);
+        if (resolveResult && resolveResult.length) {
+            host = resolveResult[0];
+        }
+    }
+
+    switch (protocol) {
+        // Connect using a HTTP CONNECT method
+        case 'http':
+        case 'https': {
+            try {
+                let socket = await httpProxyClientAsync(proxyUrl.href, port, host);
+                if (socket) {
+                    if (proxyUrl.password) {
+                        proxyUrl.password = '(hidden)';
+                    }
+                    logger.info({
+                        msg: 'Established a socket via HTTP proxy',
+                        proxyUrl: proxyUrl.href,
+                        port,
+                        host
+                    });
+                }
+                return socket;
+            } catch (err) {
+                if (proxyUrl.password) {
+                    proxyUrl.password = '(hidden)';
+                }
+                logger.error({
+                    msg: 'Failed to establish a socket via HTTP proxy',
+                    proxyUrl: proxyUrl.href,
+                    port,
+                    host,
+                    err
+                });
+                throw err;
+            }
+        }
+
+        // SOCKS proxy
+        case 'socks':
+        case 'socks5':
+        case 'socks4':
+        case 'socks4a': {
+            let proxyType = Number(protocol.replace(/\D/g, '')) || 5;
+
+            let targetHost = proxyUrl.hostname;
+            if (!net.isIP(targetHost)) {
+                let resolveResult = await dns.resolve(targetHost);
+                if (resolveResult && resolveResult.length) {
+                    targetHost = resolveResult[0];
+                }
+            }
+
+            let connectionOpts = {
+                proxy: {
+                    host: targetHost,
+                    port: Number(proxyUrl.port) || 1080,
+                    type: proxyType
+                },
+                destination: {
+                    host,
+                    port
+                },
+                command: 'connect',
+                set_tcp_nodelay: true
+            };
+
+            if (proxyUrl.username || proxyUrl.password) {
+                connectionOpts.proxy.userId = proxyUrl.username;
+                connectionOpts.proxy.password = proxyUrl.password;
+            }
+
+            try {
+                const info = await SocksClient.createConnection(connectionOpts);
+                if (info && info.socket) {
+                    if (proxyUrl.password) {
+                        proxyUrl.password = '(hidden)';
+                    }
+                    logger.info({
+                        msg: 'Established a socket via SOCKS proxy',
+                        proxyUrl: proxyUrl.href,
+                        port,
+                        host
+                    });
+                }
+                return info.socket;
+            } catch (err) {
+                if (proxyUrl.password) {
+                    proxyUrl.password = '(hidden)';
+                }
+                logger.error({
+                    msg: 'Failed to establish a socket via SOCKS proxy',
+                    proxyUrl: proxyUrl.href,
+                    port,
+                    host,
+                    err
+                });
+                throw err;
+            }
+        }
+    }
+};
+
+module.exports = { proxyConnection };

package/lib/search-compiler.js

@@ -117,6 +117,11 @@ module.exports.searchCompiler = (connection, query) => {
                         setOpt(attributes, 'X-GM-THRID', params[term]);
                     }
                     break;
+                case 'GMRAW':
+                    if (connection.capabilities.has('X-GM-EXT-1')) {
+                        setOpt(attributes, 'X-GM-RAW', params[term]);
+                    }
+                    break;
 
                 case 'BEFORE':
                 case 'ON':

package/lib/types.d.ts

@@ -26,6 +26,7 @@ declare module "imapflow" {
      * @param [options.logRaw = false] - If true then log data read from and written to socket encoded in base64
      * @param [options.emitLogs = false] - If `true` then in addition of sending data to logger, ImapFlow emits 'log' events with the same data
      * @param [options.verifyOnly = false] - If `true` then logs out automatically after successful authentication
+     * @param [options.proxy] - Optional proxy URL. Supports HTTP CONNECT (`http://`, `https://`) and SOCKS (`socks://`, `socks4://`, `socks5://`) proxies
      */
     class ImapFlow extends EventEmitter {
         constructor(options: {
@@ -50,6 +51,7 @@ declare module "imapflow" {
             logRaw?: boolean;
             emitLogs?: boolean;
             verifyOnly?: boolean;
+            proxy?: boolean;
         });
         /**
          * Current module version as a static class property
@@ -481,6 +483,7 @@ declare module "imapflow" {
  * @property permanentFlags - A Set of flags available to use in this mailbox. If it is not set or includes special flag "\\\*" then any flag can be used.
  * @property [mailboxId] - unique mailbox ID if server has `OBJECTID` extension enabled
  * @property [highestModseq] - latest known modseq value if server has CONDSTORE or XYMHIGHESTMODSEQ enabled
+ * @property [noModseq] - if true then the server doesn't support the persistent storage of mod-sequences for the mailbox
  * @property uidValidity - Mailbox `UIDVALIDITY` value
  * @property uidNext - Next predicted UID
  * @property exists - Messages in this folder
@@ -495,6 +498,7 @@ declare type MailboxObject = {
     permanentFlags: Set<string>;
     mailboxId?: string;
     highestModseq?: bigint;
+    noModseq?: string;
     uidValidity: bigint;
     uidNext: number;
     exists: number;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "imapflow",
-    "version": "1.0.76",
+    "version": "1.0.80",
     "description": "IMAP Client for Node",
     "main": "./lib/imap-flow.js",
     "scripts": {
@@ -26,20 +26,20 @@
     },
     "homepage": "https://imapflow.com/",
     "devDependencies": {
-        "@babel/eslint-parser": "7.15.8",
-        "@babel/eslint-plugin": "7.14.5",
+        "@babel/eslint-parser": "7.17.0",
+        "@babel/eslint-plugin": "7.16.5",
         "@babel/plugin-syntax-class-properties": "7.12.13",
-        "@babel/preset-env": "7.15.8",
-        "@types/node": "16.11.1",
+        "@babel/preset-env": "7.16.11",
+        "@types/node": "17.0.15",
         "braintree-jsdoc-template": "3.3.0",
-        "eslint": "8.0.1",
+        "eslint": "8.8.0",
         "eslint-config-nodemailer": "1.2.0",
         "eslint-config-prettier": "8.3.0",
         "grunt": "1.4.1",
         "grunt-cli": "1.4.3",
         "grunt-contrib-nodeunit": "4.0.0",
         "grunt-eslint": "24.0.0",
-        "jsdoc": "3.6.7",
+        "jsdoc": "3.6.10",
         "tsd-jsdoc": "2.5.0"
     },
     "dependencies": {
@@ -49,6 +49,8 @@
         "libmime": "5.0.0",
         "libqp": "1.1.0",
         "mailsplit": "5.3.1",
-        "pino": "7.0.2"
+        "nodemailer": "6.7.2",
+        "pino": "7.6.5",
+        "socks": "2.6.2"
     }
 }