imapflow 1.0.91 → 1.0.92

package/.gitattributes

@@ -0,0 +1 @@
+*.js text eol=lf

package/.github/workflows/test.yml

@@ -0,0 +1,21 @@
+name: Run tests
+
+on:
+    push:
+    pull_request:
+
+jobs:
+    test:
+        strategy:
+            matrix:
+                node: [16.x, 18.x]
+                os: [ubuntu-latest]
+        runs-on: ${{ matrix.os }}
+        steps:
+            - uses: actions/checkout@v2
+            - name: Use Node.js ${{ matrix.node }}
+              uses: actions/setup-node@v1
+              with:
+                  node-version: ${{ matrix.node }}
+            - run: npm install
+            - run: npm test

package/CHANGELOG.md

@@ -1,10 +1,15 @@
 # CHANGELOG
 
+-   1.0.92 - 2022-05-03
+
+    -   Added support for IMAP BINARY extension (rfc3516)
+    -   Logging improvements
+
 -   1.0.91 - 2022-05-02
 
     -   Do not throw if literal includes a null byte
     -   Use UTF8 path names by default
-    -   Overrides '\*' range query with then EXISTIS value
+    -   Overrides '\*' range query with the EXISTS value
 
 -   1.0.90 - 2022-04-04
 

package/lib/commands/append.js

@@ -38,7 +38,17 @@ module.exports = async (connection, destination, content, flags, idate) => {
         attributes.push({ type: 'STRING', value: idate }); // force quotes as required by date-time
     }
 
-    attributes.push({ type: 'LITERAL', value: content });
+    let isLiteral8 = false;
+    if (connection.capabilities.has('BINARY') && !connection.disableBinary) {
+        if (typeof content === 'string') {
+            content = Buffer.from(content);
+        }
+        // Value is literal8 if it contains NULL bytes. The server must support the BINARY extension
+        // and if it does not then send the value as a regular literal and hope for the best
+        isLiteral8 = content.indexOf(Buffer.from([0])) >= 0;
+    }
+
+    attributes.push({ type: 'LITERAL', value: content, isLiteral8 });
 
     let map = { destination };
     if (connection.mailbox && connection.mailbox.path) {

package/lib/commands/authenticate.js

@@ -39,7 +39,7 @@ module.exports = async (connection, username, accessToken) => {
                             try {
                                 errorResponse = JSON.parse(Buffer.from(resp.attributes[0].value, 'base64').toString());
                             } catch (err) {
-                                connection.log.debug({ data: resp.attributes[0].value, err });
+                                connection.log.debug({ errorResponse: resp.attributes[0].value, err });
                             }
                         }
 

package/lib/commands/fetch.js

@@ -18,7 +18,7 @@ module.exports = async (connection, range, query, options) => {
         list: []
     };
 
-    const commandKey = 'BODY';
+    const commandKey = connection.capabilities.has('BINARY') && !connection.disableBinary ? 'BINARY' : 'BODY';
 
     let response;
     try {

package/lib/handler/imap-compiler.js

@@ -85,7 +85,7 @@ module.exports = async (response, options) => {
                     let canAppend = !asArray || literalPlus || (literalMinus && literalLength <= 4096);
                     let usePlus = canAppend && (literalMinus || literalPlus);
 
-                    resp += `{${literalLength}${usePlus ? '+' : ''}}\r\n`;
+                    resp += `${node.isLiteral8 ? '~' : ''}{${literalLength}${usePlus ? '+' : ''}}\r\n`;
 
                     if (canAppend) {
                         resp += (node.value || '').toString('binary');

package/lib/handler/imap-stream.js

@@ -40,6 +40,7 @@ class ImapStream extends Transform {
         this.literals = [];
 
         this.compress = false;
+        this.secureConnection = this.options.secureConnection;
 
         this.processingInput = false;
         this.inputQueue = []; // unprocessed input chunks
@@ -197,7 +198,14 @@ class ImapStream extends Transform {
         }
 
         if (this.options.logRaw) {
-            this.log.trace({ src: 's', msg: 'read from socket', data: chunk.toString('base64'), compress: !!this.compress, cid: this.cid });
+            this.log.trace({
+                src: 's',
+                msg: 'read from socket',
+                data: chunk.toString('base64'),
+                compress: !!this.compress,
+                secure: !!this.secureConnection,
+                cid: this.cid
+            });
         }
 
         if (chunk && chunk.length) {

package/lib/handler/token-parser.js

@@ -29,13 +29,13 @@ class TokenParser {
             let elm;
             let partial;
 
-            if (!node.closed && node.type === 'SEQUENCE' && node.value === '*') {
-                node.closed = true;
+            if (!node.isClosed && node.type === 'SEQUENCE' && node.value === '*') {
+                node.isClosed = true;
                 node.type = 'ATOM';
             }
 
             // If the node was never closed, throw it
-            if (!node.closed) {
+            if (!node.isClosed) {
                 throw new Error('Unexpected end of input at position ' + (this.pos + this.str.length - 1));
             }
 
@@ -97,7 +97,7 @@ class TokenParser {
             childNodes: [],
             type: false,
             value: '',
-            closed: true
+            isClosed: true
         };
 
         if (parentNode) {
@@ -136,14 +136,14 @@ class TokenParser {
                             this.currentNode = this.createNode(this.currentNode, this.pos + i);
                             this.currentNode.type = 'string';
                             this.state = 'STRING';
-                            this.currentNode.closed = false;
+                            this.currentNode.isClosed = false;
                             break;
 
                         // ( starts a new list
                         case '(':
                             this.currentNode = this.createNode(this.currentNode, this.pos + i);
                             this.currentNode.type = 'LIST';
-                            this.currentNode.closed = false;
+                            this.currentNode.isClosed = false;
                             break;
 
                         // ) closes a list
@@ -152,7 +152,7 @@ class TokenParser {
                                 throw new Error('Unexpected list terminator ) at position ' + (this.pos + i));
                             }
 
-                            this.currentNode.closed = true;
+                            this.currentNode.isClosed = true;
                             this.currentNode.endPos = this.pos + i;
                             this.currentNode = this.currentNode.parentNode;
 
@@ -164,7 +164,7 @@ class TokenParser {
                             if (this.currentNode.type !== 'SECTION') {
                                 throw new Error('Unexpected section terminator ] at position ' + (this.pos + i));
                             }
-                            this.currentNode.closed = true;
+                            this.currentNode.isClosed = true;
                             this.currentNode.endPos = this.pos + i;
                             this.currentNode = this.currentNode.parentNode;
                             checkSP();
@@ -181,25 +181,36 @@ class TokenParser {
                                 this.currentNode = this.createNode(this.currentNode, this.pos + i);
                                 this.currentNode.type = 'PARTIAL';
                                 this.state = 'PARTIAL';
-                                this.currentNode.closed = false;
+                                this.currentNode.isClosed = false;
                             }
                             break;
 
+                        // binary literal8
+                        case '~': {
+                            let nextChr = this.str.charAt(i + 1);
+                            if (nextChr !== '{') {
+                                throw new Error('Unexpected literal8 marker at position ' + (this.pos + i));
+                            }
+                            this.expectedLiteralType = 'literal8';
+                            break;
+                        }
+
                         // { starts a new literal
                         case '{':
                             this.currentNode = this.createNode(this.currentNode, this.pos + i);
                             this.currentNode.type = 'LITERAL';
-                            this.currentNode.literalType = 'literal';
+                            this.currentNode.literalType = this.expectedLiteralType || 'literal';
+                            this.expectedLiteralType = false;
                             this.state = 'LITERAL';
-                            this.currentNode.closed = false;
+                            this.currentNode.isClosed = false;
                             break;
 
-                        // ( starts a new sequence
+                        // * starts a new sequence
                         case '*':
                             this.currentNode = this.createNode(this.currentNode, this.pos + i);
                             this.currentNode.type = 'SEQUENCE';
                             this.currentNode.value = chr;
-                            this.currentNode.closed = false;
+                            this.currentNode.isClosed = false;
                             this.state = 'SEQUENCE';
                             break;
 
@@ -219,7 +230,7 @@ class TokenParser {
 
                                 this.currentNode = this.createNode(this.currentNode, this.pos + i);
                                 this.currentNode.type = 'SECTION';
-                                this.currentNode.closed = false;
+                                this.currentNode.isClosed = false;
                                 this.state = 'NORMAL';
 
                                 // RFC2221 defines a response code REFERRAL whose payload is an
@@ -245,7 +256,7 @@ class TokenParser {
                                     this.currentNode = this.currentNode.parentNode;
 
                                     // close out the SECTION
-                                    this.currentNode.closed = true;
+                                    this.currentNode.isClosed = true;
                                     this.currentNode = this.currentNode.parentNode;
                                     checkSP();
                                 }
@@ -287,7 +298,7 @@ class TokenParser {
                         this.currentNode.endPos = this.pos + i - 1;
                         this.currentNode = this.currentNode.parentNode;
 
-                        this.currentNode.closed = true;
+                        this.currentNode.isClosed = true;
                         this.currentNode.endPos = this.pos + i;
                         this.currentNode = this.currentNode.parentNode;
                         this.state = 'NORMAL';
@@ -298,20 +309,20 @@ class TokenParser {
 
                     if ((chr === ',' || chr === ':') && this.currentNode.value.match(/^\d+$/)) {
                         this.currentNode.type = 'SEQUENCE';
-                        this.currentNode.closed = true;
+                        this.currentNode.isClosed = true;
                         this.state = 'SEQUENCE';
                     }
 
                     // [ starts a section group for this element
                     if (chr === '[') {
                         // allowed only for selected elements
-                        if (['BODY', 'BODY.PEEK'].indexOf(this.currentNode.value.toUpperCase()) < 0) {
+                        if (['BODY', 'BODY.PEEK', 'BINARY', 'BINARY.PEEK'].indexOf(this.currentNode.value.toUpperCase()) < 0) {
                             throw new Error('Unexpected section start char [ at position ' + this.pos);
                         }
                         this.currentNode.endPos = this.pos + i;
                         this.currentNode = this.createNode(this.currentNode.parentNode, this.pos + i);
                         this.currentNode.type = 'SECTION';
-                        this.currentNode.closed = false;
+                        this.currentNode.isClosed = false;
                         this.state = 'NORMAL';
                         break;
                     }
@@ -334,7 +345,7 @@ class TokenParser {
                     // DQUOTE ends the string sequence
                     if (chr === '"') {
                         this.currentNode.endPos = this.pos + i;
-                        this.currentNode.closed = true;
+                        this.currentNode.isClosed = true;
                         this.currentNode = this.currentNode.parentNode;
                         this.state = 'NORMAL';
 
@@ -360,7 +371,7 @@ class TokenParser {
                             throw new Error('Unexpected end of partial at position ' + this.pos);
                         }
                         this.currentNode.endPos = this.pos + i;
-                        this.currentNode.closed = true;
+                        this.currentNode.isClosed = true;
                         this.currentNode = this.currentNode.parentNode;
                         this.state = 'NORMAL';
                         checkSP();
@@ -386,7 +397,7 @@ class TokenParser {
                     if (this.currentNode.started) {
                         // only relevant if literals are not already parsed out from input
 
-                        // Disabled NUL check
+                        // Disabled NULL byte check
                         // See https://github.com/emailjs/emailjs-imap-handler/commit/f11b2822bedabe492236e8263afc630134a3c41c
                         /*
                         if (chr === '\u0000') {
@@ -398,7 +409,7 @@ class TokenParser {
 
                         if (this.currentNode.chPos >= this.currentNode.literalLength) {
                             this.currentNode.endPos = this.pos + i;
-                            this.currentNode.closed = true;
+                            this.currentNode.isClosed = true;
                             this.currentNode.value = this.currentNode.chBuffer.toString('binary');
                             this.currentNode.chBuffer = Buffer.alloc(0);
                             this.currentNode = this.currentNode.parentNode;
@@ -431,7 +442,7 @@ class TokenParser {
                             // special case where literal content length is 0
                             // close the node right away, do not wait for additional input
                             this.currentNode.endPos = this.pos + i;
-                            this.currentNode.closed = true;
+                            this.currentNode.isClosed = true;
                             this.currentNode = this.currentNode.parentNode;
                             this.state = 'NORMAL';
                             checkSP();
@@ -449,7 +460,7 @@ class TokenParser {
                             this.currentNode.endPos = this.pos + i + this.currentNode.value.length;
 
                             this.currentNode.started = false;
-                            this.currentNode.closed = true;
+                            this.currentNode.isClosed = true;
                             this.currentNode = this.currentNode.parentNode;
                             this.state = 'NORMAL';
                             checkSP();
@@ -482,7 +493,7 @@ class TokenParser {
                             throw new Error('Unexpected whitespace at position ' + (this.pos + i));
                         }
 
-                        this.currentNode.closed = true;
+                        this.currentNode.isClosed = true;
                         this.currentNode.endPos = this.pos + i - 1;
                         this.currentNode = this.currentNode.parentNode;
                         this.state = 'NORMAL';
@@ -491,7 +502,7 @@ class TokenParser {
                         this.currentNode.endPos = this.pos + i - 1;
                         this.currentNode = this.currentNode.parentNode;
 
-                        this.currentNode.closed = true;
+                        this.currentNode.isClosed = true;
                         this.currentNode.endPos = this.pos + i;
                         this.currentNode = this.currentNode.parentNode;
                         this.state = 'NORMAL';

package/lib/imap-flow.js

@@ -130,6 +130,7 @@ class ImapFlow extends EventEmitter {
      * @param {Boolean} [options.proxy] Optional proxy URL. Supports HTTP CONNECT (`http://`, `https://`) and SOCKS (`socks://`, `socks4://`, `socks5://`) proxies
      * @param {Boolean} [options.qresync=false] If true, then enables QRESYNC support. EXPUNGE notifications will include `uid` property instead of `seq`
      * @param {Number} [options.maxIdleTime] If set, then breaks and restarts IDLE every maxIdleTime ms
+     * @param {Boolean} [options.disableBinary=false] If true, then ignores the BINARY extension when making FETCH and APPEND calls
      */
     constructor(options) {
         super({ captureRejections: true });
@@ -183,7 +184,8 @@ class ImapFlow extends EventEmitter {
         this.streamer = new ImapStream({
             logger: this.log,
             cid: this.id,
-            logRaw: this.logRaw
+            logRaw: this.logRaw,
+            secureConnection: this.secureConnection
         });
 
         this.reading = false;
@@ -263,6 +265,8 @@ class ImapFlow extends EventEmitter {
 
         this.maxIdleTime = this.options.maxIdleTime || false;
 
+        this.disableBinary = !!this.options.disableBinary;
+
         this.streamer.on('error', err => {
             if (['Z_BUF_ERROR', 'ECONNRESET', 'EPIPE', 'ETIMEDOUT', 'EHOSTUNREACH'].includes(err.code)) {
                 // just close the connection, usually nothing but noise
@@ -310,7 +314,14 @@ class ImapFlow extends EventEmitter {
         }
 
         if (this.logRaw) {
-            this.log.trace({ src: 'c', msg: 'write to socket', data: chunk.toString('base64'), compress: !!this._deflate, cid: this.id });
+            this.log.trace({
+                src: 'c',
+                msg: 'write to socket',
+                data: chunk.toString('base64'),
+                compress: !!this._deflate,
+                secure: !!this.secureConnection,
+                cid: this.id
+            });
         }
 
         this.writeSocket.write(chunk);
@@ -750,13 +761,14 @@ class ImapFlow extends EventEmitter {
             this.upgrading = true;
             this.socket = tls.connect(opts, () => {
                 clearTimeout(this.upgradeTimeout);
-                if (this.closed) {
+                if (this.isClosed) {
                     // not sure if this is possible?
                     return this.close();
                 }
 
                 this.secureConnection = true;
                 this.upgrading = false;
+                this.streamer.secureConnection = true;
                 this.socket.pipe(this.streamer);
                 this.tls = typeof this.socket.getCipher === 'function' ? this.socket.getCipher() : false;
                 if (this.tls) {
@@ -838,7 +850,7 @@ class ImapFlow extends EventEmitter {
         this.untaggedHandlers.OK = null;
         this.untaggedHandlers.PREAUTH = null;
 
-        if (this.closed) {
+        if (this.isClosed) {
             return;
         }
 
@@ -871,7 +883,7 @@ class ImapFlow extends EventEmitter {
         this.untaggedHandlers.OK = null;
         this.untaggedHandlers.PREAUTH = null;
 
-        if (this.closed) {
+        if (this.isClosed) {
             return;
         }
 
@@ -1281,7 +1293,9 @@ class ImapFlow extends EventEmitter {
             let reject = this.initialReject;
             this.initialResolve = false;
             this.initialReject = false;
-            reject(new Error('Unexpected close'));
+            let err = new Error('Unexpected close');
+            err.code = `ClosedAfterConnect${this.secureConnection ? 'TLS' : 'Text'}`;
+            reject(err);
         }
 
         if (typeof this.preCheck === 'function') {
@@ -1311,10 +1325,10 @@ class ImapFlow extends EventEmitter {
         }
 
         this.state = this.states.LOGOUT;
-        if (this.closed) {
+        if (this.isClosed) {
             return;
         }
-        this.closed = true;
+        this.isClosed = true;
 
         if (this.writeSocket && !this.writeSocket.destroyed) {
             try {

package/lib/tools.js

@@ -307,9 +307,9 @@ module.exports = {
                 }
 
                 default: {
-                    let match = key.match(/(body)\[/i);
+                    let match = key.match(/(body|binary)\[/i);
                     if (match) {
-                        let partKey = key.replace(/^(body)\[|]$/gi, '');
+                        let partKey = key.replace(/^(body|binary)\[|]$/gi, '');
                         partKey = partKey.replace(/\.fields.*$/g, '');
 
                         let value = getBuffer(attribute);

package/lib/types.d.ts

@@ -29,6 +29,7 @@ declare module "imapflow" {
      * @param [options.proxy] - Optional proxy URL. Supports HTTP CONNECT (`http://`, `https://`) and SOCKS (`socks://`, `socks4://`, `socks5://`) proxies
      * @param [options.qresync = false] - If true, then enables QRESYNC support. EXPUNGE notifications will include `uid` property instead of `seq`
      * @param [options.maxIdleTime] - If set, then breaks and restarts IDLE every maxIdleTime ms
+     * @param [options.disableBinary = false] - If true, then ignores the BINARY extension when making FETCH and APPEND calls
      */
     class ImapFlow extends EventEmitter {
         constructor(options: {
@@ -56,6 +57,7 @@ declare module "imapflow" {
             proxy?: boolean;
             qresync?: boolean;
             maxIdleTime?: number;
+            disableBinary?: boolean;
         });
         /**
          * Current module version as a static class property

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "imapflow",
-    "version": "1.0.91",
+    "version": "1.0.92",
     "description": "IMAP Client for Node",
     "main": "./lib/imap-flow.js",
     "scripts": {

package/test/imap-compiler-test.js

@@ -200,8 +200,7 @@ module.exports['IMAP Compiler: hide long strings'] = test =>
                     attributes: [
                         {
                             type: 'String',
-                            value:
-                                'Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere!'
+                            value: 'Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere! Tere tere!'
                         },
                         'Vana kere'
                     ]
@@ -248,6 +247,46 @@ module.exports['IMAP Compiler: LITERAL text'] = test =>
         )
     );
 
+module.exports['IMAP Compiler: LITERAL literal'] = test =>
+    asyncWrapper(test, async test =>
+        test.equal(
+            await compiler({
+                tag: '*',
+                command: 'CMD',
+                attributes: [
+                    // keep indentation
+                    {
+                        type: 'LITERAL',
+                        value: 'Tere\x00 tere!',
+                        isLiteral8: false
+                    },
+                    'Vana kere'
+                ]
+            }),
+            '* CMD {11}\r\nTere\x00 tere! "Vana kere"'
+        )
+    );
+
+module.exports['IMAP Compiler: LITERAL literal8'] = test =>
+    asyncWrapper(test, async test =>
+        test.equal(
+            await compiler({
+                tag: '*',
+                command: 'CMD',
+                attributes: [
+                    // keep indentation
+                    {
+                        type: 'LITERAL',
+                        value: 'Tere\x00 tere!',
+                        isLiteral8: true
+                    },
+                    'Vana kere'
+                ]
+            }),
+            '* CMD ~{11}\r\nTere\x00 tere! "Vana kere"'
+        )
+    );
+
 module.exports['IMAP Compiler: LITERAL array 1'] = test =>
     asyncWrapper(test, async test =>
         test.deepEqual(

package/test/imap-parser-test.js

@@ -309,6 +309,36 @@ module.exports['IMAP Parser: Literals: single literal'] = test =>
         ])
     );
 
+module.exports['IMAP Parser: Literals: literal with NULL'] = test =>
+    asyncWrapper(test, async test =>
+        test.deepEqual((await parser('TAG1 CMD {4}\r\n', { literals: [Buffer.from('ab\x00d')] })).attributes, [
+            {
+                type: 'LITERAL',
+                value: Buffer.from('ab\x00d')
+            }
+        ])
+    );
+
+module.exports['IMAP Parser: Literals: literal8'] = test =>
+    asyncWrapper(test, async test =>
+        test.deepEqual((await parser('TAG1 CMD ~{4}\r\n', { literals: [Buffer.from('ab\x00d')] })).attributes, [
+            {
+                type: 'LITERAL',
+                value: Buffer.from('ab\x00d')
+            }
+        ])
+    );
+
+module.exports['IMAP Parser: Literals: unexpected literal8 prefix'] = test =>
+    asyncWrapper(test, async test => {
+        try {
+            await parser('TAG1 CMD ~\r\n');
+            test.ok(false, 'Must throw');
+        } catch (err) {
+            test.ok(err, 'Error must exist');
+        }
+    });
+
 module.exports['IMAP Parser: Literals: multiple literals'] = test =>
     asyncWrapper(test, async test =>
         test.deepEqual((await parser('TAG1 CMD {4}\r\n {4}\r\n', { literals: [Buffer.from('abcd'), Buffer.from('kere')] })).attributes, [