Haraka 2.8.28 → 3.0.1

package/tests/transaction.js

@@ -17,13 +17,12 @@ exports.transaction = {
     tearDown : _tear_down,
 
     'add_body_filter' (test) {
-        const self = this;
 
         test.expect(3);
 
         this.transaction.add_body_filter('text/plain', (ct, enc, buf) => {
-            // The actual functionality of these filter functions is tested in
-            // mailbody.js.  This just makes sure the plumbing is in place.
+            // The functionality of these filter functions is tested in
+            // haraka-email-message. This just assures the plumbing is in place.
 
             test.ok(ct.indexOf('text/plain') === 0, "correct body part");
             test.ok(/utf-?8/i.test(enc), "correct encoding");
@@ -43,7 +42,7 @@ exports.transaction = {
             "<p>HTML part</p>\n",
             "--abcd--\n",
         ].forEach(line => {
-            self.transaction.add_data(line);
+            this.transaction.add_data(line);
         });
         this.transaction.end_data(() => {
             test.done();
@@ -51,7 +50,6 @@ exports.transaction = {
     },
 
     'regression: attachment_hooks before set_banner/add_body_filter' (test) {
-        const self = this;
 
         test.expect(2);
 
@@ -65,10 +63,10 @@ exports.transaction = {
             "\n",
             "Some text\n",
         ].forEach(line => {
-            self.transaction.add_data(line);
+            this.transaction.add_data(line);
         });
         this.transaction.end_data(() => {
-            self.transaction.message_stream.get_data(body => {
+            this.transaction.message_stream.get_data(body => {
                 test.ok(/banner$/.test(body.toString().trim()), "banner applied");
                 test.done();
             });
@@ -103,7 +101,6 @@ exports.transaction = {
     },
 
     'no munging of bytes if not parsing body' (test) {
-        const self = this;
 
         // Czech panagram "Příliš žluťoučký kůň úpěl ďábelské ódy.\n" in ISO-8859-2 encoding
         const message = Buffer.from([0x50, 0xF8, 0xED, 0x6C, 0x69, 0xB9, 0x20, 0xBE, 0x6C, 0x75, 0xBB, 0x6F, 0x76, 0xE8, 0x6B, 0xFD, 0x20, 0x6B, 0xF9, 0xF2, 0xFA, 0xEC, 0x6C, 0x20, 0xEF, 0xE2, 0x62, 0x65, 0x6C, 0x73, 0x6b, 0xE9, 0x20, 0xF3, 0x64, 0x79, 0x2E, 0x0A]);
@@ -116,10 +113,10 @@ exports.transaction = {
         test.expect(1);
 
         payload.forEach(line => {
-            self.transaction.add_data(line);
+            this.transaction.add_data(line);
         });
         this.transaction.end_data(() => {
-            self.transaction.message_stream.get_data(body => {
+            this.transaction.message_stream.get_data(body => {
                 test.ok(body.includes(message), "message not damaged");
                 test.done();
             });
@@ -186,12 +183,11 @@ exports.base64_handling = {
     tearDown: _tear_down,
 
     'varied-base64-fold-lengths-preserve-data' (test) {
-        const self = this;
 
         const parsed_attachments = {};
-        self.transaction.parse_body = true;
+        this.transaction.parse_body = true;
         //accumulate attachment buffers.
-        self.transaction.attachment_hooks((ct, filename, body, stream) => {
+        this.transaction.attachment_hooks((ct, filename, body, stream) => {
             let attachment = Buffer.alloc(0);
             stream.on('data', data => {
                 attachment = Buffer.concat([attachment, data]);
@@ -202,9 +198,9 @@ exports.base64_handling = {
         });
 
         const specimen_path = path.join(__dirname, 'mail_specimen', 'varied-fold-lengths-preserve-data.txt');
-        write_file_data_to_transaction(self.transaction, specimen_path);
+        write_file_data_to_transaction(this.transaction, specimen_path);
 
-        test.equal(self.transaction.body.children.length, 6);
+        test.equal(this.transaction.body.children.length, 6);
 
         let first_attachment = null;
         for (const i in parsed_attachments) {
@@ -219,13 +215,70 @@ exports.base64_handling = {
     },
 
     'base64-root-html-decodes-correct-number-of-bytes' (test) {
-        const self = this;
 
-        self.transaction.parse_body = true;
+        this.transaction.parse_body = true;
         const specimen_path = path.join(__dirname, 'mail_specimen', 'base64-root-part.txt');
-        write_file_data_to_transaction(self.transaction, specimen_path);
+        write_file_data_to_transaction(this.transaction, specimen_path);
 
-        test.equal(self.transaction.body.bodytext.length, 425);
+        test.equal(this.transaction.body.bodytext.length, 425);
         test.done();
     },
 }
+
+// Test is to ensure boundary marker just after the headers, is in-tact
+// Issue:    "User1990" <--abcd
+// Expected: --abcd
+exports.boundarymarkercorrupt_test = {
+    setUp : _set_up,
+    tearDown: _tear_down,
+
+    // populate the same email data in transaction (self.transaction.add_data()) and
+    // in raw buffer, then compare
+    'fix mime boundary corruption issue' (test) {
+        const self = this;
+        let buffer = '';
+        self.transaction.add_data("Content-Type: multipart/alternative; boundary=abcd\r\n");
+        buffer += "Content-Type: multipart/alternative; boundary=abcd\r\n";
+        self.transaction.add_data('To: "User1_firstname_middlename_lastname" <user1_firstname_middlename_lastname@test.com>,\r\n');
+        buffer += 'To: "User1_firstname_middlename_lastname" <user1_firstname_middlename_lastname@test.com>,\r\n';
+        // make sure we add headers so that it exceeds 64k bytes to expose this issue
+        for (let i=0;i<725;i++){
+            self.transaction.add_data(` "User${i}_firstname_middlename_lastname" <user${i}_firstname_middlename_lastname@test.com>,\r\n`);
+            buffer += ` "User${i}_firstname_middlename_lastname" <user${i}_firstname_middlename_lastname@test.com>,\r\n`
+        }
+        self.transaction.add_data(' "Final User_firstname_middlename_lastname" <final_user_firstname_middlename_lastname@test.com>\r\n');
+        buffer += ' "Final User_firstname_middlename_lastname" <final_user_firstname_middlename_lastname@test.com>\r\n';
+        self.transaction.add_data('Message-ID: <Boundary_Marker_Test>\r\n');
+        buffer += 'Message-ID: <Boundary_Marker_Test>\r\n';
+        self.transaction.add_data('MIME-Version: 1.0\r\n');
+        buffer += 'MIME-Version: 1.0\r\n';
+        self.transaction.add_data('Date: Wed, 1 Jun 2022 16:44:39 +0530 (IST)\r\n');
+        buffer += 'Date: Wed, 1 Jun 2022 16:44:39 +0530 (IST)\r\n';
+        self.transaction.add_data('\r\n');
+        buffer += '\r\n';
+        self.transaction.add_data("--abcd\r\n");
+        buffer += "--abcd\r\n";
+
+        [
+            "Content-Type: text/plain\r\n",
+            "\r\n",
+            "Text part\r\n",
+            "--abcd\r\n",
+            "Content-Type: text/html\r\n",
+            "\r\n",
+            "<p>HTML part</p>\r\n",
+            "--abcd--\r\n",
+        ].forEach(line => {
+            self.transaction.add_data(line);
+            buffer += line;
+        });
+
+        this.transaction.end_data(function () {
+            self.transaction.message_stream.get_data(function (body) {
+                test.ok(body.includes(buffer), "message is damaged");
+                test.done();
+            });
+        });
+    },
+}
+

package/tls_socket.js

@@ -7,7 +7,7 @@
 const cluster   = require('cluster');
 const net       = require('net');
 const path      = require('path');
-const spawn     = require('child_process').spawn;
+const { spawn } = require('child_process');
 const stream    = require('stream');
 const tls       = require('tls');
 const util      = require('util');
@@ -53,61 +53,60 @@ class pluggableStream extends stream.Stream {
     }
 
     attach (socket) {
-        const self = this;
-        self.targetsocket = socket;
-        self.targetsocket.on('data', data => {
-            self.emit('data', data);
+        this.targetsocket = socket;
+        this.targetsocket.on('data', data => {
+            this.emit('data', data);
         });
-        self.targetsocket.on('connect', (a, b) => {
-            self.emit('connect', a, b);
+        this.targetsocket.on('connect', (a, b) => {
+            this.emit('connect', a, b);
         });
-        self.targetsocket.on('secureConnect', (a, b) => {
-            self.emit('secureConnect', a, b);
-            self.emit('secure', a, b);
+        this.targetsocket.on('secureConnect', (a, b) => {
+            this.emit('secureConnect', a, b);
+            this.emit('secure', a, b);
         });
-        self.targetsocket.on('secureConnection', (a, b) => {
+        this.targetsocket.on('secureConnection', (a, b) => {
             // investigate this for removal, see #2743
-            self.emit('secureConnection', a, b);
-            self.emit('secure', a, b);
+            this.emit('secureConnection', a, b);
+            this.emit('secure', a, b);
         });
-        self.targetsocket.on('secure', (a, b) => {
-            self.emit('secureConnection', a, b);
-            self.emit('secure', a, b);
+        this.targetsocket.on('secure', (a, b) => {
+            this.emit('secureConnection', a, b);
+            this.emit('secure', a, b);
         });
-        self.targetsocket.on('end', () => {
-            self.writable = self.targetsocket.writable;
-            self.emit('end');
+        this.targetsocket.on('end', () => {
+            this.writable = this.targetsocket.writable;
+            this.emit('end');
         });
-        self.targetsocket.on('close', had_error => {
-            self.writable = self.targetsocket.writable;
-            self.emit('close', had_error);
+        this.targetsocket.on('close', had_error => {
+            this.writable = this.targetsocket.writable;
+            this.emit('close', had_error);
         });
-        self.targetsocket.on('drain', () => {
-            self.emit('drain');
+        this.targetsocket.on('drain', () => {
+            this.emit('drain');
         });
-        self.targetsocket.once('error', exception => {
-            self.writable = self.targetsocket.writable;
+        this.targetsocket.once('error', exception => {
+            this.writable = this.targetsocket.writable;
             exception.source = 'tls';
-            self.emit('error', exception);
+            this.emit('error', exception);
         });
-        self.targetsocket.on('timeout', () => {
-            self.emit('timeout');
+        this.targetsocket.on('timeout', () => {
+            this.emit('timeout');
         });
-        if (self.targetsocket.remotePort) {
-            self.remotePort = self.targetsocket.remotePort;
+        if (this.targetsocket.remotePort) {
+            this.remotePort = this.targetsocket.remotePort;
         }
-        if (self.targetsocket.remoteAddress) {
-            self.remoteAddress = self.targetsocket.remoteAddress;
+        if (this.targetsocket.remoteAddress) {
+            this.remoteAddress = this.targetsocket.remoteAddress;
         }
-        if (self.targetsocket.localPort) {
-            self.localPort = self.targetsocket.localPort;
+        if (this.targetsocket.localPort) {
+            this.localPort = this.targetsocket.localPort;
         }
-        if (self.targetsocket.localAddress) {
-            self.localAddress = self.targetsocket.localAddress;
+        if (this.targetsocket.localAddress) {
+            this.localAddress = this.targetsocket.localAddress;
         }
     }
     clean (data) {
-        if (this.targetsocket && this.targetsocket.removeAllListeners) {
+        if (this.targetsocket?.removeAllListeners) {
             [   'data', 'secure', 'secureConnect', 'secureConnection',
                 'end', 'close', 'error', 'drain'
             ].forEach((name) => {
@@ -224,7 +223,6 @@ exports.parse_x509 = string => {
 }
 
 exports.load_tls_ini = (opts) => {
-    const tlss = this;
 
     log.loginfo(`loading tls.ini`); // from ${this.config.root_path}`);
 
@@ -247,7 +245,7 @@ exports.load_tls_ini = (opts) => {
             '-main.mutual_tls',
         ]
     }, () => {
-        tlss.load_tls_ini();
+        this.load_tls_ini();
     });
 
     if (cfg.no_tls_hosts === undefined) cfg.no_tls_hosts = {};
@@ -279,11 +277,11 @@ exports.load_tls_ini = (opts) => {
 
     if (!Array.isArray(cfg.main.no_starttls_ports)) cfg.main.no_starttls_ports = [];
 
-    tlss.cfg = cfg;
+    this.cfg = cfg;
 
     if (!opts || opts.role === 'server') {
-        tlss.applySocketOpts('*');
-        tlss.load_default_opts();
+        this.applySocketOpts('*');
+        this.load_default_opts();
     }
 
     return cfg;
@@ -295,7 +293,6 @@ exports.saveOpt = (name, opt, val) => {
 }
 
 exports.applySocketOpts = name => {
-    const tlss = this;
 
     if (!certsByHost[name]) certsByHost[name] = {};
 
@@ -318,56 +315,55 @@ exports.applySocketOpts = name => {
 
     for (const opt of allOpts) {
 
-        if (tlss.cfg[name] && tlss.cfg[name][opt] !== undefined) {
+        if (this.cfg[name] && this.cfg[name][opt] !== undefined) {
             // if the setting exists in tls.ini [name]
-            tlss.saveOpt(name, opt, tlss.cfg[name][opt]);
+            this.saveOpt(name, opt, this.cfg[name][opt]);
             continue;
         }
 
-        if (tlss.cfg.main[opt] !== undefined) {
+        if (this.cfg.main[opt] !== undefined) {
             // if the setting exists in tls.ini [main]
             // then save it to the certsByHost options
-            tlss.saveOpt(name, opt, tlss.cfg.main[opt]);
+            this.saveOpt(name, opt, this.cfg.main[opt]);
             continue;
         }
 
         // defaults
         switch (opt) {
             case 'sessionIdContext':
-                tlss.saveOpt(name, opt, 'haraka');
+                this.saveOpt(name, opt, 'haraka');
                 break;
             case 'isServer':
-                tlss.saveOpt(name, opt, true);
+                this.saveOpt(name, opt, true);
                 break;
             case 'key':
-                tlss.saveOpt(name, opt, 'tls_key.pem');
+                this.saveOpt(name, opt, 'tls_key.pem');
                 break;
             case 'cert':
-                tlss.saveOpt(name, opt, 'tls_cert.pem');
+                this.saveOpt(name, opt, 'tls_cert.pem');
                 break;
             case 'dhparam':
-                tlss.saveOpt(name, opt, 'dhparams.pem');
+                this.saveOpt(name, opt, 'dhparams.pem');
                 break;
             case 'SNICallback':
-                tlss.saveOpt(name, opt, SNICallback);
+                this.saveOpt(name, opt, SNICallback);
                 break;
         }
     }
 }
 
 exports.load_default_opts = () => {
-    const tlss = this;
 
     const cfg = certsByHost['*'];
 
     if (cfg.dhparam && typeof cfg.dhparam === 'string') {
         log.logdebug(`loading dhparams from ${cfg.dhparam}`);
-        tlss.saveOpt('*', 'dhparam', tlss.config.get(cfg.dhparam, 'binary'));
+        this.saveOpt('*', 'dhparam', this.config.get(cfg.dhparam, 'binary'));
     }
 
     if (cfg.ca && typeof cfg.ca === 'string') {
         log.loginfo(`loading CA certs from ${cfg.ca}`);
-        tlss.saveOpt('*', 'ca', tlss.config.get(cfg.ca, 'binary'));
+        this.saveOpt('*', 'ca', this.config.get(cfg.ca, 'binary'));
     }
 
     // make non-array key/cert option into Arrays with one entry
@@ -383,32 +379,32 @@ exports.load_default_opts = () => {
         // turn key/cert file names into actual key/cert binary data
         const asArray = cfg.key.map(keyFileName => {
             if (!keyFileName) return;
-            const key = tlss.config.get(keyFileName, 'binary');
+            const key = this.config.get(keyFileName, 'binary');
             if (!key) {
-                log.logerror(`tls key ${path.join(tlss.config.root_path, keyFileName)} could not be loaded.`);
+                log.logerror(`tls key ${path.join(this.config.root_path, keyFileName)} could not be loaded.`);
             }
             return key;
         })
-        tlss.saveOpt('*', 'key', asArray);
+        this.saveOpt('*', 'key', asArray);
     }
 
     if (typeof cfg.cert[0] === 'string') {
         const asArray = cfg.cert.map(certFileName => {
             if (!certFileName) return;
-            const cert = tlss.config.get(certFileName, 'binary');
+            const cert = this.config.get(certFileName, 'binary');
             if (!cert) {
-                log.logerror(`tls cert ${path.join(tlss.config.root_path, certFileName)} could not be loaded.`);
+                log.logerror(`tls cert ${path.join(this.config.root_path, certFileName)} could not be loaded.`);
             }
             return cert;
         })
-        tlss.saveOpt('*', 'cert', asArray);
+        this.saveOpt('*', 'cert', asArray);
     }
 
     if (cfg.cert[0] && cfg.key[0]) {
-        tlss.tls_valid = true;
+        this.tls_valid = true;
 
         // now that all opts are applied, generate TLS context
-        tlss.ensureDhparams(() => {
+        this.ensureDhparams(() => {
             ctxByHost['*'] = tls.createSecureContext(cfg);
         })
     }
@@ -423,9 +419,8 @@ function SNICallback (servername, sniDone) {
 }
 
 exports.get_certs_dir = (tlsDir, done) => {
-    const tlss = this;
 
-    tlss.config.getDir(tlsDir, {}, (iterErr, files) => {
+    this.config.getDir(tlsDir, {}, (iterErr, files) => {
         if (iterErr) return done(iterErr);
 
         async.map(files, (file, iter_done) => {
@@ -452,7 +447,7 @@ exports.get_certs_dir = (tlsDir, done) => {
                     log.logerror(e);
                 }
 
-                const expire = tlss.parse_x509_expire(file, as_str);
+                const expire = this.parse_x509_expire(file, as_str);
                 if (expire && expire < new Date()) {
                     log.logerror(`${file.path} expired on ${expire}`);
                 }
@@ -462,7 +457,7 @@ exports.get_certs_dir = (tlsDir, done) => {
                     file: path.basename(file.path),
                     key: parsed.key,
                     cert: parsed.cert,
-                    names: tlss.parse_x509_names(as_str),
+                    names: this.parse_x509_names(as_str),
                     expires: expire,
                 })
             })
@@ -486,13 +481,13 @@ exports.get_certs_dir = (tlsDir, done) => {
 
                 // log.logdebug(cert);  // DANGER: Also logs private key!
                 cert.names.forEach(name => {
-                    tlss.applySocketOpts(name);
+                    this.applySocketOpts(name);
 
-                    tlss.saveOpt(name, 'cert', cert.cert);
-                    tlss.saveOpt(name, 'key', cert.key);
+                    this.saveOpt(name, 'cert', cert.cert);
+                    this.saveOpt(name, 'key', cert.key);
                     if (certsByHost['*'] !== undefined && certsByHost['*'].dhparam) {
                         // copy in dhparam from default '*' TLS config
-                        tlss.saveOpt(name, 'dhparam', certsByHost['*'].dhparam);
+                        this.saveOpt(name, 'dhparam', certsByHost['*'].dhparam);
                     }
 
                     // now that all opts are applied, generate TLS context
@@ -507,12 +502,11 @@ exports.get_certs_dir = (tlsDir, done) => {
 }
 
 exports.getSocketOpts = (name, done) => {
-    const tlss = this;
 
     // startup time, load the config/tls dir
-    if (!certsByHost['*']) tlss.load_tls_ini();
+    if (!certsByHost['*']) this.load_tls_ini();
 
-    tlss.get_certs_dir('tls', () => {
+    this.get_certs_dir('tls', () => {
         if (certsByHost[name]) {
             // log.logdebug(certsByHost[name]);
             return done(certsByHost[name]);
@@ -538,7 +532,6 @@ function pipe (cleartext, socket) {
 }
 
 exports.ensureDhparams = done => {
-    const tlss = this;
 
     // empty/missing dhparams file
     if (certsByHost['*'].dhparam) {
@@ -547,7 +540,7 @@ exports.ensureDhparams = done => {
 
     if (cluster.isWorker) return; // only once, on the master process
 
-    const filePath = tlss.cfg.main.dhparam || 'dhparams.pem';
+    const filePath = this.cfg.main.dhparam || 'dhparams.pem';
     const fpResolved = path.resolve(exports.config.root_path, filePath);
 
     log.loginfo(`Generating a 2048 bit dhparams file at ${fpResolved}`);
@@ -568,9 +561,9 @@ exports.ensureDhparams = done => {
         }
 
         log.loginfo(`Saved to ${fpResolved}`);
-        const content = tlss.config.get(filePath, 'binary');
+        const content = this.config.get(filePath, 'binary');
 
-        tlss.saveOpt('*', 'dhparam', content);
+        this.saveOpt('*', 'dhparam', content);
         done(null, certsByHost['*'].dhparam);
     });
 }
@@ -636,9 +629,7 @@ exports.get_rejectUnauthorized = (rejectUnauthorized, port, port_list) => {
 
     if (rejectUnauthorized) return true;
 
-    if (port_list.includes(port)) return true;
-
-    return false;
+    return !!(port_list.includes(port));
 }
 
 function createServer (cb) {
@@ -704,11 +695,10 @@ function getCertFor (host) {
     return certsByHost['*'];  // the default TLS cert
 }
 
-function connect (port, host, cb) {
+function connect (port, host) {
     let conn_options = {};
     if (typeof port === 'object') {
         conn_options = port;
-        cb = host;
     }
     else {
         conn_options.port = port;

package/transaction.js

@@ -9,9 +9,7 @@ const Notes  = require('haraka-notes');
 const utils  = require('haraka-utils');
 
 // Haraka modules
-const Header = require('./mailheader').Header;
-const body   = require('./mailbody');
-const MessageStream = require('./messagestream');
+const message = require('haraka-email-message')
 
 class Transaction {
     constructor (uuid, cfg) {
@@ -31,8 +29,8 @@ class Transaction {
         this.parse_body = false;
         this.notes = new Notes();
         this.notes.skip_plugins = [];
-        this.header = new Header();
-        this.message_stream = new MessageStream(this.cfg, this.uuid, this.header.header_list);
+        this.header = new message.Header();
+        this.message_stream = new message.stream(this.cfg, this.uuid, this.header.header_list);
         this.discard_data = false;
         this.resetting = false;
         this.rcpt_count = {
@@ -50,7 +48,7 @@ class Transaction {
     ensure_body () {
         if (this.body) return;
 
-        this.body = new body.Body(this.header);
+        this.body = new message.Body(this.header);
         this.body.on('mime_boundary', m => this.incr_mime_count());
         this.attachment_start_hooks.forEach(h => {
             this.body.on('attachment_start', h);
@@ -62,7 +60,7 @@ class Transaction {
 
         for (const o of this.body_filters) {
             this.body.add_filter((ct, enc, buf) => {
-                const re_match = (util.isRegExp(o.ct_match) && o.ct_match.test(ct.toLowerCase()));
+                const re_match = (util.types.isRegExp(o.ct_match) && o.ct_match.test(ct.toLowerCase()));
                 const ct_begins = ct.toLowerCase().indexOf(String(o.ct_match).toLowerCase()) === 0;
                 if (re_match || ct_begins) return o.filter(ct, enc, buf);
             })
@@ -186,8 +184,8 @@ class Transaction {
             this.header_pos = header_pos;
             if (this.parse_body) {
                 this.ensure_body();
-                for (let j = 0; j < body_lines.length; j++) {
-                    this.body.parse_more(body_lines[j]);
+                for (const bodyLine of body_lines) {
+                    this.body.parse_more(bodyLine);
                 }
             }
         }