Haraka 3.0.3 → 3.0.4

package/docs/Transaction.md

@@ -1,19 +1,16 @@
-Transaction Object
-==================
+# Transaction Object
 
 An SMTP transaction is valid from MAIL FROM time until RSET or "final-dot".
 
-API
----
+## API
 
 * transaction.uuid
 
-A unique UUID for this transaction. Is equal to the connection.uuid + '.N'
-where N increments for each transaction on this connection.
+A unique UUID for this transaction. Is equal to the connection.uuid + '.N' where N increments for each transaction on this connection.
 
 * transaction.mail\_from
 
-The value of the MAIL FROM command as an `Address`[1] object.
+The value of the MAIL FROM command is an `Address`[1] object.
 
 * transaction.rcpt\_to
 
@@ -25,13 +22,13 @@ A node.js Readable Stream object for the message.
 
 You use it like this:
 
-    transaction.message_stream.pipe(WritableStream, options)
+```js
+transaction.message_stream.pipe(WritableStream, options)
+```
 
-Where WritableStream is a node.js Writable Stream object such as a
-net.socket, fs.writableStream, process.stdout/stderr or custom stream.
+Where WritableStream is a node.js Writable Stream object such as a net.socket, fs.writableStream, process.stdout/stderr or custom stream.
 
-The options argument should be an object that overrides the following
-properties:
+The options argument should be an object that overrides the following properties:
 
     * line_endings (default: "\r\n")
     * dot_stuffing (default: false)
@@ -42,7 +39,9 @@ properties:
 
 e.g.
 
-    transaction.message_stream.pipe(socket, { dot_stuffing: true, ending_dot: true });
+```js
+transaction.message_stream.pipe(socket, { dot_stuffing: true, ending_dot: true });
+```
 
 * transaction.data\_bytes
 
@@ -50,8 +49,7 @@ The number of bytes in the email after DATA.
 
 * transaction.add\_data(line)
 
-Adds a line of data to the email. Note this is RAW email - it isn't useful
-for adding banners to the email.
+Adds a line of data to the email. Note this is RAW email - it isn't useful for adding banners to the email.
 
 * transaction.notes
 
@@ -59,8 +57,7 @@ A safe place to store transaction specific values. See also [haraka-results](htt
 
 * transaction.add\_leading\_header(key, value)
 
-Adds a header to the top of the header list.  This should only be used in
-very specific cases.  Most people will want to use `add_header()` instead.
+Adds a header to the top of the header list.  This should only be used in very specific cases. Most cases will use `add_header()` instead.
 
 * transaction.add\_header(key, value)
 
@@ -76,8 +73,7 @@ The header of the email. See `Header Object`.
 
 * transaction.parse\_body = true|false [default: false]
 
-Set to `true` to enable parsing of the mail body. Make sure you set this in
-hook\_data or before.
+Set to `true` to enable parsing of the mail body. Make sure you set this in hook\_data or before. Storing a transaction hook (with transaction.attachment\_hooks) will set this to true.
 
 * transaction.body
 
@@ -85,81 +81,62 @@ The body of the email if you set `parse_body` above. See `Body Object`.
 
 * transaction.attachment\_hooks(start)
 
-Sets a callback for when we see an attachment if `parse_body` has been set.
-
-The `start` event will receive `(content_type, filename, body, stream)` as
-parameters.
-
-The stream is a `ReadableStream` - see http://nodejs.org/api/stream.html for
-details on how this works.
-
-If you set stream.connection then the stream will apply backpressure to the
-connection, allowing you to process attachments before the connection has
-ended. Here is an example which stores attachments in temporary files using
-the `tmp` library from npm and tells us the size of the file:
-
-    exports.hook_data = function (next, connection) {
-        // enable mail body parsing
-        connection.transaction.parse_body = true;
-        connection.transaction.attachment_hooks(
-            function (ct, fn, body, stream) {
-                start_att(connection, ct, fn, body, stream)
-            }
-        );
-        next();
-    }
-
-    function start_att (connection, ct, fn, body, stream) {
-        connection.loginfo("Got attachment: " + ct + ", " + fn + " for user id: " + connection.transaction.notes.hubdoc_user.email);
-        connection.transaction.notes.attachment_count++;
-
-        stream.connection = connection; // Allow backpressure
-        stream.pause();
-
-        var tmp = require('tmp');
-
-        tmp.file(function (err, path, fd) {
-            connection.loginfo("Got tempfile: " + path + " (" + fd + ")");
-            var ws = fs.createWriteStream(path);
-            stream.pipe(ws);
-            stream.resume();
-            ws.on('close', function () {
-                connection.loginfo("End of stream reached");
-                fs.fstat(fd, function (err, stats) {
-                    connection.loginfo("Got data of length: " + stats.size);
-                    // Close the tmp file descriptor
-                    fs.close(fd, function(){});
-                });
+Sets a callback for when we see an attachment.
+
+The `start` event will receive `(content_type, filename, body, stream)` as parameters.
+
+The stream is a [ReadableStream](http://nodejs.org/api/stream.html)
+
+If you set stream.connection then the stream will apply backpressure to the connection, allowing you to process attachments before the connection has ended. Here is an example which stores attachments in temporary files using the `tmp` library from npm and tells us the size of the file:
+
+```js
+exports.hook_data = function (next, connection) {
+    // enable mail body parsing
+    connection.transaction.attachment_hooks(
+        function (ct, fn, body, stream) {
+            start_att(connection, ct, fn, body, stream)
+        }
+    );
+    next();
+}
+
+function start_att (connection, ct, fn, body, stream) {
+    connection.loginfo(`Got attachment: ${ct}, ${fn} for user id: ${connection.transaction.notes.hubdoc_user.email}`)
+    connection.transaction.notes.attachment_count++
+
+    stream.connection = connection; // Allow backpressure
+    stream.pause();
+
+    require('tmp').file((err, path, fd) => {
+        connection.loginfo(`Got tempfile: ${path} (${fd})`)
+        const ws = fs.createWriteStream(path)
+        stream.pipe(ws);
+        stream.resume();
+        ws.on('close', () => {
+            connection.loginfo("End of stream reached");
+            fs.fstat(fd, (err, stats) => {
+                connection.loginfo(`Got data of length: ${stats.size}`);
+                fs.close(fd, () => {}); // Close the tmp file descriptor
             });
         });
-    }
+    });
+}
+```
 
 * transaction.discard\_data = true|false [default: false]
 
-Set this flag to true to discard all data as it arrives and not store in
-memory or on disk (in the message\_stream property). You can still access
-the attachments and body if you set parse\_body to true. This is useful
-for systems which do not need the full email, just the attachments or
-mail text.
+Set this flag to true to discard all data as it arrives and not store in memory or on disk (in the message\_stream property). You can still access the attachments and body if you set parse\_body to true. This is useful for systems which do not need the full email, just the attachments or mail text.
 
 * transaction.set\_banner(text, html)
 
-Sets a banner to be added to the end of the email. If the html part is not
-given (optional) then the text part will have each line ending replaced with
-`<br/>` when being inserted into HTML parts.
+Sets a banner to be added to the end of the email. If the html part is not given (optional) then the text part will have each line ending replaced with `<br/>` when being inserted into HTML parts.
 
 * transaction.add\_body\_filter(ct_match, filter)
 
-Adds a filter to be applied to body parts in the email.  ct\_match should be a
-regular expression to match against the full content-type line, or a string to
-match at the start, e.g. `/^text\/html/` or `'text/plain'`.  filter will be
-called when each body part matching ct_match is complete.  It receives three
-parameters, the content-type line, the encoding name, and a buffer with the
-full body part.  It should return a buffer with the desired contents of the
-body in the same encoding.
+Adds a filter to be applied to body parts in the email. ct\_match should be a regular expression to match against the full content-type line, or a string to match at the start, e.g. `/^text\/html/` or `'text/plain'`. filter will be called when each body part matching ct_match is complete.  It receives three parameters: the content-type line, the encoding name, and a buffer with the full body part. It should return a buffer with the desired contents of the body in the same encoding.
 
 * transaction.results
 
-Store results of processing in a structured format. See [haraka-results](https://github.com/haraka/haraka-results)
+Store [results](https://github.com/haraka/haraka-results) of processing in a structured format.
 
-[1]: `Address` objects are address-rfc2821 objects. See https://github.com/haraka/node-address-rfc2821
+[1]: `Address` objects are [address-rfc2821](https://github.com/haraka/node-address-rfc2821) objects.

package/docs/plugins/queue/smtp_proxy.md

@@ -1,16 +1,11 @@
 # queue/smtp\_proxy
 ================
 
-This plugin delivers to another mail server. This is a common setup when you
-want to have a mail server with a solid pedigree of outbound delivery to
-other hosts, and inbound delivery to users.
-
-In comparison to `queue/smtp_forward`, this plugin makes a connection at
-MAIL FROM time to the ongoing SMTP server. This can be a benefit in that
-you get any SMTP-time filtering that the ongoing server provides, in
-particular one important facility to some setups is recipient filtering.
-However be aware that other than connect and HELO-time filtering, you will
-have as many connections to your ongoing SMTP server as you have to Haraka.
+This plugin delivers to another mail server. This is a common setup when you want to have a mail server with a solid pedigree of outbound delivery to other hosts, and inbound delivery to users.
+
+In comparison to `queue/smtp_forward`, this plugin makes a connection at MAIL FROM time to the ongoing SMTP server. This can be a benefit in that you get any SMTP-time filtering that the ongoing server provides, in particular one important facility to some setups is recipient filtering.
+
+Be aware that other than connect and HELO-time filtering, you will have as many connections to your ongoing SMTP server as you have to Haraka.
 
 ## Configuration
 -------------

package/docs/plugins/tls.md

@@ -4,9 +4,11 @@ This plugin enables the use of TLS (via `STARTTLS`) in Haraka.
 
 For this plugin to work you must have SSL certificates installed correctly.
 
+Haraka has [SNI](https://en.wikipedia.org/wiki/Server_Name_Indication) support. When the remote MUA/MTA presents a servername during the TLS handshake and a TLS certificate with that Common Name matches, that certificate will be presented. If no match is found, the default certificate (see Certificate Files) is presented.
+
 ## Certificate Files
 
-Defaults are shown and can be overridden in `config/tls.ini`.
+Defaults settings are shown and can be overridden in `config/tls.ini`.
 
 ```ini
 key=tls_key.pem
@@ -16,20 +18,39 @@ dhparam=dhparams.pem
 
 ## Certificate Directory
 
-If the directory `config/tls` exists, each file within the directory is expected to be a PEM encoded TLS bundle. Generate the PEM bundles in The Usual Way[TM] by concatenating the key, certificate, and CA/chain certs in that order. Example:
+If the directory `config/tls` exists, files within the directory are PEM encoded TLS files in one of two formats: bundles or Wild Wild West.
+
+### Certificate bundles
+
+Generate PEM bundles in The Usual Way[TM] by concatenating the key, certificate, and CA/chain certs in that order. Example:
 
 ```sh
-cat example.com.key example.com.crt ca.crt > config/tls/example.com.pem
+cat example.com.key example.com.crt ca-int.crt > haraka/config/tls/example.com.pem
 ```
 
-An example [acme.sh](https://acme.sh) deployment [script](https://github.com/msimerson/Mail-Toaster-6/blob/master/provision/letsencrypt.sh) demonstrates how to install [Let's Encrypt](https://letsencrypt.org) certificates to the Haraka `config/tls`directory.
+An example [acme.sh](https://acme.sh) deployment [script](https://github.com/msimerson/Mail-Toaster-6/blob/master/provision/letsencrypt.sh) installs [Let's Encrypt](https://letsencrypt.org) certificate bundles to the Haraka `config/tls`directory.
+
+### Wild Wild West
+
+PEM encoded TLS certificates and keys can be stored in files in `config/tls`. The certificate loader is recursive, so TLS files can be in subdirs like `config/tls/mx1.example.com`. The certificate names are parsed from the 1st cert in each file and indexed by the certs Common Name(s). Subject Alternate Names are supported. The file name containing the certificates does *not* matter. Additional certificates within each file are presumed to be CA chain (intermediate) certificates.
+
+If the TLS key is stored in the same file as the matching certificate, then the name of the file does not matter. If the TLS key is alone in a file, the file MUST be named with the keys Common Name. The file extension does not matter, `.pem` and `.key` are common. If the key is used for multiple CNs, the key must be stored in a file name matching each CN. Examples of working TLS key/cert file pairs for the Common Name mx1.example.com:
 
-Haraka has [SNI](https://en.wikipedia.org/wiki/Server_Name_Indication) support. When the remote MUA/MTA presents a servername during the TLS handshake and a TLS certificate with that Common Name matches, that certificate will be presented. If no match is found, the default certificate (see Certificate Files above) is presented.
+1. certificate bundle (see above), key & cert in same file
+    - config/tls/mx1.example.com.pem (recommended)
+    - config/tls/any-unique-name.pem (CN is extracted from 1st cert)
+2. files in TLS dir
+    - config/tls/mx1.example.com.crt
+    - config/tls/mx1.example.com.key
+3. files in subdir
+    - config/tls/example.com/mx1.cert
+    - config/tls/example.com/mx1.example.com.key
+4. wildcard bundle on Windows platform (* is not allowed in file names)
+    - config/tls/_.example.com.pem
 
 ## Purchased Certificate
 
-If you have a purchased certificate, append any intermediate/chained/ca-cert
-files to the certificate in this order:
+For purchased certificate, append any intermediate/chained/ca-cert files to the certificate in this order:
 
 1. The CA signed SSL cert
 2. Any intermediate certificates
@@ -39,8 +60,7 @@ See also [Setting Up TLS](https://github.com/haraka/Haraka/wiki/Setting-up-TLS-w
 
 ## Self Issued (unsigned) Certificate
 
-Create a certificate and key file in the config directory with the following
-command:
+Create a certificate and key file in the config directory with the following command:
 
     openssl req -x509 -nodes -days 2190 -newkey rsa:2048 \
             -keyout config/tls_key.pem -out config/tls_cert.pem

package/endpoint.js

@@ -1,7 +1,7 @@
 'use strict';
 // Socket address parser/formatter and server binding helper
 
-const fs = require('fs');
+const fs = require('node:fs/promises');
 const sockaddr = require('sockaddr');
 
 module.exports = function endpoint (addr, defaultPort) {
@@ -46,21 +46,24 @@ class Endpoint {
     }
 
     // Make server listen on this endpoint, w/optional options
-    bind (server, opts) {
-        let done;
-        opts = Object.assign({}, opts || {});
+    async bind (server, opts) {
+        opts = {...opts};
+
+        const mode = this.mode ? parseInt(this.mode, 8) : false;
         if (this.path) {
-            const path = opts.path = this.path;
-            const mode = this.mode ? parseInt(this.mode, 8) : false;
-            if (mode) {
-                done = () => fs.chmodSync(path, mode);
-            }
-            if (fs.existsSync(path)) fs.unlinkSync(path);
-        }
-        else {
+            opts.path = this.path;
+            await fs.rm(this.path, { force: true }); // errors are ignored when force is true
+        } else {
             opts.host = this.host;
             opts.port = this.port;
         }
-        server.listen(opts, done);
+        
+        return new Promise((resolve, reject) => {
+            server.listen(opts, async (err) => {
+                if(err) return reject(err);
+                if (mode) await fs.chmod(opts.path, mode);
+                resolve()
+            });
+        });
     }
 }

package/haraka.js

@@ -20,20 +20,18 @@ catch (e) {
     require('module')._initPaths(); // Horrible hack
 }
 
-const fs = require('fs');
+const utils = require('haraka-utils');
 const logger = require('./logger');
 const server = require('./server');
 
-exports.version = JSON.parse(
-    fs.readFileSync(path.join(__dirname, './package.json'), 'utf8')
-).version;
+exports.version = utils.getVersion(__dirname)
 
 process.on('uncaughtException', err => {
     if (err.stack) {
-        err.stack.split("\n").forEach(line => logger.logcrit(line));
+        err.stack.split("\n").forEach(line => logger.crit(line));
     }
     else {
-        logger.logcrit(`Caught exception: ${JSON.stringify(err)}`);
+        logger.crit(`Caught exception: ${JSON.stringify(err)}`);
     }
     logger.dump_and_exit(1);
 });
@@ -41,18 +39,16 @@ process.on('uncaughtException', err => {
 let shutting_down = false;
 const signals = ['SIGINT'];
 
-if (process.pid === 1) {
-    signals.push('SIGTERM')
-}
+if (process.pid === 1) signals.push('SIGTERM')
 
-signals.forEach((sig) => {
+for (const sig of signals) {
     process.on(sig, () => {
         if (shutting_down) return process.exit(1);
         shutting_down = true;
         const [, filename] = process.argv;
         process.title = path.basename(filename, '.js');
 
-        logger.lognotice(`${sig} received`);
+        logger.notice(`${sig} received`);
         logger.dump_and_exit(() => {
             if (server.cluster?.isMaster) {
                 server.performShutdown();
@@ -62,10 +58,10 @@ signals.forEach((sig) => {
             }
         });
     });
-});
+}
 
 process.on('SIGHUP', () => {
-    logger.lognotice('Flushing the temp fail queue');
+    logger.notice('Flushing the temp fail queue');
     server.flushQueue();
 });
 
@@ -74,7 +70,7 @@ process.on('exit', code => {
     const [, filename] = process.argv;
     process.title = path.basename(filename, '.js');
 
-    logger.lognotice('Shutting down');
+    logger.notice('Shutting down');
     logger.dump_logs();
 });
 

package/host_pool.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const net    = require('net');
+const net    = require('node:net');
 const utils  = require('haraka-utils');
 
 /* HostPool:
@@ -61,7 +61,7 @@ class HostPool {
         self.dead_hosts[key] = true;
 
         function cb_if_still_dead () {
-            logger.logwarn(`${host} ${key} is still dead, will retry in ${self.retry_secs} secs`);
+            logger.warn(`${host} ${key} is still dead, will retry in ${self.retry_secs} secs`);
             self.dead_hosts[key] = true;
             // console.log(1);
             setTimeout(() => {
@@ -71,7 +71,7 @@ class HostPool {
 
         function cb_if_alive () {
             // console.log(2);
-            logger.loginfo(`${host} ${key} is back! adding back into pool`);
+            logger.info(`${host} ${key} is back! adding back into pool`);
             delete self.dead_hosts[key];
         }
 
@@ -90,7 +90,7 @@ class HostPool {
     probe_dead_host (
         host, port, cb_if_still_dead, cb_if_alive
     ){
-        logger.loginfo(`probing dead host ${host}:${port}`);
+        logger.info(`probing dead host ${host}:${port}`);
 
         const connect_timeout_ms = 200; // keep it snappy
         let s;
@@ -162,7 +162,7 @@ class HostPool {
             return host;
         }
         else {
-            logger.logwarn(
+            logger.warn(
                 `no working hosts found, retrying a dead one, config (probably from smtp_forward.forwarding_host_pool) is '${this.hostports_str}'`);
             this.last_i = first_i;
             return this.hosts[first_i];

package/line_socket.js

@@ -1,7 +1,7 @@
 'use strict';
 // A subclass of Socket which reads data by line
 
-const net   = require('net');
+const net   = require('node:net');
 const utils = require('haraka-utils');
 
 const tls_socket = require('./tls_socket');
@@ -37,17 +37,16 @@ function setup_line_processor (socket) {
 exports.Socket = Socket;
 
 // New interface - uses TLS
-exports.connect = (port, host, cb) => {
+exports.connect = (port, host) => {
     let options = {};
     if (typeof port === 'object') {
         options = port;
-        cb = host;
     }
     else {
         options.port = port;
         options.host = host;
     }
-    const sock = tls_socket.connect(options, cb);
+    const sock = tls_socket.connect(options);
     setup_line_processor(sock);
     return sock;
 }

package/logger.js

@@ -1,15 +1,13 @@
 'use strict';
 // Log class
 
-const util      = require('util');
-const tty       = require('tty');
+const util      = require('node:util');
+const tty       = require('node:tty');
 
 const config    = require('haraka-config');
 const constants = require('haraka-constants');
 
 let plugins;
-let connection;
-let outbound;
 
 const regex = /(^$|[ ="\\])/;
 const escape_replace_regex = /["\\]/g;
@@ -48,6 +46,7 @@ logger.levels = {
     ALERT:    1,
     EMERG:    0,
 }
+const level_names = Object.keys(logger.levels)
 
 for (const le in logger.levels) {
     logger.levels[`LOG${le}`] = logger.levels[le];
@@ -64,6 +63,7 @@ logger.loglevel      = logger.levels.WARN;
 logger.format        = logger.formats.DEFAULT;
 logger.timestamps    = false;
 logger.deferred_logs = [];
+logger.name          = 'logger'
 
 logger.colors = {
     "DATA" : "green",
@@ -235,14 +235,16 @@ logger._init_timestamps = function () {
 
 logger._init();
 
-logger.log_if_level = (level, key, plugin) => function () {
-    if (logger.loglevel < logger[key]) { return; }
+logger.log_if_level = (level, key, origin) => function () {
+    if (logger.loglevel < logger[key]) return;
+
     let logobj = {
         level,
         uuid: '-',
-        origin: (plugin || 'core'),
+        origin: (origin || 'core'),
         message: ''
     };
+
     for (const data of arguments) {
         if (typeof data !== 'object') {
             logobj.message += (data);
@@ -251,24 +253,17 @@ logger.log_if_level = (level, key, plugin) => function () {
         if (!data) continue;
 
         // if the object is a connection, add the connection id
-        if (data instanceof connection.Connection) {
+        if (data.constructor?.name === 'Connection') {
             logobj.uuid = data.uuid;
             if (data.tran_count > 0) logobj.uuid += `.${data.tran_count}`;
         }
         else if (data instanceof plugins.Plugin) {
             logobj.origin = data.name;
         }
-        else if (data.name) {
+        else if (Object.hasOwn(data, 'name')) { // outbound
             logobj.origin = data.name;
-        }
-        else if (data instanceof outbound.HMailItem) {
-            logobj.origin = 'outbound';
-            if (data.todo) {
-                if (data.todo.uuid) logobj.uuid = data.todo.uuid;
-                if (data.todo.client_uuid) {    // dirty hack
-                    logobj.origin = `outbound] [${data.todo.client_uuid}`;
-                }
-            }
+            if (Object.hasOwn(data, 'uuid')) logobj.uuid = data.uuid;
+            if (data.todo?.uuid) logobj.uuid = data.todo.uuid; // outbound/hmail
         }
         else if (
             logger.format === logger.formats.LOGFMT && data.constructor === Object) {
@@ -278,7 +273,7 @@ logger.log_if_level = (level, key, plugin) => function () {
             logger.format === logger.formats.JSON && data.constructor === Object) {
             logobj = Object.assign(logobj, data);
         }
-        else if (typeof data === 'object' && Object.prototype.hasOwnProperty.call(data, "uuid")) {
+        else if (Object.hasOwn(data, 'uuid')) { // outbound/client_pool
             logobj.uuid = data.uuid;
         }
         else if (data.constructor === Object) {
@@ -313,14 +308,37 @@ logger.log_if_level = (level, key, plugin) => function () {
     return true;
 }
 
-logger.add_log_methods = (object, plugin) => {
-    if (!object) return;
-    if (typeof(object) !== 'object') return;
+logger.add_log_methods = (object, logName) => {
+    if (!object) return
 
-    for (const level in logger.levels) {
-        const fname = `log${level.toLowerCase()}`;
-        if (object[fname]) continue;  // already added
-        object[fname] = logger.log_if_level(level, `LOG${level}`, plugin);
+    if (typeof object === 'function') {
+        // add logging methods to class prototypes (Connection, Plugin, etc.)
+
+        for (const level of level_names.map(l => l.toLowerCase())) {
+            object.prototype[`log${level}`] = (function (level) {
+                return function () {
+                    logger[level].apply(logger, [ this, ...arguments ]);
+                };
+            })(`log${level}`);
+        }
+    }
+    else if (typeof object === 'object') {
+        // add logging methods to objects
+
+        for (const level of level_names) {
+            // objects gets log function names: loginfo, logwarn, logdebug, ...
+            const fnNames = [`log${level.toLowerCase()}`]
+
+            // logger also gets short names
+            if (Object.hasOwn(object, 'name') && object.name === 'logger') {
+                fnNames.push(level.toLowerCase())
+            }
+
+            for (const fnName of fnNames) {
+                if (Object.hasOwn(object, fnName)) continue; // already added
+                object[fnName] = logger.log_if_level(level, `LOG${level}`, logName);
+            }
+        }
     }
 }
 
@@ -328,5 +346,3 @@ logger.add_log_methods(logger);
 
 // load these down here so it sees all the logger methods compiled above
 plugins = require('./plugins');
-connection = require('./connection');
-outbound = require('./outbound');