Haraka 3.0.1 → 3.0.2

package/Changes.md

@@ -2,6 +2,30 @@
 ### Unreleased
 
 
+### [3.0.2] - 2023-06-12
+
+#### Fixed
+
+- feat(q_forward): add LMTP routing handling #3199
+- chore(q_forward): tighten up queue.wants handling #3199
+- doc(q_forward): improve markdown formatting #3199
+- helo.checks: several fixes, #3191
+- q/smtp_forward: correct path to next_hop #3186
+- don't leak addr parsing errors into SMTP conversation #3185
+- connection: handle dns.reverse invalid throws on node v20 #3184
+- rename redis command setex to setEx #3181
+
+#### Changed
+
+- test(helo.checks): add regression tests for #3191 #3195
+- connection: handle dns.reverse invalid throws on node v20
+- build(deps): bump ipaddr.js from 2.0.1 to 2.1.0 #3194
+- chore: bump a few dependency versions #3184
+- dns_list_base: avoid test failure when public DNS used #3184
+- doc(outbound.ini) update link #3159
+- doc(clamd.md) fixed spelling error #3155
+
+
 ### [3.0.1] - 2023-01-19
 
 #### Fixed
@@ -1345,3 +1369,4 @@
 
 [3.0.0]: https://github.com/haraka/Haraka/releases/tag/3.0.0
 [3.0.1]: https://github.com/haraka/Haraka/releases/tag/3.0.1
+[3.0.2]: https://github.com/haraka/Haraka/releases/tag/3.0.2

package/config/outbound.ini

@@ -1,4 +1,4 @@
-; see http://haraka.github.io/manual/Outbound.html
+; see http://haraka.github.io/core/Outbound
 ;
 ; disabled (default: false)
 ; disabled=true

package/connection.js

@@ -15,7 +15,7 @@ const constants   = require('haraka-constants');
 const net_utils   = require('haraka-net-utils');
 const Notes       = require('haraka-notes');
 const utils       = require('haraka-utils');
-const { Address }     = require('address-rfc2821');
+const { Address } = require('address-rfc2821');
 const ResultStore = require('haraka-results');
 
 // Haraka libs
@@ -734,9 +734,16 @@ class Connection {
                 });
                 break;
             default:
-                dns.reverse(this.remote.ip, (err, domains) => {
-                    this.rdns_response(err, domains);
-                });
+                // BUG: dns.reverse throws on invalid input (and sometimes valid
+                // input nodejs/node#47847). Also throws when empty results
+                try {
+                    dns.reverse(this.remote.ip, (err, domains) => {
+                        this.rdns_response(err, domains);
+                    })
+                }
+                catch (err) {
+                    this.rdns_response(err, []);
+                }
         }
     }
     rdns_response (err, domains) {
@@ -1319,16 +1326,15 @@ class Connection {
             this.errors++;
             return this.respond(503, 'Use EHLO/HELO before MAIL');
         }
-        // Require authentication on connections to port 587 & 465
+        // Require authentication on ports 587 & 465
         if (!this.relaying && [587,465].includes(this.local.port)) {
             this.errors++;
             return this.respond(550, 'Authentication required');
         }
+
         let results;
-        let from;
         try {
             results = rfc1869.parse('mail', line, this.cfg.main.strict_rfc1869 && !this.relaying);
-            from    = new Address (results.shift());
         }
         catch (err) {
             this.errors++;
@@ -1343,9 +1349,18 @@ class Connection {
                 return this.respond(452, 'Internal Server Error');
             }
             else {
-                return this.respond(501, ["Command parsing failed", err]);
+                return this.respond(501, ['Command parsing failed', err]);
             }
         }
+
+        let from;
+        try {
+            from = new Address(results.shift());
+        }
+        catch (err) {
+            return this.respond(501, `Invalid MAIL FROM address`);
+        }
+
         // Get rest of key=value pairs
         const params = {};
         results.forEach(param => {
@@ -1382,10 +1397,8 @@ class Connection {
         }
 
         let results;
-        let recip;
         try {
             results = rfc1869.parse('rcpt', line, this.cfg.main.strict_rfc1869 && !this.relaying);
-            recip   = new Address(results.shift());
         }
         catch (err) {
             this.errors++;
@@ -1403,6 +1416,15 @@ class Connection {
                 return this.respond(501, ["Command parsing failed", err]);
             }
         }
+
+        let recip;
+        try {
+            recip = new Address(results.shift());
+        }
+        catch (err) {
+            return this.respond(501, `Invalid RCPT TO address`);
+        }
+
         // Get rest of key=value pairs
         const params = {};
         results.forEach((param) => {

package/docs/plugins/clamd.md

@@ -133,7 +133,7 @@ meeting following criteria.
   via regexp by enclosing the pattern in //.
 
   To negate a match (e.g. reject if it matches), prefix the match with !.
-  Negative matches are always tested fist.
+  Negative matches are always tested first.
 
   Example:
 

package/docs/plugins/queue/smtp_forward.md

@@ -1,27 +1,18 @@
-queue/smtp\_forward
+# queue/smtp\_forward
 ==================
 
-This plugin delivers to another mail server. This is a common setup when you
-want to have a mail server with a solid pedigree of outbound delivery to
-other hosts, and inbound delivery to users.
+This plugin delivers to another mail server. This is a common setup when you want to have a mail server with a solid pedigree of outbound delivery to other hosts, and inbound delivery to users.
 
-In comparison to `queue/smtp_proxy`, this plugin waits until queue time to
-attempt the ongoing connection. This can be a benefit in reducing connections
-to your inbound mail server when you have content filtering (such as
-spamassassin) enabled. A possible downside is that it also delays recipient
-validation that the ongoing mail server may provide until queue time.
+In comparison to `queue/smtp_proxy`, this plugin waits until queue time to attempt the ongoing connection. This can be a benefit in reducing connections to your inbound mail server when you have content filtering (such as spamassassin) enabled. A possible downside is that it also delays recipient validation that the ongoing mail server may provide until queue time.
 
-Configuration
+## Configuration
 -------------
 
-* smtp\_forward.ini
-
-  Configuration is stored in this file in the following keys:
+Configuration is stored in smtp\_forward.ini in the following keys:
 
   * enable\_outbound=[true]
 
-    SMTP forward outbound messages (set to false to enable Haraka's separate
-    Outbound mail routing (MX based delivery)).
+    SMTP forward outbound messages (set to false to enable Haraka's separate Outbound mail routing (MX based delivery)).
 
   * host=HOST
 
@@ -33,14 +24,11 @@ Configuration
 
   * connect\_timeout=SECONDS
 
-    The maximum amount of time to wait when creating a new connection
-    to the host.  Default: 30 seconds.
+    The maximum amount of time to wait when creating a new connection to the host.  Default: 30 seconds.
 
   * timeout=SECONDS
 
-    The amount of seconds to let a backend connection live idle in the
-    connection pool.  This should always be less than the global plugin
-    timeout, which should in turn be less than the connection timeout.
+    The amount of seconds to let a backend connection live idle in the connection pool.  This should always be less than the global plugin timeout, which should in turn be less than the connection timeout.
 
   * max\_connections=NUMBER
 
@@ -48,12 +36,9 @@ Configuration
 
   * enable\_tls=[true]
 
-    Enable TLS with the forward host (if supported). TLS uses options
-    from the tls plugin. If key and cert are provided in the the outbound section of the tls plugin,
-    that certificate will be used as a TLS Client Certificate.
+    Enable TLS with the forward host (if supported). TLS uses options from the tls plugin. If key and cert are provided in the the outbound section of the tls plugin, that certificate will be used as a TLS Client Certificate.
 
-    This option controls the use of TLS via `STARTTLS`. This plugin does not work with
-    SMTP over TLS.
+    This option controls the use of TLS via `STARTTLS`. This plugin does not work with SMTP over TLS.
 
   * auth\_type=[plain\|login]
 
@@ -69,9 +54,7 @@ Configuration
 
   * queue
 
-    Which queue plugin to use. Default: undefined. The default bahavior is to
-    use smtp_forward for inbound connections and outbound for relaying
-    connections. This option is used for complex mail routes.
+    Which queue plugin to use. Default: undefined. The default bahavior is to use smtp_forward for inbound connections and outbound for relaying connections. This option is used for complex mail routes.
 
   * check_sender=false
 
@@ -86,18 +69,13 @@ Configuration
 
 # Per-Domain Configuration
 
-More specific forward routes for domains can be defined. The domain is
-chosen based on the value of the `domain_selector` config variable.
+More specific forward routes for domains can be defined. The domain is chosen based on the value of the `domain_selector` config variable.
 
-When `domain_selector` is set to `rcpt_to` (the default), more specific
-routes are only honored for SMTP connections with a single recipient or SMTP
-connections where every recipient host is identical.
+When `domain_selector` is set to `rcpt_to` (the default), more specific routes are only honored for SMTP connections with a single recipient or SMTP connections where every recipient host is identical.
 
-When `domain_selector` is set to `mail_from`, the domain of the MAIL FROM
-address is used.
+When `domain_selector` is set to `mail_from`, the domain of the MAIL FROM address is used.
 
-enable\_outbound can be set or unset on a per-domain level to enable or disable
-forwarding for specific domains.
+enable\_outbound can be set or unset on a per-domain level to enable or disable forwarding for specific domains.
 
     # default SMTP host
     host=1.2.3.4

package/docs/plugins/queue/smtp_proxy.md

@@ -1,4 +1,4 @@
-queue/smtp\_proxy
+# queue/smtp\_proxy
 ================
 
 This plugin delivers to another mail server. This is a common setup when you
@@ -12,14 +12,12 @@ particular one important facility to some setups is recipient filtering.
 However be aware that other than connect and HELO-time filtering, you will
 have as many connections to your ongoing SMTP server as you have to Haraka.
 
-Configuration
+## Configuration
 -------------
 
-* smtp\_proxy.ini
-  
-  Configuration is stored in this file in the following keys:
+Configuration is stored in smtp\_proxy.ini in the following keys:
 
-    * enable\_outbound=[true]
+  * enable\_outbound=[true]
 
     SMTP proxy outbound messages (set to false to enable Haraka's
     separate Outbound mail routing (MX based delivery)).
@@ -27,9 +25,9 @@ Configuration
   * host=HOST
     
     The host to connect to.
-    
+
   * port=PORT
-    
+
     The port to connect to.
 
   * connect\_timeout=SECONDS
@@ -38,17 +36,17 @@ Configuration
     to the host.  Default if unspecified is 30 seconds.
 
   * timeout=SECONDS
-    
+
     The amount of seconds to let a backend connection live idle in the
     proxy pool.  This should always be less than the global plugin timeout,
     which should in turn be less than the connection timeout.
 
   * max\_connections=NUMBER
-    
+
     Maximum number of connections to create at any given time.
 
   * enable\_tls=[true|yes|1]
- 
+
     Enable TLS with the forward host (if supported). TLS uses options from
     the tls plugin.
 

package/outbound/tls.js

@@ -95,7 +95,7 @@ class OutboundTLS {
 
         logger.lognotice(this, `TLS connection failed. Marking ${host} as non-TLS for ${expiry} seconds`);
 
-        this.db.setex(dbkey, expiry, (new Date()).toISOString())
+        this.db.setEx(dbkey, expiry, (new Date()).toISOString())
             .then(cb)
             .catch(err => {
                 logger.logerror(this, `Redis returned error: ${err}`);

package/package.json

@@ -9,7 +9,7 @@
     "server",
     "email"
   ],
-  "version": "3.0.1",
+  "version": "3.0.2",
   "homepage": "http://haraka.github.io",
   "repository": {
     "type": "git",
@@ -24,11 +24,11 @@
     "address-rfc2822": "^2.1.0",
     "async": "^3.2.4",
     "daemon": "~1.1.0",
-    "ipaddr.js": "~2.0.1",
-    "node-gyp": "^9.3.1",
-    "nopt": "~7.0.0",
+    "ipaddr.js": "~2.1.0",
+    "node-gyp": "^9.4.0",
+    "nopt": "~7.2.0",
     "npid": "~0.4.0",
-    "semver": "~7.3.8",
+    "semver": "~7.5.2",
     "sprintf-js": "~1.1.2",
     "haraka-config": "^1.1.0",
     "haraka-constants": "^1.0.6",
@@ -40,8 +40,8 @@
     "haraka-plugin-attachment": "^1.0.7",
     "haraka-plugin-spf": "1.2.0",
     "haraka-plugin-redis": "^2.0.5",
-    "haraka-results": "^2.2.2",
-    "haraka-tld": "^1.1.0",
+    "haraka-results": "^2.2.3",
+    "haraka-tld": "^1.1.1",
     "haraka-utils": "^1.0.3",
     "openssl-wrapper": "^0.3.4",
     "sockaddr": "^1.0.1"
@@ -60,7 +60,7 @@
     "haraka-plugin-karma": "^2.1.0",
     "haraka-plugin-limit": "^1.1.0",
     "haraka-plugin-p0f": "^1.0.9",
-    "haraka-plugin-qmail-deliverable": "^1.1.1",
+    "haraka-plugin-qmail-deliverable": "^1.2.1",
     "haraka-plugin-known-senders": "^1.0.8",
     "haraka-plugin-rcpt-ldap": "^1.0.0",
     "haraka-plugin-recipient-routes": "^1.0.4",
@@ -69,16 +69,16 @@
     "haraka-plugin-uribl": "^1.0.6",
     "haraka-plugin-watch": "^2.0.2",
     "ocsp": "~1.2.0",
-    "redis": "~4.5.1",
+    "redis": "^4.5.1",
     "tmp": "~0.2.1"
   },
   "devDependencies": {
-    "nodeunit-x": "^0.15.0",
-    "haraka-test-fixtures": "^1.2.1",
+    "nodeunit-x": "^0.16.0",
+    "haraka-test-fixtures": "^1.3.0",
     "mock-require": "^3.0.3",
-    "eslint": "^8.27.0",
+    "eslint": "^8.42.0",
     "eslint-plugin-haraka": "^1.0.15",
-    "nodemailer": "^6.7.7"
+    "nodemailer": "^6.9.3"
   },
   "bugs": {
     "mail": "haraka.mail@gmail.com",

package/plugins/dns_list_base.js

@@ -126,7 +126,7 @@ exports.multi = function (lookup, zones, cb) {
             }
             cb(err, zone, a, true);
             done();
-        });
+        })
     }
     function zonesDone (err) {
         cb(err, null, null, false);
@@ -148,8 +148,8 @@ exports.first = function (lookup, zones, cb, cb_each) {
 
         // has pending queries OR this one is a positive result
         ran_cb = true;
-        return cb(err, zone, a);
-    });
+        cb(err, zone, a);
+    })
 }
 
 exports.check_zones = function (interval) {

package/plugins/helo.checks.js

@@ -11,7 +11,6 @@ const checks = [
     'bare_ip',            // HELO is bare IP (vs required Address Literal)
     'dynamic',            // HELO hostname looks dynamic (dsl|dialup|etc...)
     'big_company',        // Well known HELOs that must match rdns
-    'literal_mismatch',   // IP literal that doesn't match remote IP
     'valid_hostname',     // HELO hostname is a legal DNS name
     'rdns_match',         // HELO hostname matches rDNS
     'forward_dns',        // HELO hostname resolves to the connecting IP
@@ -31,7 +30,7 @@ exports.register = function () {
     this.register_hook('helo', 'init');
     this.register_hook('ehlo', 'init');
 
-    for (const c in checks) {
+    for (const c of checks) {
         if (!this.cfg.check[c]) continue; // disabled in config
         this.register_hook('helo', c);
         this.register_hook('ehlo', c);
@@ -41,6 +40,13 @@ exports.register = function () {
     this.register_hook('helo', 'emit_log');
     this.register_hook('ehlo', 'emit_log');
 
+    // IP literal that doesn't match remote IP
+    this.register_hook('helo', 'literal_mismatch');
+    this.register_hook('ehlo', 'literal_mismatch');
+
+    this.cfg.check.literal_mismatch = this.cfg.check.literal_mismatch ?? 2;
+    this.cfg.reject.literal_mismatch = this.cfg.reject.literal_mismatch ?? false;
+
     if (this.cfg.check.match_re) {
         const load_re_file = () => {
             const regex_list = utils.valid_regexes(this.config.get('helo.checks.regexps', 'list', load_re_file));
@@ -95,17 +101,19 @@ exports.load_helo_checks_ini = function () {
 }
 
 exports.init = function (next, connection, helo) {
-
-    const hc = connection.results.get('helo.checks');
-    if (!hc) {     // first HELO result
+    if (!connection.results.has('helo.checks', 'helo_host', helo)) {
         connection.results.add(this, {helo_host: helo});
-        return next();
     }
 
     next();
 }
 
 exports.should_skip = function (connection, test_name) {
+    if (connection.results.has('helo.checks', '_skip_hooks', test_name)) {
+        this.logdebug(connection, `SKIPPING: ${test_name}`);
+        return true;
+    }
+    connection.results.push(this, {_skip_hooks: test_name});
 
     if (this.cfg.skip.relaying && connection.relaying) {
         connection.results.add(this, {skip: `${test_name}(relay)`});

package/plugins/queue/smtp_forward.js

@@ -26,10 +26,12 @@ exports.register = function () {
     this.register_hook('queue', 'queue_forward');
 
     if (this.cfg.main.enable_outbound) {
+        // deliver local message via smtp forward when relaying=true
         this.register_hook('queue_outbound', 'queue_forward');
-    }
 
-    this.register_hook('get_mx', 'get_mx'); // for relaying outbound messages
+        // may specify more specific routes for outbound
+        this.register_hook('get_mx', 'get_mx');
+    }
 }
 
 exports.load_smtp_forward_ini = function () {
@@ -75,7 +77,7 @@ exports.is_outbound_enabled = function (dom_cfg) {
     if ('enable_outbound' in dom_cfg) return dom_cfg.enable_outbound; // per-domain flag
 
     return this.cfg.main.enable_outbound; // follow the global configuration
-};
+}
 
 exports.check_sender = function (next, connection, params) {
     const txn = connection?.transaction;
@@ -99,7 +101,7 @@ exports.check_sender = function (next, connection, params) {
     }
 
     txn.results.add(this, {pass: 'mail_from'});
-    return next();
+    next();
 }
 
 exports.set_queue = function (connection, queue_wanted, domain) {
@@ -110,7 +112,6 @@ exports.set_queue = function (connection, queue_wanted, domain) {
     if (!queue_wanted) queue_wanted = dom_cfg.queue || this.cfg.main.queue;
     if (!queue_wanted) return true;
 
-
     let dst_host = dom_cfg.host || this.cfg.main.host;
     if (dst_host) dst_host = `smtp://${dst_host}`;
 
@@ -164,7 +165,7 @@ exports.check_recipient = function (next, connection, params) {
     // the MAIL FROM domain is not local and neither is the RCPT TO
     // Another RCPT plugin may vouch for this recipient.
     txn.results.add(this, {msg: 'rcpt!local'});
-    return next();
+    next();
 }
 
 exports.auth = function (cfg, connection, smtp_client) {
@@ -303,20 +304,24 @@ exports.queue_forward = function (next, connection) {
 
         smtp_client.on('bad_code', (code, msg) => {
             if (dead_sender() || !txn) return;
-            smtp_client.call_next(((code && code[0] === '5') ? DENY : DENYSOFT),
-                msg);
+            smtp_client.call_next(((code && code[0] === '5') ? DENY : DENYSOFT), msg);
             smtp_client.release();
         });
     });
 }
 
 exports.get_mx_next_hop = next_hop => {
-    const dest = url.parse(next_hop);
+    // queue.wants && queue.next_hop are mechanisms for fine-grained MX routing.
+    // Plugins can specify a queue to perform the delivery as well as a route. A
+    // plugin that uses this is qmail-deliverable, which can direct email delivery
+    // via smtp_forward, outbound (SMTP), and outbound (LMTP).
+    const dest = new url.URL(next_hop);
     const mx = {
         priority: 0,
-        port: dest.port || 25,
+        port: dest.port || (dest.protocol === 'lmtp:' ? 24 : 25),
         exchange: dest.hostname,
     }
+    if (dest.protocol === 'lmtp:') mx.using_lmtp = true;
     if (dest.auth) {
         mx.auth_type = 'plain';
         mx.auth_user = dest.auth.split(':')[0];
@@ -327,9 +332,11 @@ exports.get_mx_next_hop = next_hop => {
 
 exports.get_mx = function (next, hmail, domain) {
 
-    // hmail.todo not defined in tests.
-    if (hmail.todo.notes.next_hop) {
-        return next(OK, this.get_mx_next_hop(hmail.todo.notes.next_hop));
+    const qw = hmail.todo.notes.get('queue.wants')
+    if (qw && qw !== 'smtp_forward') return next()
+
+    if (qw === 'smtp_forward' && hmail.todo.notes.get('queue.next_hop')) {
+        return next(OK, this.get_mx_next_hop(hmail.todo.notes.get('queue.next_hop')));
     }
 
     const dom = this.cfg.main.domain_selector === 'mail_from' ? hmail.todo.mail_from.host.toLowerCase() : domain.toLowerCase();
@@ -341,8 +348,7 @@ exports.get_mx = function (next, hmail, domain) {
     }
 
     const mx_opts = [
-        'auth_type', 'auth_user', 'auth_pass', 'bind', 'bind_helo',
-        'using_lmtp'
+        'auth_type', 'auth_user', 'auth_pass', 'bind', 'bind_helo', 'using_lmtp'
     ]
 
     const mx = {
@@ -357,5 +363,5 @@ exports.get_mx = function (next, hmail, domain) {
         mx[o] = this.cfg[dom][o];
     })
 
-    return next(OK, mx);
+    next(OK, mx);
 }

package/plugins/tls.js

@@ -75,7 +75,7 @@ exports.set_notls = function (connection) {
 
     this.lognotice(connection, `STARTTLS failed. Marking ${connection.remote.ip} as non-TLS host for ${expiry} seconds`);
 
-    server.notes.redis.setex(`no_tls|${connection.remote.ip}`, expiry, (new Date()).toISOString());
+    server.notes.redis.setEx(`no_tls|${connection.remote.ip}`, expiry, (new Date()).toISOString());
 }
 
 exports.upgrade_connection = function (next, connection, params) {

package/tests/config/helo.checks.ini

@@ -0,0 +1,52 @@
+; disable checks or reject for each test if you are worried about strictness
+
+;dns_timeout=30
+
+[check]
+match_re=true
+bare_ip=true
+dynamic=true
+big_company=true
+; literal_mismatch:  1 = exact IP match, 2 = IP/24 match, 3 = /24 or RFC1918
+literal_mismatch=2
+valid_hostname=true
+forward_dns=true
+rdns_match=true
+; host_mismatch:  hostname differs between EHLO invocations
+host_mismatch=true
+proto_mismatch: host sent EHLO but then tries to sent HELO or vice-versa
+proto_mismatch=true
+
+[reject]
+host_mismatch=true
+proto_mismatch=true
+rdns_match=true
+dynamic=true
+bare_ip=true
+literal_mismatch=true
+valid_hostname=true
+forward_dns=true
+big_company=true
+
+[skip]
+private_ip=true
+relaying=true
+whitelist=true
+
+[bigco]
+msn.com=msn.com
+hotmail.com=hotmail.com
+yahoo.com=yahoo.com,yahoo.co.jp
+yahoo.co.jp=yahoo.com,yahoo.co.jp
+yahoo.co.uk=yahoo.co.uk
+excite.com=excite.com,excitenetwork.com
+mailexcite.com=excite.com,excitenetwork.com
+yahoo.co.jp=yahoo.com,yahoo.co.jp
+mailexcite.com=excite.com,excitenetwork.com
+aol.com=aol.com
+compuserve.com=compuserve.com,adelphia.net
+nortelnetworks.com=nortelnetworks.com,nortel.com
+earthlink.net=earthlink.net
+earthling.net=earthling.net
+google.com=google.com
+gmail.com=google.com,gmail.com