Haraka 3.0.0 → 3.0.2

package/Changes.md

@@ -1,6 +1,45 @@
 
 ### Unreleased
 
+
+### [3.0.2] - 2023-06-12
+
+#### Fixed
+
+- feat(q_forward): add LMTP routing handling #3199
+- chore(q_forward): tighten up queue.wants handling #3199
+- doc(q_forward): improve markdown formatting #3199
+- helo.checks: several fixes, #3191
+- q/smtp_forward: correct path to next_hop #3186
+- don't leak addr parsing errors into SMTP conversation #3185
+- connection: handle dns.reverse invalid throws on node v20 #3184
+- rename redis command setex to setEx #3181
+
+#### Changed
+
+- test(helo.checks): add regression tests for #3191 #3195
+- connection: handle dns.reverse invalid throws on node v20
+- build(deps): bump ipaddr.js from 2.0.1 to 2.1.0 #3194
+- chore: bump a few dependency versions #3184
+- dns_list_base: avoid test failure when public DNS used #3184
+- doc(outbound.ini) update link #3159
+- doc(clamd.md) fixed spelling error #3155
+
+
+### [3.0.1] - 2023-01-19
+
+#### Fixed
+
+- fix(bin/haraka): set server.cfg and pass to conn, fixes #3143
+- fix(bin/haraka): correct error messages for help options #3142
+- fix: dkim_verify fails to find record #3149
+
+#### Changed
+
+- plugins: Add haraka-plugin-outbound-logger to registry #3146
+- dep(pi-spf): bump version 1.1.3 to 1.2.0
+
+
 ### [3.0.0] - 2022-12-17
 
 #### Added
@@ -1329,3 +1368,5 @@
 
 
 [3.0.0]: https://github.com/haraka/Haraka/releases/tag/3.0.0
+[3.0.1]: https://github.com/haraka/Haraka/releases/tag/3.0.1
+[3.0.2]: https://github.com/haraka/Haraka/releases/tag/3.0.2

package/Plugins.md

@@ -69,6 +69,7 @@ Create a PR adding yours to this list.
 | [messagesniffer][url-msgsniff]    | Anti-spam via [MessageSniffer][url-ms] |
 | [milter][url-milter]              | milter support |
 | [mongodb][mongo-url]              | Queue emails to MongoDB |
+| [outbound-logger][url-outbound-logger] | JSON logging of outbound email traffic. Logs useful metadata about delivered/bounced emails |
 | [prevent_credential_leaks][url-creds]  | Prevent users from emailing their credentials |
 | [process_title][url-proctitle]    | Populate `ps` output with activity counters |
 | queue/[discard][url-qdisc]        | queues messages to /dev/null |
@@ -190,4 +191,5 @@ Create a PR adding yours to this list.
 [url-wildduck]: https://github.com/nodemailer/haraka-plugin-wildduck
 [url-xclient]: https://github.com/haraka/Haraka/blob/master/docs/plugins/xclient.md
 [mongo-url]: https://github.com/Helpmonks/haraka-plugin-mongodb
+[url-outbound-logger]: https://github.com/mr-karan/haraka-plugin-outbound-logger
 

package/bin/haraka

@@ -383,7 +383,7 @@ else if (parsed.qlist) {
 }
 else if (parsed.qstat) {
     if (!parsed.configs) {
-        fail("qlist option requires config path");
+        fail("qstat option requires config path");
     }
     process.env.HARAKA = parsed.configs;
     logger = require(path.join(base, "logger"));
@@ -435,7 +435,7 @@ else if (parsed.graceful) {
 }
 else if (parsed.qempty) {
     if (!parsed.configs) {
-        fail("qlist option requires config path");
+        fail("qempty option requires config path");
     }
     fail("qempty is unimplemented");
 }
@@ -476,7 +476,7 @@ else if (parsed.order) {
 }
 else if (parsed.test) {
     if (!parsed.configs) {
-        fail("order option requires config path");
+        fail("test option requires config path");
     }
     process.env.HARAKA = parsed.configs;
     try {
@@ -528,9 +528,10 @@ else if (parsed.test) {
         address () {
             return { port: 25, family: 'ipv4', address: '127.0.0.1' };
         },
+        cfg  : require('haraka-config').get('smtp.ini'),
         notes: new Notes(),
     }
-    const connection = Connection.createConnection(client, server);
+    const connection = Connection.createConnection(client, server, server.cfg);
     if (parsed['set-relay']) connection.relaying = true;
 
     const run_next_hook = function () {

package/config/outbound.ini

@@ -1,4 +1,4 @@
-; see http://haraka.github.io/manual/Outbound.html
+; see http://haraka.github.io/core/Outbound
 ;
 ; disabled (default: false)
 ; disabled=true

package/connection.js

@@ -15,7 +15,7 @@ const constants   = require('haraka-constants');
 const net_utils   = require('haraka-net-utils');
 const Notes       = require('haraka-notes');
 const utils       = require('haraka-utils');
-const { Address }     = require('address-rfc2821');
+const { Address } = require('address-rfc2821');
 const ResultStore = require('haraka-results');
 
 // Haraka libs
@@ -734,9 +734,16 @@ class Connection {
                 });
                 break;
             default:
-                dns.reverse(this.remote.ip, (err, domains) => {
-                    this.rdns_response(err, domains);
-                });
+                // BUG: dns.reverse throws on invalid input (and sometimes valid
+                // input nodejs/node#47847). Also throws when empty results
+                try {
+                    dns.reverse(this.remote.ip, (err, domains) => {
+                        this.rdns_response(err, domains);
+                    })
+                }
+                catch (err) {
+                    this.rdns_response(err, []);
+                }
         }
     }
     rdns_response (err, domains) {
@@ -1319,16 +1326,15 @@ class Connection {
             this.errors++;
             return this.respond(503, 'Use EHLO/HELO before MAIL');
         }
-        // Require authentication on connections to port 587 & 465
+        // Require authentication on ports 587 & 465
         if (!this.relaying && [587,465].includes(this.local.port)) {
             this.errors++;
             return this.respond(550, 'Authentication required');
         }
+
         let results;
-        let from;
         try {
             results = rfc1869.parse('mail', line, this.cfg.main.strict_rfc1869 && !this.relaying);
-            from    = new Address (results.shift());
         }
         catch (err) {
             this.errors++;
@@ -1343,9 +1349,18 @@ class Connection {
                 return this.respond(452, 'Internal Server Error');
             }
             else {
-                return this.respond(501, ["Command parsing failed", err]);
+                return this.respond(501, ['Command parsing failed', err]);
             }
         }
+
+        let from;
+        try {
+            from = new Address(results.shift());
+        }
+        catch (err) {
+            return this.respond(501, `Invalid MAIL FROM address`);
+        }
+
         // Get rest of key=value pairs
         const params = {};
         results.forEach(param => {
@@ -1382,10 +1397,8 @@ class Connection {
         }
 
         let results;
-        let recip;
         try {
             results = rfc1869.parse('rcpt', line, this.cfg.main.strict_rfc1869 && !this.relaying);
-            recip   = new Address(results.shift());
         }
         catch (err) {
             this.errors++;
@@ -1403,6 +1416,15 @@ class Connection {
                 return this.respond(501, ["Command parsing failed", err]);
             }
         }
+
+        let recip;
+        try {
+            recip = new Address(results.shift());
+        }
+        catch (err) {
+            return this.respond(501, `Invalid RCPT TO address`);
+        }
+
         // Get rest of key=value pairs
         const params = {};
         results.forEach((param) => {

package/dkim.js

@@ -302,7 +302,8 @@ class DKIMObject {
                 }
             }
             if (!res) return this.result('no key for signature', 'invalid');
-            for (const record of res) {
+            for (const recordSegments of res) {
+                const record = recordSegments.join('');
                 if (!record.includes('p=')) {
                     this.debug(`${this.identity}: ignoring TXT record: ${record}`);
                     continue;

package/docs/plugins/clamd.md

@@ -133,7 +133,7 @@ meeting following criteria.
   via regexp by enclosing the pattern in //.
 
   To negate a match (e.g. reject if it matches), prefix the match with !.
-  Negative matches are always tested fist.
+  Negative matches are always tested first.
 
   Example:
 

package/docs/plugins/queue/smtp_forward.md

@@ -1,27 +1,18 @@
-queue/smtp\_forward
+# queue/smtp\_forward
 ==================
 
-This plugin delivers to another mail server. This is a common setup when you
-want to have a mail server with a solid pedigree of outbound delivery to
-other hosts, and inbound delivery to users.
+This plugin delivers to another mail server. This is a common setup when you want to have a mail server with a solid pedigree of outbound delivery to other hosts, and inbound delivery to users.
 
-In comparison to `queue/smtp_proxy`, this plugin waits until queue time to
-attempt the ongoing connection. This can be a benefit in reducing connections
-to your inbound mail server when you have content filtering (such as
-spamassassin) enabled. A possible downside is that it also delays recipient
-validation that the ongoing mail server may provide until queue time.
+In comparison to `queue/smtp_proxy`, this plugin waits until queue time to attempt the ongoing connection. This can be a benefit in reducing connections to your inbound mail server when you have content filtering (such as spamassassin) enabled. A possible downside is that it also delays recipient validation that the ongoing mail server may provide until queue time.
 
-Configuration
+## Configuration
 -------------
 
-* smtp\_forward.ini
-
-  Configuration is stored in this file in the following keys:
+Configuration is stored in smtp\_forward.ini in the following keys:
 
   * enable\_outbound=[true]
 
-    SMTP forward outbound messages (set to false to enable Haraka's separate
-    Outbound mail routing (MX based delivery)).
+    SMTP forward outbound messages (set to false to enable Haraka's separate Outbound mail routing (MX based delivery)).
 
   * host=HOST
 
@@ -33,14 +24,11 @@ Configuration
 
   * connect\_timeout=SECONDS
 
-    The maximum amount of time to wait when creating a new connection
-    to the host.  Default: 30 seconds.
+    The maximum amount of time to wait when creating a new connection to the host.  Default: 30 seconds.
 
   * timeout=SECONDS
 
-    The amount of seconds to let a backend connection live idle in the
-    connection pool.  This should always be less than the global plugin
-    timeout, which should in turn be less than the connection timeout.
+    The amount of seconds to let a backend connection live idle in the connection pool.  This should always be less than the global plugin timeout, which should in turn be less than the connection timeout.
 
   * max\_connections=NUMBER
 
@@ -48,12 +36,9 @@ Configuration
 
   * enable\_tls=[true]
 
-    Enable TLS with the forward host (if supported). TLS uses options
-    from the tls plugin. If key and cert are provided in the the outbound section of the tls plugin,
-    that certificate will be used as a TLS Client Certificate.
+    Enable TLS with the forward host (if supported). TLS uses options from the tls plugin. If key and cert are provided in the the outbound section of the tls plugin, that certificate will be used as a TLS Client Certificate.
 
-    This option controls the use of TLS via `STARTTLS`. This plugin does not work with
-    SMTP over TLS.
+    This option controls the use of TLS via `STARTTLS`. This plugin does not work with SMTP over TLS.
 
   * auth\_type=[plain\|login]
 
@@ -69,9 +54,7 @@ Configuration
 
   * queue
 
-    Which queue plugin to use. Default: undefined. The default bahavior is to
-    use smtp_forward for inbound connections and outbound for relaying
-    connections. This option is used for complex mail routes.
+    Which queue plugin to use. Default: undefined. The default bahavior is to use smtp_forward for inbound connections and outbound for relaying connections. This option is used for complex mail routes.
 
   * check_sender=false
 
@@ -86,18 +69,13 @@ Configuration
 
 # Per-Domain Configuration
 
-More specific forward routes for domains can be defined. The domain is
-chosen based on the value of the `domain_selector` config variable.
+More specific forward routes for domains can be defined. The domain is chosen based on the value of the `domain_selector` config variable.
 
-When `domain_selector` is set to `rcpt_to` (the default), more specific
-routes are only honored for SMTP connections with a single recipient or SMTP
-connections where every recipient host is identical.
+When `domain_selector` is set to `rcpt_to` (the default), more specific routes are only honored for SMTP connections with a single recipient or SMTP connections where every recipient host is identical.
 
-When `domain_selector` is set to `mail_from`, the domain of the MAIL FROM
-address is used.
+When `domain_selector` is set to `mail_from`, the domain of the MAIL FROM address is used.
 
-enable\_outbound can be set or unset on a per-domain level to enable or disable
-forwarding for specific domains.
+enable\_outbound can be set or unset on a per-domain level to enable or disable forwarding for specific domains.
 
     # default SMTP host
     host=1.2.3.4

package/docs/plugins/queue/smtp_proxy.md

@@ -1,4 +1,4 @@
-queue/smtp\_proxy
+# queue/smtp\_proxy
 ================
 
 This plugin delivers to another mail server. This is a common setup when you
@@ -12,14 +12,12 @@ particular one important facility to some setups is recipient filtering.
 However be aware that other than connect and HELO-time filtering, you will
 have as many connections to your ongoing SMTP server as you have to Haraka.
 
-Configuration
+## Configuration
 -------------
 
-* smtp\_proxy.ini
-  
-  Configuration is stored in this file in the following keys:
+Configuration is stored in smtp\_proxy.ini in the following keys:
 
-    * enable\_outbound=[true]
+  * enable\_outbound=[true]
 
     SMTP proxy outbound messages (set to false to enable Haraka's
     separate Outbound mail routing (MX based delivery)).
@@ -27,9 +25,9 @@ Configuration
   * host=HOST
     
     The host to connect to.
-    
+
   * port=PORT
-    
+
     The port to connect to.
 
   * connect\_timeout=SECONDS
@@ -38,17 +36,17 @@ Configuration
     to the host.  Default if unspecified is 30 seconds.
 
   * timeout=SECONDS
-    
+
     The amount of seconds to let a backend connection live idle in the
     proxy pool.  This should always be less than the global plugin timeout,
     which should in turn be less than the connection timeout.
 
   * max\_connections=NUMBER
-    
+
     Maximum number of connections to create at any given time.
 
   * enable\_tls=[true|yes|1]
- 
+
     Enable TLS with the forward host (if supported). TLS uses options from
     the tls plugin.
 

package/outbound/tls.js

@@ -95,7 +95,7 @@ class OutboundTLS {
 
         logger.lognotice(this, `TLS connection failed. Marking ${host} as non-TLS for ${expiry} seconds`);
 
-        this.db.setex(dbkey, expiry, (new Date()).toISOString())
+        this.db.setEx(dbkey, expiry, (new Date()).toISOString())
             .then(cb)
             .catch(err => {
                 logger.logerror(this, `Redis returned error: ${err}`);

package/package.json

@@ -9,7 +9,7 @@
     "server",
     "email"
   ],
-  "version": "3.0.0",
+  "version": "3.0.2",
   "homepage": "http://haraka.github.io",
   "repository": {
     "type": "git",
@@ -20,65 +20,65 @@
     "node": ">=16"
   },
   "dependencies": {
-    "address-rfc2821"       : "^2.0.1",
-    "address-rfc2822"       : "^2.1.0",
-    "async"                 : "^3.2.4",
-    "daemon"                : "~1.1.0",
-    "ipaddr.js"             : "~2.0.1",
-    "node-gyp"              : "^9.3.1",
-    "nopt"                  : "~7.0.0",
-    "npid"                  : "~0.4.0",
-    "semver"                : "~7.3.8",
-    "sprintf-js"            : "~1.1.2",
-    "haraka-config"         : "^1.1.0",
-    "haraka-constants"      : "^1.0.6",
-    "haraka-dsn"            : "^1.0.4",
-    "haraka-email-message"  : "^1.2.0",
-    "haraka-message-stream" : "^1.2.0",
-    "haraka-net-utils"      : "^1.5.0",
-    "haraka-notes"          : "^1.0.6",
+    "address-rfc2821": "^2.0.1",
+    "address-rfc2822": "^2.1.0",
+    "async": "^3.2.4",
+    "daemon": "~1.1.0",
+    "ipaddr.js": "~2.1.0",
+    "node-gyp": "^9.4.0",
+    "nopt": "~7.2.0",
+    "npid": "~0.4.0",
+    "semver": "~7.5.2",
+    "sprintf-js": "~1.1.2",
+    "haraka-config": "^1.1.0",
+    "haraka-constants": "^1.0.6",
+    "haraka-dsn": "^1.0.4",
+    "haraka-email-message": "^1.2.0",
+    "haraka-message-stream": "^1.2.0",
+    "haraka-net-utils": "^1.5.0",
+    "haraka-notes": "^1.0.6",
     "haraka-plugin-attachment": "^1.0.7",
-    "haraka-plugin-spf"     : "1.1.3",
-    "haraka-plugin-redis"   : "^2.0.5",
-    "haraka-results"        : "^2.2.2",
-    "haraka-tld"            : "^1.1.0",
-    "haraka-utils"          : "^1.0.3",
-    "openssl-wrapper"       : "^0.3.4",
-    "sockaddr"              : "^1.0.1"
+    "haraka-plugin-spf": "1.2.0",
+    "haraka-plugin-redis": "^2.0.5",
+    "haraka-results": "^2.2.3",
+    "haraka-tld": "^1.1.1",
+    "haraka-utils": "^1.0.3",
+    "openssl-wrapper": "^0.3.4",
+    "sockaddr": "^1.0.1"
   },
   "optionalDependencies": {
-    "haraka-plugin-access"  : "^1.1.5",
-    "haraka-plugin-aliases" : "^1.0.1",
-    "haraka-plugin-asn"     : "^2.0.1",
+    "haraka-plugin-access": "^1.1.5",
+    "haraka-plugin-aliases": "^1.0.1",
+    "haraka-plugin-asn": "^2.0.1",
     "haraka-plugin-auth-ldap": "^1.0.2",
-    "haraka-plugin-dcc"     : "^1.0.1",
-    "haraka-plugin-elasticsearch" : "^1.0.6",
-    "haraka-plugin-fcrdns"  : "^1.1.0",
-    "haraka-plugin-graph"   : "^1.0.5",
-    "haraka-plugin-geoip"   : "^1.0.17",
-    "haraka-plugin-headers" : "^1.0.3",
-    "haraka-plugin-karma"   : "^2.1.0",
-    "haraka-plugin-limit"   : "^1.1.0",
-    "haraka-plugin-p0f"     : "^1.0.9",
-    "haraka-plugin-qmail-deliverable" : "^1.1.1",
+    "haraka-plugin-dcc": "^1.0.1",
+    "haraka-plugin-elasticsearch": "^1.0.6",
+    "haraka-plugin-fcrdns": "^1.1.0",
+    "haraka-plugin-graph": "^1.0.5",
+    "haraka-plugin-geoip": "^1.0.17",
+    "haraka-plugin-headers": "^1.0.3",
+    "haraka-plugin-karma": "^2.1.0",
+    "haraka-plugin-limit": "^1.1.0",
+    "haraka-plugin-p0f": "^1.0.9",
+    "haraka-plugin-qmail-deliverable": "^1.2.1",
     "haraka-plugin-known-senders": "^1.0.8",
     "haraka-plugin-rcpt-ldap": "^1.0.0",
-    "haraka-plugin-recipient-routes" : "^1.0.4",
-    "haraka-plugin-rspamd"  : "^1.2.0",
-    "haraka-plugin-syslog"  : "^1.0.3",
-    "haraka-plugin-uribl"   : "^1.0.6",
-    "haraka-plugin-watch"   : "^2.0.2",
-    "ocsp"                  : "~1.2.0",
-    "redis"                 : "~4.5.1",
-    "tmp"                   : "~0.2.1"
+    "haraka-plugin-recipient-routes": "^1.0.4",
+    "haraka-plugin-rspamd": "^1.2.0",
+    "haraka-plugin-syslog": "^1.0.3",
+    "haraka-plugin-uribl": "^1.0.6",
+    "haraka-plugin-watch": "^2.0.2",
+    "ocsp": "~1.2.0",
+    "redis": "^4.5.1",
+    "tmp": "~0.2.1"
   },
   "devDependencies": {
-    "nodeunit-x"            : "^0.15.0",
-    "haraka-test-fixtures"  : "^1.2.1",
-    "mock-require"          : "^3.0.3",
-    "eslint"                : "^8.27.0",
-    "eslint-plugin-haraka"  : "^1.0.15",
-    "nodemailer"            : "^6.7.7"
+    "nodeunit-x": "^0.16.0",
+    "haraka-test-fixtures": "^1.3.0",
+    "mock-require": "^3.0.3",
+    "eslint": "^8.42.0",
+    "eslint-plugin-haraka": "^1.0.15",
+    "nodemailer": "^6.9.3"
   },
   "bugs": {
     "mail": "haraka.mail@gmail.com",

package/plugins/dns_list_base.js

@@ -126,7 +126,7 @@ exports.multi = function (lookup, zones, cb) {
             }
             cb(err, zone, a, true);
             done();
-        });
+        })
     }
     function zonesDone (err) {
         cb(err, null, null, false);
@@ -148,8 +148,8 @@ exports.first = function (lookup, zones, cb, cb_each) {
 
         // has pending queries OR this one is a positive result
         ran_cb = true;
-        return cb(err, zone, a);
-    });
+        cb(err, zone, a);
+    })
 }
 
 exports.check_zones = function (interval) {

package/plugins/helo.checks.js

@@ -11,7 +11,6 @@ const checks = [
     'bare_ip',            // HELO is bare IP (vs required Address Literal)
     'dynamic',            // HELO hostname looks dynamic (dsl|dialup|etc...)
     'big_company',        // Well known HELOs that must match rdns
-    'literal_mismatch',   // IP literal that doesn't match remote IP
     'valid_hostname',     // HELO hostname is a legal DNS name
     'rdns_match',         // HELO hostname matches rDNS
     'forward_dns',        // HELO hostname resolves to the connecting IP
@@ -31,7 +30,7 @@ exports.register = function () {
     this.register_hook('helo', 'init');
     this.register_hook('ehlo', 'init');
 
-    for (const c in checks) {
+    for (const c of checks) {
         if (!this.cfg.check[c]) continue; // disabled in config
         this.register_hook('helo', c);
         this.register_hook('ehlo', c);
@@ -41,6 +40,13 @@ exports.register = function () {
     this.register_hook('helo', 'emit_log');
     this.register_hook('ehlo', 'emit_log');
 
+    // IP literal that doesn't match remote IP
+    this.register_hook('helo', 'literal_mismatch');
+    this.register_hook('ehlo', 'literal_mismatch');
+
+    this.cfg.check.literal_mismatch = this.cfg.check.literal_mismatch ?? 2;
+    this.cfg.reject.literal_mismatch = this.cfg.reject.literal_mismatch ?? false;
+
     if (this.cfg.check.match_re) {
         const load_re_file = () => {
             const regex_list = utils.valid_regexes(this.config.get('helo.checks.regexps', 'list', load_re_file));
@@ -95,17 +101,19 @@ exports.load_helo_checks_ini = function () {
 }
 
 exports.init = function (next, connection, helo) {
-
-    const hc = connection.results.get('helo.checks');
-    if (!hc) {     // first HELO result
+    if (!connection.results.has('helo.checks', 'helo_host', helo)) {
         connection.results.add(this, {helo_host: helo});
-        return next();
     }
 
     next();
 }
 
 exports.should_skip = function (connection, test_name) {
+    if (connection.results.has('helo.checks', '_skip_hooks', test_name)) {
+        this.logdebug(connection, `SKIPPING: ${test_name}`);
+        return true;
+    }
+    connection.results.push(this, {_skip_hooks: test_name});
 
     if (this.cfg.skip.relaying && connection.relaying) {
         connection.results.add(this, {skip: `${test_name}(relay)`});