Haraka 3.0.2 → 3.0.4

package/outbound/index.js

@@ -1,10 +1,9 @@
 'use strict';
 
-const fs          = require('fs');
-const path        = require('path');
+const fs          = require('node:fs');
+const path        = require('node:path');
 
-const async       = require('async');
-const { Address }     = require('address-rfc2821');
+const { Address } = require('address-rfc2821');
 const config      = require('haraka-config');
 const constants   = require('haraka-constants');
 const net_utils   = require('haraka-net-utils');
@@ -24,21 +23,21 @@ const _qfile = exports.qfile = require('./qfile');
 
 const { queue_dir, temp_fail_queue, delivery_queue } = queuelib;
 
+const smtp_ini = config.get('smtp.ini', { booleans: [ '+headers.add_received' ] })
+
 exports.temp_fail_queue = temp_fail_queue;
 exports.delivery_queue = delivery_queue;
 
+exports.name = 'outbound';
 exports.net_utils = net_utils;
 exports.config = config;
 
-exports.get_stats = queuelib.get_stats;
-exports.list_queue = queuelib.list_queue;
-exports.stat_queue = queuelib.stat_queue;
-exports.scan_queue_pids = queuelib.scan_queue_pids;
-exports.flush_queue = queuelib.flush_queue;
-exports.load_pid_queue = queuelib.load_pid_queue;
-exports.ensure_queue_dir = queuelib.ensure_queue_dir;
-exports.load_queue = queuelib.load_queue;
-exports.stats = queuelib.stats;
+const qlfns = ['get_stats', 'list_queue', 'stat_queue', 'scan_queue_pids', 'flush_queue',
+    'load_pid_queue', 'ensure_queue_dir', 'load_queue', 'stats'
+]
+for (const n of qlfns) {
+    exports[n] = queuelib[n];
+}
 
 process.on('message', msg => {
     if (!msg.event) return
@@ -52,40 +51,27 @@ process.on('message', msg => {
         return;
     }
     if (msg.event === 'outbound.shutdown') {
-        logger.loginfo("[outbound] Shutting down temp fail queue");
+        logger.info(exports, "Shutting down temp fail queue");
         temp_fail_queue.shutdown();
         return;
     }
     // ignores the message
 });
 
-exports.send_email = function () {
+exports.send_email = function (from, to, contents, next, options = {}) {
 
-    if (arguments.length === 2) {
-        logger.logdebug("[outbound] Sending email as a transaction");
-        return this.send_trans_email(arguments[0], arguments[1]);
-    }
+    const dot_stuffed = options.dot_stuffed ?? false;
+    const notes = options.notes ?? null;
+    const origin = options.origin ?? exports;
 
-    let from = arguments[0];
-    let to   = arguments[1];
-    let contents = arguments[2];
-    const next = arguments[3];
-    const options = arguments[4] || {};
+    logger.info("Sending email via params", origin);
 
-    const dot_stuffed = options.dot_stuffed ? options.dot_stuffed : false;
-    const notes = options.notes ? options.notes : null;
-    const origin = options.origin ? options.origin : null;
+    const transaction = trans.createTransaction(null, smtp_ini);
 
-    logger.loginfo("[outbound] Sending email via params", origin);
+    logger.info(`Created transaction: ${transaction.uuid}`, origin);
 
-    const transaction = trans.createTransaction();
-
-    logger.loginfo(`[outbound] Created transaction: ${transaction.uuid}`, origin);
-
-    //Adding notes passed as parameter
-    if (notes) {
-        transaction.notes = notes;
-    }
+    // Adding notes passed as parameter
+    if (notes) transaction.notes = notes;
 
     // set MAIL FROM address, and parse if it's not an Address object
     if (from instanceof Address) {
@@ -102,10 +88,7 @@ exports.send_email = function () {
     }
 
     // Make sure to is an array
-    if (!(Array.isArray(to))) {
-        // turn into an array
-        to = [ to ];
-    }
+    if (!(Array.isArray(to))) to = [ to ];
 
     if (to.length === 0) {
         return next(constants.deny, "No recipients for email");
@@ -199,10 +182,10 @@ function get_deliveries (transaction) {
     const deliveries = [];
 
     if (obc.cfg.always_split) {
-        logger.logdebug({name: "outbound"}, "always split");
-        transaction.rcpt_to.forEach(rcpt => {
+        logger.debug(exports, "always split");
+        for (const rcpt of transaction.rcpt_to) {
             deliveries.push({domain: rcpt.host, rcpts: [ rcpt ]});
-        });
+        }
         return deliveries;
     }
 
@@ -223,11 +206,16 @@ exports.send_trans_email = function (transaction, next) {
 
     // add potentially missing headers
     if (!transaction.header.get_all('Message-Id').length) {
-        logger.loginfo("[outbound] Adding missing Message-Id header");
+        logger.info(exports, "Adding missing Message-Id header");
+        transaction.add_header('Message-Id', `<${transaction.uuid}@${net_utils.get_primary_host_name()}>`);
+    }
+    if (transaction.header.get('Message-Id') === '<>') {
+        logger.info(exports, "Replacing empty Message-Id header");
+        transaction.remove_header('Message-Id');
         transaction.add_header('Message-Id', `<${transaction.uuid}@${net_utils.get_primary_host_name()}>`);
     }
     if (!transaction.header.get_all('Date').length) {
-        logger.loginfo("[outbound] Adding missing Date header");
+        logger.info(exports, "Adding missing Date header");
         transaction.add_header('Date', utils.date_to_str(new Date()));
     }
 
@@ -239,79 +227,80 @@ exports.send_trans_email = function (transaction, next) {
 
     logger.add_log_methods(connection);
     if (!transaction.results) {
-        logger.logdebug('adding results store');
+        logger.debug(exports, 'adding results store');
         transaction.results = new ResultStore(connection);
     }
 
-    connection.pre_send_trans_email_respond = retval => {
+    connection.pre_send_trans_email_respond = async (retval) => {
         const deliveries = get_deliveries(transaction);
         const hmails = [];
         const ok_paths = [];
 
         let todo_index = 1;
 
-        async.forEachSeries(deliveries, (deliv, cb) => {
-            const todo = new TODOItem(deliv.domain, deliv.rcpts, transaction);
-            todo.uuid = `${todo.uuid}.${todo_index}`;
-            todo_index++;
-            this.process_delivery(ok_paths, todo, hmails, cb);
-        },
-        (err) => {
-            if (err) {
-                for (let i=0, l=ok_paths.length; i<l; i++) {
-                    fs.unlink(ok_paths[i], () => {});
-                }
-                transaction.results.add({ name: 'outbound'}, { err });
-                if (next) next(constants.denysoft, err);
-                return;
+        try {
+            for (const deliv of deliveries) {
+                const todo = new TODOItem(deliv.domain, deliv.rcpts, transaction);
+                todo.uuid = `${todo.uuid}.${todo_index}`;
+                todo_index++;
+                await this.process_delivery(ok_paths, todo, hmails);
             }
-
-            for (const hmail of hmails) {
-                delivery_queue.push(hmail);
+        }
+        catch (err) {
+            for (let i=0, l=ok_paths.length; i<l; i++) {
+                fs.unlink(ok_paths[i], () => {});
             }
+            transaction.results.add({ name: 'outbound'}, { err });
+            if (next) next(constants.denysoft, err);
+            return;
+        }
 
-            transaction.results.add({ name: 'outbound'}, { pass: "queued" });
-            if (next) {
-                next(constants.ok, `Message Queued (${transaction.uuid})`);
-            }
-        });
+        for (const hmail of hmails) {
+            delivery_queue.push(hmail);
+        }
+
+        transaction.results.add({ name: 'outbound'}, { pass: "queued" });
+        if (next) next(constants.ok, `Message Queued (${transaction.uuid})`);
     }
 
     plugins.run_hooks('pre_send_trans_email', connection);
 }
 
-exports.process_delivery = function (ok_paths, todo, hmails, cb) {
-    logger.loginfo(`[outbound] Transaction delivery for domain: ${todo.domain}`);
-    const fname = _qfile.name();
-    const tmp_path = path.join(queue_dir, `${_qfile.platformDOT}${fname}`);
-    const ws = new FsyncWriteStream(tmp_path, { flags: constants.WRITE_EXCL });
+exports.process_delivery = function (ok_paths, todo, hmails) {
+    return new Promise((resolve, reject) => {
+
+        logger.info(exports, `Transaction delivery for domain: ${todo.domain}`);
+        const fname = _qfile.name();
+        const tmp_path = path.join(queue_dir, `${_qfile.platformDOT}${fname}`);
+        const ws = new FsyncWriteStream(tmp_path, { flags: constants.WRITE_EXCL });
+
+        ws.on('close', () => {
+            const dest_path = path.join(queue_dir, fname);
+            fs.rename(tmp_path, dest_path, err => {
+                if (err) {
+                    logger.error(exports, `Unable to rename tmp file!: ${err}`);
+                    fs.unlink(tmp_path, () => {});
+                    reject("Queue error");
+                }
+                else {
+                    hmails.push(new HMailItem (fname, dest_path, todo.notes));
+                    ok_paths.push(dest_path);
+                    resolve();
+                }
+            })
+        })
 
-    ws.on('close', () => {
-        const dest_path = path.join(queue_dir, fname);
-        fs.rename(tmp_path, dest_path, err => {
-            if (err) {
-                logger.logerror(`[outbound] Unable to rename tmp file!: ${err}`);
-                fs.unlink(tmp_path, () => {});
-                cb("Queue error");
-            }
-            else {
-                hmails.push(new HMailItem (fname, dest_path, todo.notes));
-                ok_paths.push(dest_path);
-                cb();
-            }
+        ws.on('error', err => {
+            logger.error(exports, `Unable to write queue file (${fname}): ${err}`);
+            ws.destroy();
+            fs.unlink(tmp_path, () => {});
+            reject("Queueing failed");
         })
-    })
 
-    ws.on('error', err => {
-        logger.logerror(`[outbound] Unable to write queue file (${fname}): ${err}`);
-        ws.destroy();
-        fs.unlink(tmp_path, () => {});
-        cb("Queueing failed");
+        this.build_todo(todo, ws, () => {
+            todo.message_stream.pipe(ws, { dot_stuffing: true });
+        });
     })
-
-    this.build_todo(todo, ws, () => {
-        todo.message_stream.pipe(ws, { line_endings: '\r\n', dot_stuffing: true, ending_dot: false });
-    });
 }
 
 exports.build_todo = (todo, ws, write_more) => {
@@ -346,5 +335,3 @@ function exclude_from_json (key, value) {
 exports.TODOItem = TODOItem;
 
 exports.HMailItem = HMailItem;
-
-exports.lookup_mx = require('./mx_lookup').lookup_mx;

package/outbound/qfile.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const os = require('os');
+const os = require('node:os');
 const platform_dot = `${(['win32','win64'].includes(process.platform)) ? '' : '__tmp__'}.`;
 
 let QFILECOUNTER = 0;

package/outbound/queue.js

@@ -1,10 +1,11 @@
 'use strict';
 
-const async       = require('async');
-const fs          = require('fs');
-const path        = require('path');
+const child_process = require('node:child_process');
+const fs          = require('node:fs');
+const path        = require('node:path');
 
-const { Address }     = require('address-rfc2821');
+const async       = require('async');
+const { Address } = require('address-rfc2821');
 const config      = require('haraka-config');
 
 const logger      = require('../logger');
@@ -14,6 +15,8 @@ const obc         = require('./config');
 const _qfile      = require('./qfile');
 const obtls       = require('./tls');
 
+exports.name = 'outbound/queue';
+
 let queue_dir;
 if (config.get('queue_dir')) {
     queue_dir = path.resolve(config.get('queue_dir'));
@@ -22,7 +25,7 @@ else if (process.env.HARAKA) {
     queue_dir = path.resolve(process.env.HARAKA, 'queue');
 }
 else {
-    queue_dir = path.resolve('tests', 'test-queue');
+    queue_dir = path.resolve('test', 'test-queue');
 }
 
 exports.queue_dir = queue_dir;
@@ -72,40 +75,43 @@ exports.stat_queue = cb => {
 exports.load_queue = pid => {
     // Initialise and load queue
     // This function is called first when not running under cluster,
-    // so we create the queue directory if it doesn't already exist.
     exports.ensure_queue_dir();
     exports.delete_dot_files();
 
     exports._load_cur_queue(pid, exports._add_file, () => {
-        logger.loginfo(`[outbound] [pid: ${pid}] ${delivery_queue.length()} files in my delivery queue`);
-        logger.loginfo(`[outbound] [pid: ${pid}] ${load_queue.length()} files in my load queue`);
-        logger.loginfo(`[outbound] [pid: ${pid}] ${temp_fail_queue.length()} files in my temp fail queue`);
+        logger.info(exports, `[pid: ${pid}] ${delivery_queue.length()} files in my delivery queue`);
+        logger.info(exports, `[pid: ${pid}] ${load_queue.length()} files in my load queue`);
+        logger.info(exports, `[pid: ${pid}] ${temp_fail_queue.length()} files in my temp fail queue`);
     });
 }
 
 exports._load_cur_queue = (pid, iteratee, cb) => {
-    const self = exports;
-    logger.loginfo("[outbound] Loading outbound queue from ", queue_dir);
+    logger.info(exports, "Loading outbound queue from ", queue_dir);
     fs.readdir(queue_dir, (err, files) => {
         if (err) {
-            return logger.logerror(`[outbound] Failed to load queue directory (${queue_dir}): ${err}`);
+            return logger.error(exports, `Failed to load queue directory (${queue_dir}): ${err}`);
         }
 
-        self.cur_time = new Date(); // set once so we're not calling it a lot
+        this.cur_time = new Date(); // set once so we're not calling it a lot
 
-        self.load_queue_files(pid, files, iteratee, cb);
+        this.load_queue_files(pid, files, iteratee, cb);
     });
 }
 
 exports.read_parts = file => {
     if (file.indexOf(_qfile.platformDOT) === 0) {
-        logger.logwarn(`[outbound] 'Skipping' dot-file in queue folder: ${file}`);
+        logger.warn(exports, `'Skipping' dot-file in queue folder: ${file}`);
+        return false;
+    }
+
+    if (file.startsWith('error.')) {
+        logger.warn(exports, `'Skipping' error file in queue folder: ${file}`);
         return false;
     }
 
     const parts = _qfile.parts(file);
     if (!parts) {
-        logger.logerror(`[outbound] Unrecognized file in queue folder: ${file}`);
+        logger.error(exports, `Unrecognized file in queue folder: ${file}`);
         return false;
     }
 
@@ -135,11 +141,11 @@ exports._add_file = (file, cb) => {
     const parts = _qfile.parts(file);
 
     if (parts.next_attempt <= self.cur_time) {
-        logger.logdebug(`[outbound] File ${file} needs processing now`);
+        logger.debug(exports, `File ${file} needs processing now`);
         load_queue.push(file);
     }
     else {
-        logger.logdebug(`[outbound] File ${file} needs processing later: ${parts.next_attempt - self.cur_time}ms`);
+        logger.debug(exports, `File ${file} needs processing later: ${parts.next_attempt - self.cur_time}ms`);
         temp_fail_queue.add(file, parts.next_attempt - self.cur_time, () => { load_queue.push(file);});
     }
 
@@ -154,10 +160,10 @@ exports.load_queue_files = (pid, input_files, iteratee, callback = function () {
     let stat_loaded = 0;
 
     if (searchPid) {
-        logger.loginfo(`[outbound] Grabbing queue files for pid: ${pid}`);
+        logger.info(exports, `Grabbing queue files for pid: ${pid}`);
     }
     else {
-        logger.loginfo("[outbound] Loading the queue...");
+        logger.info(exports, "Loading the queue...");
     }
 
     async.map(input_files, (file, cb) => {
@@ -169,7 +175,7 @@ exports.load_queue_files = (pid, input_files, iteratee, callback = function () {
 
             self.rename_to_actual_pid(file, parts, (error, renamed_file) => {
                 if (error) {
-                    logger.logerror(`[outbound] ${error}`);
+                    logger.error(exports, `${error}`);
                     return cb();
                 }
 
@@ -184,16 +190,15 @@ exports.load_queue_files = (pid, input_files, iteratee, callback = function () {
         }
 
     }, (err, results) => {
-        if (err) logger.logerr(`[outbound] [pid: ${pid}] ${err}`);
-        if (searchPid) logger.loginfo(`[outbound] [pid: ${pid}] ${stat_renamed} files old PID queue fixed up`);
-        logger.logdebug(`[outbound] [pid: ${pid}] ${stat_loaded} files loaded`);
+        if (err) logger.err(exports, `[pid: ${pid}] ${err}`);
+        if (searchPid) logger.info(exports, `[pid: ${pid}] ${stat_renamed} files old PID queue fixed up`);
+        logger.debug(exports, `[pid: ${pid}] ${stat_loaded} files loaded`);
 
         async.map(results.filter((i) => i), iteratee, callback);
     });
 }
 
 exports.stats = () => {
-
     return {
         queue_dir,
         queue_count,
@@ -218,7 +223,7 @@ exports._list_file = (file, cb) => {
                 // we read everything
                 const todo_struct = JSON.parse(todo);
                 todo_struct.rcpt_to = todo_struct.rcpt_to.map(a => new Address (a));
-                todo_struct.mail_from = new Address (todo_struct.mail_from);
+                todo_struct.mail_from = new Address(todo_struct.mail_from);
                 todo_struct.file = file;
                 todo_struct.full_path = path.join(queue_dir, file);
                 const parts = _qfile.parts(file);
@@ -238,13 +243,13 @@ exports._list_file = (file, cb) => {
 exports.flush_queue = (domain, pid) => {
     if (domain) {
         exports.list_queue((err, qlist) => {
-            if (err) return logger.logerror(`[outbound] Failed to load queue: ${err}`);
-            qlist.forEach(todo => {
+            if (err) return logger.error(exports, `Failed to load queue: ${err}`);
+            for (const todo of qlist) {
                 if (todo.domain.toLowerCase() != domain.toLowerCase()) return;
                 if (pid && todo.pid != pid) return;
                 // console.log("requeue: ", todo);
                 delivery_queue.push(new HMailItem(todo.file, todo.full_path));
-            });
+            }
         })
     }
     else {
@@ -253,36 +258,44 @@ exports.flush_queue = (domain, pid) => {
 }
 
 exports.load_pid_queue = pid => {
-    logger.loginfo(`[outbound] Loading queue for pid: ${pid}`);
+    logger.info(exports, `Loading queue for pid: ${pid}`);
     exports.load_queue(pid);
 }
 
 exports.ensure_queue_dir = () => {
-    // No reason to do this asynchronously
     // this code is only run at start-up.
     if (fs.existsSync(queue_dir)) return;
 
-    logger.logdebug(`[outbound] Creating queue directory ${queue_dir}`);
+    logger.debug(exports, `Creating queue directory ${queue_dir}`);
     try {
         fs.mkdirSync(queue_dir, 493); // 493 == 0755
+        const cfg = config.get('smtp.ini');
+        let uid
+        let gid
+        if (cfg.user) uid = child_process.execSync(`id -u ${cfg.user}`).toString().trim();
+        if (cfg.group) gid = child_process.execSync(`id -g ${cfg.group}`).toString().trim();
+        if (uid && gid) {
+            fs.chown(queue_dir, uid, gid)
+        }
+        else if (uid) {
+            fs.chown(queue_dir, uid)
+        }
     }
     catch (err) {
         if (err.code !== 'EEXIST') {
-            logger.logerror(`[outbound] Error creating queue directory: ${err}`);
+            logger.error(exports, `Error creating queue directory: ${err}`);
             throw err;
         }
     }
 }
 
 exports.delete_dot_files = () => {
-    const files = fs.readdirSync(queue_dir);
-
-    files.forEach(file => {
+    for (const file of fs.readdirSync(queue_dir)) {
         if (file.indexOf(_qfile.platformDOT) === 0) {
-            logger.logwarn(`[outbound] Removing left over dot-file: ${file}`);
+            logger.warn(exports, `Removing left over dot-file: ${file}`);
             return fs.unlinkSync(path.join(queue_dir, file));
         }
-    });
+    }
 }
 
 exports._add_hmail = hmail => {
@@ -299,23 +312,22 @@ exports._add_hmail = hmail => {
 exports.scan_queue_pids = cb => {
     const self = exports;
 
-    // Under cluster, this is called first by the master so
-    // we create the queue directory if it doesn't exist.
+    // Under cluster, this is called first by the master
     self.ensure_queue_dir();
     self.delete_dot_files();
 
     fs.readdir(queue_dir, (err, files) => {
         if (err) {
-            logger.logerror(`[outbound] Failed to load queue directory (${queue_dir}): ${err}`);
+            logger.error(exports, `Failed to load queue directory (${queue_dir}): ${err}`);
             return cb(err);
         }
 
         const pids = {};
 
-        files.forEach(file => {
+        for (const file of files) {
             const parts = self.read_parts(file);
             if (parts) pids[parts.pid] = true;
-        });
+        }
 
         return cb(null, Object.keys(pids));
     });

package/outbound/timer_queue.js

@@ -19,8 +19,10 @@ class TQTimer {
 class TimerQueue {
 
     constructor (interval = 1000) {
+        this.name = 'outbound/timer_queue'
         this.queue = [];
         this.interval_timer = setInterval(() => { this.fire(); }, interval);
+        this.interval_timer.unref() // allow server to exit
     }
 
     add (id, ms, cb) {
@@ -71,7 +73,7 @@ class TimerQueue {
     }
 
     drain () {
-        logger.logdebug(`Draining ${this.queue.length} items from the queue`);
+        logger.debug(this, `Draining ${this.queue.length} items from the queue`);
         while (this.queue.length) {
             const to_run = this.queue.shift();
             if (to_run.cb) to_run.cb();
@@ -84,4 +86,3 @@ class TimerQueue {
 }
 
 module.exports = TimerQueue;
-

package/outbound/tls.js

@@ -1,10 +1,13 @@
 'use strict';
 
-const logger       = require('../logger');
-const tls_socket   = require('../tls_socket');
+const net          = require('node:net')
+
 const config       = require('haraka-config');
 const hkredis      = require('haraka-plugin-redis');
 
+const logger       = require('../logger');
+const tls_socket   = require('../tls_socket');
+
 const inheritable_opts = [
     'key', 'cert', 'ciphers', 'minVersion', 'dhparam',
     'requestCert', 'honorCipherOrder', 'rejectUnauthorized',
@@ -62,14 +65,23 @@ class OutboundTLS {
         this.load_config();
         // changing this var in-flight won't work
         if (this.cfg.redis && !this.cfg.redis.disable_for_failed_hosts) return cb();
-        logger.logdebug(this, 'Will disable outbound TLS for failing TLS hosts');
+        logger.debug(this, 'Will disable outbound TLS for failing TLS hosts');
         Object.assign(this, hkredis);
         this.merge_redis_ini();
         this.init_redis_plugin(cb);
     }
 
     get_tls_options (mx) {
-        return Object.assign(this.cfg, {servername: mx.exchange});
+        // do NOT set servername to an IP address
+        if (net.isIP(mx.exchange)) {
+            // when mx.exchange looked up in DNS, from_dns has the hostname
+            if (mx.from_dns) return { ...this.cfg, servername: mx.from_dns }
+            return { ...this.cfg }
+        }
+        else {
+            // mx.exchange is a hostname
+            return { ...this.cfg, servername: mx.exchange }
+        }
     }
 
     // Check for if host is prohibited from TLS negotiation
@@ -82,7 +94,7 @@ class OutboundTLS {
                 dbr ? cb_nogo(dbr) : cb_ok();
             })
             .catch(err => {
-                this.logdebug(this, `Redis returned error: ${err}`);
+                logger.debug(this, `Redis returned error: ${err}`);
                 cb_ok();
             })
     }
@@ -93,12 +105,12 @@ class OutboundTLS {
 
         if (!this.cfg.redis.disable_for_failed_hosts) return cb();
 
-        logger.lognotice(this, `TLS connection failed. Marking ${host} as non-TLS for ${expiry} seconds`);
+        logger.notice(this, `TLS connection failed. Marking ${host} as non-TLS for ${expiry} seconds`);
 
         this.db.setEx(dbkey, expiry, (new Date()).toISOString())
             .then(cb)
             .catch(err => {
-                logger.logerror(this, `Redis returned error: ${err}`);
+                logger.error(this, `Redis returned error: ${err}`);
             })
     }
 }