Haraka 3.1.1 → 3.1.2

package/outbound/queue.js

@@ -1,121 +1,119 @@
-'use strict';
+'use strict'
 
-const child_process = require('node:child_process');
-const fs          = require('node:fs');
-const path        = require('node:path');
+const child_process = require('node:child_process')
+const fs = require('node:fs')
+const path = require('node:path')
 
-const async       = require('async');
-const { Address } = require('address-rfc2821');
-const config      = require('haraka-config');
+const async = require('async')
+const { Address } = require('address-rfc2821')
+const config = require('haraka-config')
 
-const logger      = require('../logger');
-const TimerQueue  = require('./timer_queue');
-const HMailItem   = require('./hmail');
-const obc         = require('./config');
-const _qfile      = require('./qfile');
-const obtls       = require('./tls');
+const logger = require('../logger')
+const TimerQueue = require('./timer_queue')
+const HMailItem = require('./hmail')
+const obc = require('./config')
+const _qfile = require('./qfile')
+const obtls = require('./tls')
 
-exports.name = 'outbound/queue';
+exports.name = 'outbound/queue'
 
-let queue_dir;
+let queue_dir
 if (config.get('queue_dir')) {
-    queue_dir = path.resolve(config.get('queue_dir'));
-}
-else if (process.env.HARAKA) {
-    queue_dir = path.resolve(process.env.HARAKA, 'queue');
-}
-else {
-    queue_dir = path.resolve('test', 'test-queue');
+    queue_dir = path.resolve(config.get('queue_dir'))
+} else if (process.env.HARAKA) {
+    queue_dir = path.resolve(process.env.HARAKA, 'queue')
+} else {
+    queue_dir = path.resolve('test', 'test-queue')
 }
 
-exports.queue_dir = queue_dir;
+exports.queue_dir = queue_dir
 
 const load_queue = async.queue((file, cb) => {
-    const hmail = new HMailItem(file, path.join(queue_dir, file));
-    exports._add_hmail(hmail);
-    hmail.once('ready', cb);
-}, obc.cfg.concurrency_max);
-
-let in_progress = 0;
-const delivery_queue = exports.delivery_queue = async.queue((hmail, cb) => {
-    in_progress++;
+    const hmail = new HMailItem(file, path.join(queue_dir, file))
+    exports._add_hmail(hmail)
+    hmail.once('ready', cb)
+}, obc.cfg.concurrency_max)
+
+let in_progress = 0
+const delivery_queue = (exports.delivery_queue = async.queue((hmail, cb) => {
+    in_progress++
     hmail.next_cb = () => {
-        in_progress--;
-        cb();
-    };
-    if (obtls.cfg) return hmail.send();
+        in_progress--
+        cb()
+    }
+    if (obtls.cfg) return hmail.send()
     obtls.init(() => {
-        hmail.send();
-    });
-}, obc.cfg.concurrency_max);
+        hmail.send()
+    })
+}, obc.cfg.concurrency_max))
 
-const temp_fail_queue = exports.temp_fail_queue = new TimerQueue();
+const temp_fail_queue = (exports.temp_fail_queue = new TimerQueue())
 
-let queue_count = 0;
+let queue_count = 0
 
-exports.get_stats = () => `${in_progress}/${exports.delivery_queue.length()}/${exports.temp_fail_queue.length()}`;
+exports.get_stats = () => `${in_progress}/${exports.delivery_queue.length()}/${exports.temp_fail_queue.length()}`
 
-exports.list_queue = cb => {
-    exports._load_cur_queue(null, exports._list_file, cb);
+exports.list_queue = (cb) => {
+    exports._load_cur_queue(null, exports._list_file, cb)
 }
 
 exports._stat_file = (file, cb) => {
-    queue_count++;
-    setImmediate(cb);
+    queue_count++
+    setImmediate(cb)
 }
 
-exports.stat_queue = cb => {
-    const self = exports;
-    exports._load_cur_queue(null, exports._stat_file, err => {
-        if (err) return cb(err);
-        return cb(null, self.stats());
-    });
+exports.stat_queue = (cb) => {
+    const self = exports
+    exports._load_cur_queue(null, exports._stat_file, (err) => {
+        if (err) return cb(err)
+        return cb(null, self.stats())
+    })
 }
 
-exports.load_queue = pid => {
+exports.load_queue = (pid) => {
     // Initialise and load queue
     // This function is called first when not running under cluster,
-    exports.ensure_queue_dir();
-    exports.delete_dot_files();
+    exports.ensure_queue_dir()
+    exports.delete_dot_files()
 
     exports._load_cur_queue(pid, exports._add_file, () => {
-        logger.info(exports, `[pid: ${pid}] ${delivery_queue.length()} files in my delivery queue`);
-        logger.info(exports, `[pid: ${pid}] ${load_queue.length()} files in my load queue`);
-        logger.info(exports, `[pid: ${pid}] ${temp_fail_queue.length()} files in my temp fail queue`);
-    });
+        logger.info(exports, `[pid: ${pid}] ${delivery_queue.length()} files in my delivery queue`)
+        logger.info(exports, `[pid: ${pid}] ${load_queue.length()} files in my load queue`)
+        logger.info(exports, `[pid: ${pid}] ${temp_fail_queue.length()} files in my temp fail queue`)
+    })
 }
 
 exports._load_cur_queue = (pid, iteratee, cb) => {
-    logger.info(exports, "Loading outbound queue from ", queue_dir);
+    logger.info(exports, 'Loading outbound queue from ', queue_dir)
     fs.readdir(queue_dir, (err, files) => {
         if (err) {
-            return logger.error(exports, `Failed to load queue directory (${queue_dir}): ${err}`);
+            return logger.error(exports, `Failed to load queue directory (${queue_dir}): ${err}`)
         }
 
-        this.cur_time = new Date(); // set once so we're not calling it a lot
+        this.cur_time = new Date() // set once so we're not calling it a lot
 
-        this.load_queue_files(pid, files, iteratee, cb);
-    });
+        this.load_queue_files(pid, files, iteratee, cb)
+    })
 }
 
-exports.read_parts = file => {
+exports.read_parts = (file) => {
     if (file.indexOf(_qfile.platformDOT) === 0) {
-        logger.warn(exports, `'Skipping' dot-file in queue folder: ${file}`);
-        return false;
+        logger.warn(exports, `'Skipping' dot-file in queue folder: ${file}`)
+        return false
     }
 
     if (file.startsWith('error.')) {
-        logger.warn(exports, `'Skipping' error file in queue folder: ${file}`);
-        return false;
+        logger.warn(exports, `'Skipping' error file in queue folder: ${file}`)
+        return false
     }
 
-    const parts = _qfile.parts(file);
+    const parts = _qfile.parts(file)
     if (!parts) {
-        logger.error(exports, `Unrecognized file in queue folder: ${file}`);
-        return false;
+        logger.error(exports, `Unrecognized file in queue folder: ${file}`)
+        return false
     }
 
-    return parts;
+    return parts
 }
 
 exports.rename_to_actual_pid = (file, parts, cb) => {
@@ -125,166 +123,176 @@ exports.rename_to_actual_pid = (file, parts, cb) => {
         uid: parts.uid,
         next_attempt: parts.next_attempt,
         attempts: parts.attempts,
-    });
+    })
 
-    fs.rename(path.join(queue_dir, file), path.join(queue_dir, new_filename), err => {
+    fs.rename(path.join(queue_dir, file), path.join(queue_dir, new_filename), (err) => {
         if (err) {
-            return cb(`Unable to rename queue file: ${file} to ${new_filename} : ${err}`);
+            return cb(`Unable to rename queue file: ${file} to ${new_filename} : ${err}`)
         }
 
-        cb(null, new_filename);
-    });
+        cb(null, new_filename)
+    })
 }
 
 exports._add_file = (file, cb) => {
-    const self = exports;
-    const parts = _qfile.parts(file);
+    const self = exports
+    const parts = _qfile.parts(file)
 
     if (parts.next_attempt <= self.cur_time) {
-        logger.debug(exports, `File ${file} needs processing now`);
-        load_queue.push(file);
-    }
-    else {
-        logger.debug(exports, `File ${file} needs processing later: ${parts.next_attempt - self.cur_time}ms`);
-        temp_fail_queue.add(file, parts.next_attempt - self.cur_time, () => { load_queue.push(file);});
+        logger.debug(exports, `File ${file} needs processing now`)
+        load_queue.push(file)
+    } else {
+        logger.debug(exports, `File ${file} needs processing later: ${parts.next_attempt - self.cur_time}ms`)
+        temp_fail_queue.add(file, parts.next_attempt - self.cur_time, () => {
+            load_queue.push(file)
+        })
     }
 
-    cb();
+    cb()
 }
 
 exports.load_queue_files = (pid, input_files, iteratee, callback = function () {}) => {
-    const self = exports;
-    const searchPid = parseInt(pid);
+    const self = exports
+    const searchPid = parseInt(pid)
 
-    let stat_renamed = 0;
-    let stat_loaded = 0;
+    let stat_renamed = 0
+    let stat_loaded = 0
 
     if (searchPid) {
-        logger.info(exports, `Grabbing queue files for pid: ${pid}`);
-    }
-    else {
-        logger.info(exports, "Loading the queue...");
+        logger.info(exports, `Grabbing queue files for pid: ${pid}`)
+    } else {
+        logger.info(exports, 'Loading the queue...')
     }
 
-    async.map(input_files, (file, cb) => {
-        const parts = self.read_parts(file);
-        if (!parts) return cb();
-
-        if (searchPid) {
-            if (parts.pid !== searchPid) return cb();
-
-            self.rename_to_actual_pid(file, parts, (error, renamed_file) => {
-                if (error) {
-                    logger.error(exports, `${error}`);
-                    return cb();
-                }
-
-                stat_renamed++;
-                stat_loaded++;
-                cb(null, renamed_file);
-            });
-        }
-        else {
-            stat_loaded++;
-            cb(null, file);
-        }
-
-    }, (err, results) => {
-        if (err) logger.err(exports, `[pid: ${pid}] ${err}`);
-        if (searchPid) logger.info(exports, `[pid: ${pid}] ${stat_renamed} files old PID queue fixed up`);
-        logger.debug(exports, `[pid: ${pid}] ${stat_loaded} files loaded`);
-
-        async.map(results.filter((i) => i), iteratee, callback);
-    });
+    async.map(
+        input_files,
+        (file, cb) => {
+            const parts = self.read_parts(file)
+            if (!parts) return cb()
+
+            if (searchPid) {
+                if (parts.pid !== searchPid) return cb()
+
+                self.rename_to_actual_pid(file, parts, (error, renamed_file) => {
+                    if (error) {
+                        logger.error(exports, `${error}`)
+                        return cb()
+                    }
+
+                    stat_renamed++
+                    stat_loaded++
+                    cb(null, renamed_file)
+                })
+            } else {
+                stat_loaded++
+                cb(null, file)
+            }
+        },
+        (err, results) => {
+            if (err) logger.err(exports, `[pid: ${pid}] ${err}`)
+            if (searchPid) logger.info(exports, `[pid: ${pid}] ${stat_renamed} files old PID queue fixed up`)
+            logger.debug(exports, `[pid: ${pid}] ${stat_loaded} files loaded`)
+
+            async.map(
+                results.filter((i) => i),
+                iteratee,
+                callback,
+            )
+        },
+    )
 }
 
 exports.stats = () => {
     return {
         queue_dir,
         queue_count,
-    };
+    }
 }
 
 exports._list_file = (file, cb) => {
-    const tl_reader = fs.createReadStream(path.join(queue_dir, file), {start: 0, end: 3});
-    tl_reader.on('error', err => {
-        console.error(`Error reading queue file: ${file}:`, err);
-    });
-    tl_reader.once('data', buf => {
+    const tl_reader = fs.createReadStream(path.join(queue_dir, file), {
+        start: 0,
+        end: 3,
+    })
+    tl_reader.on('error', (err) => {
+        console.error(`Error reading queue file: ${file}:`, err)
+    })
+    tl_reader.once('data', (buf) => {
         // I'm making the assumption here we won't ever read less than 4 bytes
         // as no filesystem on the planet should be that dumb...
-        tl_reader.destroy();
-        const todo_len = (buf[0] << 24) + (buf[1] << 16) + (buf[2] << 8) + buf[3];
-        const td_reader = fs.createReadStream(path.join(queue_dir, file), {encoding: 'utf8', start: 4, end: todo_len + 3});
-        let todo = '';
-        td_reader.on('data', str => {
-            todo += str;
+        tl_reader.destroy()
+        const todo_len = (buf[0] << 24) + (buf[1] << 16) + (buf[2] << 8) + buf[3]
+        const td_reader = fs.createReadStream(path.join(queue_dir, file), {
+            encoding: 'utf8',
+            start: 4,
+            end: todo_len + 3,
+        })
+        let todo = ''
+        td_reader.on('data', (str) => {
+            todo += str
             if (Buffer.byteLength(todo) === todo_len) {
                 // we read everything
-                const todo_struct = JSON.parse(todo);
-                todo_struct.rcpt_to = todo_struct.rcpt_to.map(a => new Address (a));
-                todo_struct.mail_from = new Address(todo_struct.mail_from);
-                todo_struct.file = file;
-                todo_struct.full_path = path.join(queue_dir, file);
-                const parts = _qfile.parts(file);
-                todo_struct.pid = (parts?.pid) || null;
-                cb(null, todo_struct);
+                const todo_struct = JSON.parse(todo)
+                todo_struct.rcpt_to = todo_struct.rcpt_to.map((a) => new Address(a))
+                todo_struct.mail_from = new Address(todo_struct.mail_from)
+                todo_struct.file = file
+                todo_struct.full_path = path.join(queue_dir, file)
+                const parts = _qfile.parts(file)
+                todo_struct.pid = parts?.pid || null
+                cb(null, todo_struct)
             }
-        });
+        })
         td_reader.on('end', () => {
             if (Buffer.byteLength(todo) !== todo_len) {
-                console.error("Didn't find right amount of data in todo for file:", file);
-                return cb();
+                console.error("Didn't find right amount of data in todo for file:", file)
+                return cb()
             }
-        });
-    });
+        })
+    })
 }
 
 exports.flush_queue = (domain, pid) => {
     if (domain) {
         exports.list_queue((err, qlist) => {
-            if (err) return logger.error(exports, `Failed to load queue: ${err}`);
+            if (err) return logger.error(exports, `Failed to load queue: ${err}`)
             for (const todo of qlist) {
-                if (todo.domain.toLowerCase() != domain.toLowerCase()) return;
-                if (pid && todo.pid != pid) return;
+                if (todo.domain.toLowerCase() != domain.toLowerCase()) return
+                if (pid && todo.pid != pid) return
                 // console.log("requeue: ", todo);
-                delivery_queue.push(new HMailItem(todo.file, todo.full_path));
+                delivery_queue.push(new HMailItem(todo.file, todo.full_path))
             }
         })
-    }
-    else {
-        temp_fail_queue.drain();
+    } else {
+        temp_fail_queue.drain()
     }
 }
 
-exports.load_pid_queue = pid => {
-    logger.info(exports, `Loading queue for pid: ${pid}`);
-    exports.load_queue(pid);
+exports.load_pid_queue = (pid) => {
+    logger.info(exports, `Loading queue for pid: ${pid}`)
+    exports.load_queue(pid)
 }
 
 exports.ensure_queue_dir = () => {
     // this code is only run at start-up.
-    if (fs.existsSync(queue_dir)) return;
+    if (fs.existsSync(queue_dir)) return
 
-    logger.debug(exports, `Creating queue directory ${queue_dir}`);
+    logger.debug(exports, `Creating queue directory ${queue_dir}`)
     try {
-        fs.mkdirSync(queue_dir, 493); // 493 == 0755
-        const cfg = config.get('smtp.ini');
+        fs.mkdirSync(queue_dir, 493) // 493 == 0755
+        const cfg = config.get('smtp.ini')
         let uid
         let gid
-        if (cfg.user) uid = child_process.execSync(`id -u ${cfg.user}`).toString().trim();
-        if (cfg.group) gid = child_process.execSync(`id -g ${cfg.group}`).toString().trim();
+        if (cfg.user) uid = child_process.execSync(`id -u ${cfg.user}`).toString().trim()
+        if (cfg.group) gid = child_process.execSync(`id -g ${cfg.group}`).toString().trim()
         if (uid && gid) {
             fs.chown(queue_dir, uid, gid)
-        }
-        else if (uid) {
+        } else if (uid) {
             fs.chown(queue_dir, uid)
         }
-    }
-    catch (err) {
+    } catch (err) {
         if (err.code !== 'EEXIST') {
-            logger.error(exports, `Error creating queue directory: ${err}`);
-            throw err;
+            logger.error(exports, `Error creating queue directory: ${err}`)
+            throw err
         }
     }
 }
@@ -292,43 +300,42 @@ exports.ensure_queue_dir = () => {
 exports.delete_dot_files = () => {
     for (const file of fs.readdirSync(queue_dir)) {
         if (file.indexOf(_qfile.platformDOT) === 0) {
-            logger.warn(exports, `Removing left over dot-file: ${file}`);
-            return fs.unlinkSync(path.join(queue_dir, file));
+            logger.warn(exports, `Removing left over dot-file: ${file}`)
+            return fs.unlinkSync(path.join(queue_dir, file))
         }
     }
 }
 
-exports._add_hmail = hmail => {
+exports._add_hmail = (hmail) => {
     if (hmail.next_process < exports.cur_time) {
-        delivery_queue.push(hmail);
-    }
-    else {
+        delivery_queue.push(hmail)
+    } else {
         temp_fail_queue.add(hmail.filename, hmail.next_process - exports.cur_time, () => {
-            delivery_queue.push(hmail);
-        });
+            delivery_queue.push(hmail)
+        })
     }
 }
 
-exports.scan_queue_pids = cb => {
-    const self = exports;
+exports.scan_queue_pids = (cb) => {
+    const self = exports
 
     // Under cluster, this is called first by the master
-    self.ensure_queue_dir();
-    self.delete_dot_files();
+    self.ensure_queue_dir()
+    self.delete_dot_files()
 
     fs.readdir(queue_dir, (err, files) => {
         if (err) {
-            logger.error(exports, `Failed to load queue directory (${queue_dir}): ${err}`);
-            return cb(err);
+            logger.error(exports, `Failed to load queue directory (${queue_dir}): ${err}`)
+            return cb(err)
         }
 
-        const pids = {};
+        const pids = {}
 
         for (const file of files) {
-            const parts = self.read_parts(file);
-            if (parts) pids[parts.pid] = true;
+            const parts = self.read_parts(file)
+            if (parts) pids[parts.pid] = true
         }
 
-        return cb(null, Object.keys(pids));
-    });
+        return cb(null, Object.keys(pids))
+    })
 }

package/outbound/timer_queue.js

@@ -1,88 +1,86 @@
-'use strict';
+'use strict'
 
-const logger = require('../logger');
+const logger = require('../logger')
 
 class TQTimer {
-
-    constructor (id, fire_time, cb) {
-        this.id = id;
-        this.fire_time = fire_time;
-        this.cb = cb;
+    constructor(id, fire_time, cb) {
+        this.id = id
+        this.fire_time = fire_time
+        this.cb = cb
     }
 
-    cancel () {
-        this.cb = null;
+    cancel() {
+        this.cb = null
     }
-
 }
 
 class TimerQueue {
-
-    constructor (interval = 1000) {
+    constructor(interval = 1000) {
         this.name = 'outbound/timer_queue'
-        this.queue = [];
-        this.interval_timer = setInterval(() => { this.fire(); }, interval);
+        this.queue = []
+        this.interval_timer = setInterval(() => {
+            this.fire()
+        }, interval)
         this.interval_timer.unref() // allow server to exit
     }
 
-    add (id, ms, cb) {
-        const fire_time = Date.now() + ms;
+    add(id, ms, cb) {
+        const fire_time = Date.now() + ms
 
-        const timer = new TQTimer(id, fire_time, cb);
+        const timer = new TQTimer(id, fire_time, cb)
 
-        if ((this.queue.length === 0) ||
-            fire_time >= this.queue[this.queue.length - 1].fire_time) {
-            this.queue.push(timer);
-            return timer;
+        if (this.queue.length === 0 || fire_time >= this.queue[this.queue.length - 1].fire_time) {
+            this.queue.push(timer)
+            return timer
         }
 
-        for (let i=0; i < this.queue.length; i++) {
+        for (let i = 0; i < this.queue.length; i++) {
             if (this.queue[i].fire_time > fire_time) {
-                this.queue.splice(i, 0, timer);
-                return timer;
+                this.queue.splice(i, 0, timer)
+                return timer
             }
         }
 
-        throw "Should never get here";
+        throw 'Should never get here'
     }
 
-    discard (id) {
+    discard(id) {
         for (let i = 0; i < this.queue.length; i++) {
             if (this.queue[i].id === id) {
-                this.queue[i].cancel();
-                return this.queue.splice(i, 1);
+                this.queue[i].cancel()
+                return this.queue.splice(i, 1)
             }
         }
 
-        throw `${id} not found`;
+        throw `${id} not found`
     }
 
-    fire () {
-        if (this.queue.length === 0) return;
+    fire() {
+        if (this.queue.length === 0) return
 
-        const now = Date.now();
+        const now = Date.now()
 
         while (this.queue.length && this.queue[0].fire_time <= now) {
-            const to_run = this.queue.shift();
-            if (to_run.cb) to_run.cb();
+            const to_run = this.queue.shift()
+            if (to_run.cb) to_run.cb()
         }
     }
 
-    length () {
-        return this.queue.length;
+    length() {
+        return this.queue.length
     }
 
-    drain () {
-        logger.debug(this, `Draining ${this.queue.length} items from the queue`);
+    drain() {
+        logger.debug(this, `Draining ${this.queue.length} items from the queue`)
         while (this.queue.length) {
-            const to_run = this.queue.shift();
-            if (to_run.cb) to_run.cb();
+            const to_run = this.queue.shift()
+            if (to_run.cb) to_run.cb()
         }
     }
 
-    shutdown () {
-        clearInterval(this.interval_timer);
+    shutdown() {
+        clearInterval(this.interval_timer)
     }
 }
 
-module.exports = TimerQueue;
+module.exports = TimerQueue