haraka 0.0.33 → 3.3.1

package/outbound/queue.js

@@ -0,0 +1,399 @@
+'use strict'
+
+const child_process = require('node:child_process')
+const fs = require('node:fs/promises')
+const path = require('node:path')
+
+const { Address } = require('@haraka/email-address')
+const config = require('haraka-config')
+const utils = require('haraka-utils')
+
+const logger = require('../logger')
+const TimerQueue = utils.TimerQueue
+const HMailItem = require('./hmail')
+const obc = require('./config')
+const _qfile = require('./qfile')
+const obtls = require('./tls')
+
+class Queue {
+    constructor(worker) {
+        this.worker = worker
+        this.tasks = []
+        this.running = 0
+        this._scheduled = false
+    }
+
+    push(task) {
+        this.tasks.push(task)
+        this._schedule()
+    }
+
+    length() {
+        return this.tasks.length + this.running
+    }
+
+    _schedule() {
+        if (this._scheduled) return
+        this._scheduled = true
+        setImmediate(() => {
+            this._scheduled = false
+            this._process()
+        })
+    }
+
+    _process() {
+        while (this.running < obc.cfg.concurrency_max && this.tasks.length > 0) {
+            this.running++
+
+            this.worker(this.tasks.shift())
+                .catch((err) => {
+                    logger.error(exports, `Queue worker error: ${err}`)
+                })
+                .finally(() => {
+                    this.running--
+                    this._schedule()
+                })
+        }
+    }
+}
+
+exports.name = 'outbound/queue'
+
+if (config.get('queue_dir')) {
+    exports.queue_dir = path.resolve(config.get('queue_dir'))
+} else if (process.env.HARAKA) {
+    exports.queue_dir = path.resolve(process.env.HARAKA, 'queue')
+} else {
+    exports.queue_dir = path.resolve('test', 'test-queue')
+}
+
+const load_queue = new Queue(async (file) => {
+    const hmail = new HMailItem(file, path.join(exports.queue_dir, file))
+    exports._add_hmail(hmail)
+    await new Promise((resolve, reject) => {
+        const onReady = () => {
+            hmail.off('error', onError)
+            resolve()
+        }
+        const onError = (err) => {
+            hmail.off('ready', onReady)
+            reject(err)
+        }
+        hmail.once('ready', onReady)
+        hmail.once('error', onError)
+    })
+})
+
+let in_progress = 0
+const delivery_queue = (exports.delivery_queue = new Queue(async (hmail) => {
+    in_progress++
+    await new Promise((resolve) => {
+        hmail.next_cb = () => {
+            in_progress--
+            resolve()
+        }
+        if (obtls.cfg) {
+            hmail.send()
+        } else {
+            obtls.init(() => {
+                hmail.send()
+            })
+        }
+    })
+}))
+
+const temp_fail_queue = (exports.temp_fail_queue = new TimerQueue(1000, { logger }))
+
+let queue_count = 0
+
+exports.get_stats = () => `${in_progress}/${exports.delivery_queue.length()}/${exports.temp_fail_queue.length()}`
+
+exports.list_queue = async () => {
+    return exports._load_cur_queue(null, exports._list_file)
+}
+
+exports._stat_file = async () => {
+    queue_count++
+}
+
+exports.stat_queue = async () => {
+    await exports._load_cur_queue(null, exports._stat_file)
+    return exports.stats()
+}
+
+exports.init_queue = async (pid) => {
+    // Initialise and load queue
+    // This function is called first when not running under cluster,
+    await exports.ensure_queue_dir()
+    await exports.delete_dot_files()
+
+    await exports._load_cur_queue(pid, exports._add_file)
+    logger.info(exports, `[pid: ${pid}] ${delivery_queue.length()} files in my delivery queue`)
+    logger.info(exports, `[pid: ${pid}] ${load_queue.length()} files in my load queue`)
+    logger.info(exports, `[pid: ${pid}] ${temp_fail_queue.length()} files in my temp fail queue`)
+}
+
+exports._load_cur_queue = async (pid, iteratee) => {
+    logger.info(exports, 'Loading outbound queue from ', exports.queue_dir)
+    let files
+    try {
+        files = await fs.readdir(exports.queue_dir)
+    } catch (err) {
+        logger.error(exports, `Failed to load queue directory (${exports.queue_dir}): ${err}`)
+        throw err
+    }
+
+    exports.cur_time = new Date() // set once so we're not calling it a lot
+
+    return await exports.load_queue_files(pid, files, iteratee)
+}
+
+exports.read_parts = (file) => {
+    if (file.startsWith(_qfile.platformDOT)) {
+        logger.warn(exports, `'Skipping' dot-file in queue folder: ${file}`)
+        return false
+    }
+
+    if (file.startsWith('error.')) {
+        logger.warn(exports, `'Skipping' error file in queue folder: ${file}`)
+        return false
+    }
+
+    const parts = _qfile.parts(file)
+    if (!parts) {
+        logger.error(exports, `Unrecognized file in queue folder: ${file}`)
+        return false
+    }
+
+    return parts
+}
+
+exports.rename_to_actual_pid = async (file, parts) => {
+    // maintain some original details for the rename
+    const new_filename = _qfile.name({
+        arrival: parts.arrival,
+        uid: parts.uid,
+        next_attempt: parts.next_attempt,
+        attempts: parts.attempts,
+    })
+
+    try {
+        await fs.rename(path.join(exports.queue_dir, file), path.join(exports.queue_dir, new_filename))
+        return new_filename
+    } catch (err) {
+        throw new Error(`Unable to rename queue file: ${file} to ${new_filename}`, {
+            cause: err,
+        })
+    }
+}
+
+exports._add_file = async (file) => {
+    const parts = _qfile.parts(file)
+
+    if (parts.next_attempt <= exports.cur_time) {
+        logger.debug(exports, `File ${file} needs processing now`)
+        load_queue.push(file)
+    } else {
+        logger.debug(exports, `File ${file} needs processing later: ${parts.next_attempt - exports.cur_time}ms`)
+        temp_fail_queue.add(file, parts.next_attempt - exports.cur_time, () => {
+            load_queue.push(file)
+        })
+    }
+    return file
+}
+
+exports.load_queue_files = async (pid, input_files, iteratee) => {
+    const searchPid = parseInt(pid)
+
+    let stat_renamed = 0
+    let stat_loaded = 0
+
+    if (searchPid) {
+        logger.info(exports, `Grabbing queue files for pid: ${pid}`)
+    } else {
+        logger.info(exports, 'Loading the queue...')
+    }
+
+    const results = await Promise.all(
+        input_files.map(async (file) => {
+            const parts = exports.read_parts(file)
+            if (!parts) return null
+
+            if (!searchPid) {
+                stat_loaded++
+                return file
+            }
+
+            if (parts.pid !== searchPid) return null
+
+            try {
+                const renamed_file = await exports.rename_to_actual_pid(file, parts)
+                stat_renamed++
+                stat_loaded++
+                return renamed_file
+            } catch (error) {
+                logger.error(exports, `${error.message}`)
+                return null
+            }
+        }),
+    )
+
+    if (searchPid) logger.info(exports, `[pid: ${pid}] ${stat_renamed} files old PID queue fixed up`)
+    logger.debug(exports, `[pid: ${pid}] ${stat_loaded} files loaded`)
+
+    const iterateeResults = await Promise.all(results.filter((i) => i).map(async (item) => await iteratee(item)))
+
+    return iterateeResults.filter((result) => result !== null && result !== undefined)
+}
+
+exports.stats = () => {
+    return {
+        queue_dir: exports.queue_dir,
+        queue_count,
+    }
+}
+
+// position `position`. Loops to handle partial reads.
+// Read exactly `length` bytes into `buffer` starting at `offset`, from file
+async function readFull(handle, buffer, offset, length, position) {
+    let totalRead = 0
+    while (totalRead < length) {
+        const { bytesRead } = await handle.read(buffer, offset + totalRead, length - totalRead, position + totalRead)
+        if (bytesRead === 0) {
+            throw new Error(`Unexpected end of file: read ${totalRead} of ${length} bytes`)
+        }
+        totalRead += bytesRead
+    }
+}
+
+exports._list_file = async (file) => {
+    let handle
+    try {
+        const filePath = path.join(exports.queue_dir, file)
+
+        handle = await fs.open(filePath, 'r')
+
+        // Read first 4 bytes to get the todo length
+        const buf = Buffer.alloc(4)
+        await readFull(handle, buf, 0, 4, 0)
+        const todo_len = (buf[0] << 24) + (buf[1] << 16) + (buf[2] << 8) + buf[3]
+
+        const todoBuf = Buffer.alloc(todo_len)
+        await readFull(handle, todoBuf, 0, todo_len, 4)
+
+        const todo = todoBuf.toString('utf8')
+        const todo_struct = JSON.parse(todo)
+        todo_struct.rcpt_to = todo_struct.rcpt_to.map((a) => new Address(a))
+        todo_struct.mail_from = new Address(todo_struct.mail_from)
+        todo_struct.file = file
+        todo_struct.full_path = filePath
+        const parts = _qfile.parts(file)
+        todo_struct.pid = parts?.pid || null
+        return todo_struct
+    } catch (err) {
+        console.error(`Error reading queue file: ${file}:`, err)
+        return null
+    } finally {
+        if (handle)
+            await handle.close().catch((err) => console.error(`Failed to close queue file handle for ${file}:`, err))
+    }
+}
+
+exports.flush_queue = async (domain, pid) => {
+    if (domain) {
+        try {
+            const qlist = await exports.list_queue()
+            for (const todo of qlist) {
+                if (todo.domain.toLowerCase() !== domain.toLowerCase()) continue
+                if (pid && todo.pid !== pid) continue
+                delivery_queue.push(new HMailItem(todo.file, todo.full_path))
+            }
+        } catch (err) {
+            logger.error(exports, `Failed to load queue: ${err.message}`)
+        }
+    } else {
+        temp_fail_queue.drain()
+    }
+}
+
+exports.load_pid_queue = async (pid) => {
+    logger.info(exports, `Loading queue for pid: ${pid}`)
+    await exports.init_queue(pid)
+}
+
+exports.ensure_queue_dir = async () => {
+    // this code is only run at start-up.
+    try {
+        await fs.access(exports.queue_dir)
+        return // directory already exists
+    } catch (ignore) {
+        // directory doesn't exist, try to create it
+    }
+
+    logger.debug(exports, `Creating queue directory ${exports.queue_dir}`)
+    try {
+        await fs.mkdir(exports.queue_dir, { mode: 493 }) // 493 == 0755
+        const cfg = config.get('smtp.ini')
+        let uid
+        let gid
+        if (cfg.user) uid = parseInt(child_process.execSync(`id -u ${cfg.user}`).toString().trim(), 10)
+        if (cfg.group) gid = parseInt(child_process.execSync(`id -g ${cfg.group}`).toString().trim(), 10)
+        if (uid && gid) {
+            await fs.chown(exports.queue_dir, uid, gid)
+        } else if (uid) {
+            await fs.chown(exports.queue_dir, uid, -1)
+        }
+    } catch (err) {
+        if (err.code !== 'EEXIST') {
+            logger.error(exports, `Error creating queue directory: ${err}`)
+            throw err
+        }
+    }
+}
+
+exports.delete_dot_files = async () => {
+    try {
+        const files = await fs.readdir(exports.queue_dir)
+        for (const file of files) {
+            if (file.startsWith(_qfile.platformDOT)) {
+                logger.warn(exports, `Removing left over dot-file: ${file}`)
+                await fs.unlink(path.join(exports.queue_dir, file))
+            }
+        }
+    } catch (err) {
+        logger.error(exports, `Error deleting dot files: ${err}`)
+    }
+}
+
+exports._add_hmail = (hmail) => {
+    if (hmail.next_process <= exports.cur_time) {
+        delivery_queue.push(hmail)
+    } else {
+        temp_fail_queue.add(hmail.filename, hmail.next_process - exports.cur_time, () => {
+            delivery_queue.push(hmail)
+        })
+    }
+}
+
+exports.scan_queue_pids = async () => {
+    // Under cluster, this is called first by the master
+    await exports.ensure_queue_dir()
+    await exports.delete_dot_files()
+
+    let files
+    try {
+        files = await fs.readdir(exports.queue_dir)
+    } catch (err) {
+        logger.error(exports, `Failed to load queue directory (${exports.queue_dir}): ${err}`)
+        throw err
+    }
+
+    const pids = {}
+
+    for (const file of files) {
+        const parts = exports.read_parts(file)
+        if (parts) pids[parts.pid] = true
+    }
+
+    return Object.keys(pids)
+}

package/outbound/tls.js

@@ -0,0 +1,85 @@
+'use strict'
+
+const net = require('node:net')
+
+const config = require('haraka-config')
+const hkredis = require('haraka-plugin-redis')
+
+const logger = require('../logger')
+const tls_socket = require('../tls_socket')
+
+class OutboundTLS {
+    constructor() {
+        this.config = config
+        this.name = 'OutboundTLS'
+        logger.add_log_methods(this)
+    }
+
+    test_config(tls_config, our_config) {
+        tls_socket.config = tls_config
+        this.config = our_config
+    }
+
+    load_config() {
+        const tls_cfg = tls_socket.load_tls_ini({ role: 'client' })
+        this.cfg = tls_socket.load_plugin_tls_options(tls_cfg.outbound || {})
+        this.cfg.redis = tls_cfg.redis // outbound-only: TLS NO-GO db (don't clone — has methods)
+    }
+
+    init(cb) {
+        this.load_config()
+        // changing this var in-flight won't work
+        if (this.cfg.redis && !this.cfg.redis.disable_for_failed_hosts) return cb()
+        logger.debug(this, 'Will disable outbound TLS for failing TLS hosts')
+        Object.assign(this, hkredis)
+        this.merge_redis_ini()
+        this.init_redis_plugin(cb)
+    }
+
+    get_tls_options(mx) {
+        // do NOT set servername to an IP address
+        if (net.isIP(mx.exchange)) {
+            // when mx.exchange looked up in DNS, from_dns has the hostname
+            if (mx.from_dns) return { ...this.cfg, servername: mx.from_dns }
+            return { ...this.cfg }
+        } else {
+            // mx.exchange is a hostname
+            return { ...this.cfg, servername: mx.exchange }
+        }
+    }
+
+    // Check for if host is prohibited from TLS negotiation
+    check_tls_nogo(host, cb_ok, cb_nogo) {
+        if (!this.cfg.redis.disable_for_failed_hosts) return cb_ok()
+
+        const dbkey = `no_tls|${host}`
+        this.db
+            .get(dbkey)
+            .then((dbr) => {
+                dbr ? cb_nogo(dbr) : cb_ok()
+            })
+            .catch((err) => {
+                logger.debug(this, `Redis returned error: ${err}`)
+                cb_ok()
+            })
+    }
+
+    mark_tls_nogo(host, cb) {
+        const dbkey = `no_tls|${host}`
+        const expiry = this.cfg.redis.disable_expiry || 604800
+
+        if (!this.cfg.redis.disable_for_failed_hosts) return cb()
+
+        logger.notice(this, `TLS connection failed. Marking ${host} as non-TLS for ${expiry} seconds`)
+
+        this.db
+            .setEx(dbkey, expiry, new Date().toISOString())
+            .then(cb)
+            .catch((err) => {
+                logger.error(this, `Redis returned error: ${err}`)
+            })
+    }
+}
+
+// this is a singleton
+module.exports = new OutboundTLS()

package/outbound/todo.js

@@ -0,0 +1,17 @@
+'use strict'
+
+// queue file header data
+class TODOItem {
+    constructor(domain, recipients, transaction) {
+        this.queue_time = Date.now()
+        this.domain = domain
+        this.rcpt_to = recipients
+        this.mail_from = transaction.mail_from
+        this.message_stream = transaction.message_stream
+        this.notes = transaction.notes
+        this.uuid = transaction.uuid
+        this.force_tls = false
+    }
+}
+
+module.exports = TODOItem

package/package.json

@@ -1,7 +1,103 @@
 {
-  "deprecated": true,
-  "description": "Inactive module name",
-  "license": "MIT",
+  "author": "Matt Sergeant <helpme@gmail.com>",
   "name": "haraka",
-  "version": "0.0.33"
+  "license": "MIT",
+  "description": "An SMTP Server project.",
+  "keywords": [
+    "haraka",
+    "smtp",
+    "server",
+    "email"
+  ],
+  "version": "3.3.1",
+  "homepage": "http://haraka.github.io",
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/haraka/Haraka.git"
+  },
+  "main": "haraka.js",
+  "engines": {
+    "node": ">=20"
+  },
+  "dependencies": {
+    "@haraka/email-address": "~3.1.6",
+    "haraka-config": "~1.6.3",
+    "haraka-constants": "~1.0.8",
+    "haraka-dsn": "~1.2.0",
+    "haraka-email-message": "~1.4.0",
+    "haraka-net-utils": "~1.9.2",
+    "haraka-notes": "~1.1.3",
+    "haraka-plugin-redis": "~2.1.0",
+    "haraka-results": "~2.4.0",
+    "haraka-tld": "~1.3.6",
+    "haraka-utils": "~2.2.1",
+    "ipaddr.js": "~2.4.0",
+    "node-gyp": "~12.4.0",
+    "nopt": "~10.0.1",
+    "redis": "~6.0.0",
+    "semver": "~7.8.4"
+  },
+  "optionalDependencies": {
+    "@haraka/ocsp": "~1.2.0",
+    "haraka-plugin-access": "~1.4.0",
+    "haraka-plugin-aliases": "~1.1.0",
+    "haraka-plugin-asn": "~2.2.0",
+    "haraka-plugin-attachment": "~1.2.1",
+    "haraka-plugin-bounce": "~2.2.1",
+    "haraka-plugin-clamd": "~1.0.3",
+    "haraka-plugin-dcc": "~1.0.3",
+    "haraka-plugin-dkim": "~1.2.0",
+    "haraka-plugin-dns-list": "~1.3.0",
+    "haraka-plugin-early_talker": "~1.0.2",
+    "haraka-plugin-fcrdns": "~1.2.1",
+    "haraka-plugin-geoip": "~1.1.2",
+    "haraka-plugin-greylist": "~1.2.1",
+    "haraka-plugin-headers": "~1.2.0",
+    "haraka-plugin-helo.checks": "~1.1.3",
+    "haraka-plugin-karma": "~2.5.2",
+    "haraka-plugin-known-senders": "~1.2.0",
+    "haraka-plugin-limit": "~1.3.2",
+    "haraka-plugin-mail_from.is_resolvable": "~1.3.0",
+    "haraka-plugin-messagesniffer": "~1.0.2",
+    "haraka-plugin-qmail-deliverable": "~1.4.0",
+    "haraka-plugin-relay": "~1.0.2",
+    "haraka-plugin-rspamd": "~1.6.0",
+    "haraka-plugin-spamassassin": "~1.1.0",
+    "haraka-plugin-spf": "~1.3.0",
+    "haraka-plugin-syslog": "~1.1.2",
+    "haraka-plugin-uribl": "~2.0.0"
+  },
+  "devDependencies": {
+    "@haraka/eslint-config": "~3.0.0",
+    "haraka-test-fixtures": "^1.7.1",
+    "mock-require": "~3.0.3",
+    "toobusy-js": "^0.5.1"
+  },
+  "bugs": {
+    "mail": "haraka.mail@gmail.com",
+    "url": "https://github.com/haraka/Haraka/issues"
+  },
+  "bin": {
+    "haraka": "bin/haraka",
+    "haraka_grep": "bin/haraka_grep"
+  },
+  "scripts": {
+    "prepare": "git rev-parse --git-dir >/dev/null 2>&1 && git config core.hooksPath .githooks || true",
+    "format": "npm run prettier:fix && npm run lint:fix",
+    "lint": "npx eslint *.js outbound plugins plugins/*/*.js test test/*/*.js test/*/*/*.js",
+    "lint:fix": "npx eslint --fix *.js outbound plugins plugins/*/*.js test test/*/*.js test/*/*/*.js",
+    "prettier": "npx prettier . --check",
+    "prettier:fix": "npx prettier . --write --log-level=warn",
+    "qlty": "qlty smells --all",
+    "test": "sh ./run_tests",
+    "test:coverage": "node --test --test-concurrency=1 --experimental-test-coverage --test-coverage-include=*.js --test-coverage-include=plugins/**/*.js --test-coverage-include=outbound/*.js",
+    "test:coverage:lcov": "mkdir -p coverage && node --test --test-concurrency=1 --experimental-test-coverage --test-reporter=lcov --test-reporter-destination=coverage/lcov.info --test-coverage-include=*.js --test-coverage-include=plugins/**/*.js --test-coverage-include=outbound/*.js",
+    "versions": "npx npm-dep-mgr check",
+    "versions:fix": "npx npm-dep-mgr update"
+  },
+  "prettier": {
+    "singleQuote": true,
+    "printWidth": 120,
+    "semi": false
+  }
 }

package/plugins/.eslintrc.yaml

@@ -0,0 +1,3 @@
+globals:
+  server: true
+  NEXT_HOOK: true