Haraka 3.1.5 → 3.1.6

package/test/connection.js

@@ -5,6 +5,7 @@ const assert = require('node:assert/strict')
 
 const constants = require('haraka-constants')
 const DSN = require('haraka-dsn')
+const { Address } = require('address-rfc2821')
 
 const connection = require('../connection')
 const Server = require('../server')
@@ -442,4 +443,237 @@ describe('connection', () => {
             })
         })
     })
+
+    describe('queue responses', () => {
+        beforeEach(setUp)
+
+        const prepQueueTestConnection = () => {
+            const calls = { respond: [], reset: 0, disconnect: 0, queue_ok: 0, results: [] }
+            const plugins = require('../plugins')
+            const originalRunHooks = plugins.run_hooks
+
+            this.connection.transaction = {
+                uuid: 'txn-123',
+                msg_status: null,
+                results: {
+                    add(_meta, payload) {
+                        calls.results.push(payload)
+                    },
+                },
+            }
+
+            this.connection.respond = (code, msg, cb) => {
+                calls.respond.push({ code, msg })
+                if (cb) cb()
+            }
+            this.connection.reset_transaction = (cb) => {
+                calls.reset++
+                this.connection.transaction = this.connection.transaction || {}
+                if (cb) cb()
+            }
+            this.connection.disconnect = () => {
+                calls.disconnect++
+            }
+            plugins.run_hooks = (hook) => {
+                if (hook === 'queue_ok') calls.queue_ok++
+            }
+
+            return {
+                calls,
+                restore() {
+                    plugins.run_hooks = originalRunHooks
+                },
+            }
+        }
+
+        it('queue_respond handles denydisconnect and marks message rejected', () => {
+            const harness = prepQueueTestConnection()
+            try {
+                this.connection.queue_respond(constants.denydisconnect)
+                assert.equal(harness.calls.respond[0].code, 550)
+                assert.equal(this.connection.msg_count.reject, 1)
+                assert.equal(this.connection.transaction.msg_status, 'rejected')
+                assert.equal(harness.calls.disconnect, 1)
+                assert.deepEqual(harness.calls.results[0], { fail: 'Message denied' })
+            } finally {
+                harness.restore()
+            }
+        })
+
+        it('queue_respond handles denysoft and resets transaction', () => {
+            const harness = prepQueueTestConnection()
+            try {
+                this.connection.queue_respond(constants.denysoft)
+                assert.equal(harness.calls.respond[0].code, 450)
+                assert.equal(this.connection.msg_count.tempfail, 1)
+                assert.equal(this.connection.transaction.msg_status, 'deferred')
+                assert.equal(harness.calls.reset, 1)
+                assert.deepEqual(harness.calls.results[0], { fail: 'Message denied temporarily' })
+            } finally {
+                harness.restore()
+            }
+        })
+
+        it('queue_respond handles denysoftdisconnect and disconnects', () => {
+            const harness = prepQueueTestConnection()
+            try {
+                this.connection.queue_respond(constants.denysoftdisconnect)
+                assert.equal(harness.calls.respond[0].code, 450)
+                assert.equal(this.connection.msg_count.tempfail, 1)
+                assert.equal(this.connection.transaction.msg_status, 'deferred')
+                assert.equal(harness.calls.disconnect, 1)
+            } finally {
+                harness.restore()
+            }
+        })
+
+        it('queue_respond default path returns 451 and resets transaction', () => {
+            const harness = prepQueueTestConnection()
+            try {
+                this.connection.queue_respond(constants.cont)
+                assert.equal(harness.calls.respond[0].code, 451)
+                assert.equal(this.connection.msg_count.tempfail, 1)
+                assert.equal(this.connection.transaction.msg_status, 'deferred')
+                assert.equal(harness.calls.reset, 1)
+            } finally {
+                harness.restore()
+            }
+        })
+
+        it('queue_ok_respond accepts and resets transaction', () => {
+            const harness = prepQueueTestConnection()
+            try {
+                this.connection.queue_ok_respond(constants.ok, null, 'queued')
+                assert.equal(harness.calls.respond[0].code, 250)
+                assert.equal(this.connection.msg_count.accept, 1)
+                assert.equal(this.connection.transaction.msg_status, 'accepted')
+                assert.equal(harness.calls.reset, 1)
+            } finally {
+                harness.restore()
+            }
+        })
+    })
+
+    describe('smtp command/response branches', () => {
+        beforeEach(setUp)
+
+        it('rcpt_respond deny removes recipient and records reject', () => {
+            const plugins = require('../plugins')
+            const originalRunHooks = plugins.run_hooks
+            const rcpt = new Address('<to@example.com>')
+            const sender = new Address('<from@example.com>')
+            const actions = []
+
+            this.connection.transaction = {
+                rcpt_to: [rcpt],
+                mail_from: sender,
+                results: { push() {} },
+            }
+            this.connection.rcpt_incr = (_rcpt, action) => actions.push(action)
+            this.connection.respond = (_code, _msg, cb) => cb && cb()
+            plugins.run_hooks = () => {}
+
+            try {
+                this.connection.rcpt_respond(constants.deny, 'no')
+                assert.equal(actions[0], 'reject')
+                assert.equal(this.connection.transaction.rcpt_to.length, 0)
+            } finally {
+                plugins.run_hooks = originalRunHooks
+            }
+        })
+
+        it('rcpt_respond ok runs rcpt_ok hook', () => {
+            const plugins = require('../plugins')
+            const originalRunHooks = plugins.run_hooks
+            const rcpt = new Address('<to@example.com>')
+            const sender = new Address('<from@example.com>')
+            const hooks = []
+
+            this.connection.transaction = {
+                rcpt_to: [rcpt],
+                mail_from: sender,
+                results: { push() {} },
+            }
+            this.connection.respond = (_code, _msg, cb) => cb && cb()
+            plugins.run_hooks = (hook) => hooks.push(hook)
+
+            try {
+                this.connection.rcpt_respond(constants.ok, 'ok')
+                assert.equal(hooks.includes('rcpt_ok'), true)
+                assert.equal(this.connection.last_rcpt_msg, 'ok')
+            } finally {
+                plugins.run_hooks = originalRunHooks
+            }
+        })
+
+        it('cmd_proxy rejects when not allowed', () => {
+            let code
+            this.connection.proxy.allowed = false
+            this.connection.respond = (c) => {
+                code = c
+            }
+            this.connection.disconnect = () => {}
+            this.connection.cmd_proxy('TCP4 1.2.3.4 5.6.7.8 100 25')
+            assert.equal(code, 421)
+        })
+
+        it('cmd_proxy accepts valid TCP4 proxy line and runs connect_init', () => {
+            const plugins = require('../plugins')
+            const originalRunHooks = plugins.run_hooks
+            const hooks = []
+            this.connection.proxy.allowed = true
+            this.connection.remote.ip = '10.0.0.1'
+            this.connection.reset_transaction = (cb) => cb && cb()
+            this.connection.respond = () => {}
+            plugins.run_hooks = (hook) => hooks.push(hook)
+
+            try {
+                this.connection.cmd_proxy('TCP4 1.2.3.4 5.6.7.8 100 25')
+                assert.equal(this.connection.proxy.type, 'haproxy')
+                assert.equal(this.connection.remote.ip, '1.2.3.4')
+                assert.equal(this.connection.local.ip, '5.6.7.8')
+                assert.equal(hooks.includes('connect_init'), true)
+            } finally {
+                plugins.run_hooks = originalRunHooks
+            }
+        })
+
+        it('cmd_data validates argument/transaction/recipient preconditions', () => {
+            const responses = []
+            this.connection.respond = (code, msg) => {
+                responses.push([code, msg])
+            }
+
+            this.connection.cmd_data('unexpected')
+            this.connection.cmd_data()
+            this.connection.transaction = { rcpt_to: [] }
+            this.connection.cmd_data()
+
+            assert.equal(responses[0][0], 501)
+            assert.equal(responses[1][0], 503)
+            assert.equal(responses[2][0], 503)
+        })
+
+        it('data_respond denysoftdisconnect disconnects and default enters DATA', () => {
+            const responses = []
+            let disconnected = 0
+            this.connection.transaction = { data_bytes: 5 }
+            this.connection.respond = (code, _msg, cb) => {
+                responses.push(code)
+                if (cb) cb()
+            }
+            this.connection.disconnect = () => {
+                disconnected++
+            }
+
+            this.connection.data_respond(constants.denysoftdisconnect, 'tmpfail')
+            this.connection.data_respond(constants.ok, 'ok')
+
+            assert.equal(responses[0], 451)
+            assert.equal(disconnected, 1)
+            assert.equal(responses[1], 354)
+            assert.equal(this.connection.state, constants.connection.state.DATA)
+            assert.equal(this.connection.transaction.data_bytes, 0)
+        })
+    })
 })

package/test/endpoint.js

@@ -20,6 +20,20 @@ describe('endpoint', () => {
             assert.deepEqual(endpoint(25), { host: '::0', port: 25 })
         })
 
+        it('Unbracketed IPv6 host uses default port', () => {
+            assert.deepEqual(endpoint('::0', 25), {
+                host: '::0',
+                port: 25,
+            })
+        })
+
+        it('Unbracketed IPv6 host:port parses correctly (PR #3552 compatibility)', () => {
+            assert.deepEqual(endpoint('::0:25'), {
+                host: '::0',
+                port: 25,
+            })
+        })
+
         it('Default port if only host', () => {
             assert.deepEqual(endpoint('10.0.0.3', 42), {
                 host: '10.0.0.3',
@@ -27,6 +41,13 @@ describe('endpoint', () => {
             })
         })
 
+        it('Bracketed IPv6 host is normalized to lowercase', () => {
+            assert.deepEqual(endpoint('[ABCD::EF01]:2525'), {
+                host: 'abcd::ef01',
+                port: 2525,
+            })
+        })
+
         it('Unix socket', () => {
             assert.deepEqual(endpoint('/foo/bar.sock'), {
                 path: '/foo/bar.sock',
@@ -39,6 +60,12 @@ describe('endpoint', () => {
                 mode: '770',
             })
         })
+
+        it('Invalid unbracketed IPv6 host with non-numeric tail returns Error', () => {
+            const ep = endpoint('::0:port')
+            assert.equal(ep instanceof Error, true)
+            assert.match(ep.message, /Invalid socket address/)
+        })
     })
 
     describe('bind()', () => {

package/test/outbound/hmail.js

@@ -167,6 +167,25 @@ describe('outbound/hmail.HMailItem — queue file loading', () => {
         assert.ok(h)
     })
 
+    it('releases queue slot when stat fails on exhausted-retry item (regression #3560)', async () => {
+        // When fs.stat fails and num_failures already equals temp_fail_intervals.length,
+        // temp_fail() calls convert_temp_failed_to_bounce() while this.todo is null.
+        // That must not crash, and must call next_cb() to release the queue slot.
+        // attempts=12 in the filename causes num_failures=12 at construction; after
+        // temp_fail() increments it to 13 (> temp_fail_intervals.length=12) the
+        // overflow path fires with this.todo still null.
+        const fname = '1508455115683_1508455115683_12_90253_9Q4o4V_1_haraka'
+        const h = new Hmail(fname, '/nonexistent/path/that/cannot/be/stat/ed', {})
+        await new Promise((resolve, reject) => {
+            const timer = setTimeout(() => reject(new Error('next_cb was never called')), 2000)
+            h.next_cb = () => {
+                clearTimeout(timer)
+                resolve()
+            }
+        })
+        assert.equal(h.todo, null, 'todo must remain null — file was never readable')
+    })
+
     it('lifecycle: reads and writes a queue file', async () => {
         const h = new Hmail('1507509981169_1507509981169_0_61403_e0Y0Ym_2_qfile', 'test/fixtures/todo_qfile.txt', {})
 

package/test/outbound/index.js

@@ -152,6 +152,195 @@ describe('outbound', () => {
             })
             assert.ok(true)
         })
+
+        it('waits for drain when stream backpressure is applied', async () => {
+            const todo = {
+                queue_time: Date.now(),
+                domain: 'example.com',
+                rcpt_to: [],
+                mail_from: {},
+                notes: {},
+                uuid: 'u1',
+            }
+            let drained = false
+
+            await new Promise((resolve) => {
+                const ws = {
+                    write() {
+                        return false
+                    },
+                    once(event, cb) {
+                        assert.equal(event, 'drain')
+                        setImmediate(() => {
+                            drained = true
+                            cb()
+                            resolve()
+                        })
+                    },
+                }
+                outbound.build_todo(todo, ws, () => {})
+            })
+
+            assert.equal(drained, true)
+        })
+    })
+
+    describe('send_trans_email', () => {
+        const queueDir = path.resolve('test', 'test-queue')
+
+        beforeEach(() => {
+            process.env.HARAKA_TEST_DIR = path.resolve('test')
+            fs.mkdirSync(queueDir, { recursive: true })
+        })
+
+        afterEach(() => {
+            delete process.env.HARAKA_TEST_DIR
+            try {
+                for (const f of fs.readdirSync(queueDir)) {
+                    fs.unlinkSync(path.join(queueDir, f))
+                }
+            } catch (ignore) {}
+        })
+
+        // Regression test for haraka/Haraka#3551:
+        // When dkim_verify (data_post) pipes the message_stream and DKIMVerifyStream
+        // fires its callback early via process.nextTick (no DKIM-Signature found),
+        // the chain runs synchronously into process_delivery → pipe() while the
+        // first pipe is still in flight. pre_send_trans_email_respond must yield
+        // (via setImmediate) before opening a new pipe.
+        it('yields to setImmediate before opening process_delivery pipes', async () => {
+            const stream = require('node:stream')
+            const Transaction = require('../../transaction')
+            const Address = require('address-rfc2821').Address
+            const outbound = require('../../outbound')
+            const plugins = require('../../plugins')
+
+            const txn = Transaction.createTransaction()
+            const origRunHooks = plugins.run_hooks
+            try {
+                txn.mail_from = new Address('<from@example.com>')
+                txn.rcpt_to = [new Address('<to@example.com>')]
+                txn.message_stream.add_line(Buffer.from('From: from@example.com\r\n'))
+                txn.message_stream.add_line(Buffer.from('To: to@example.com\r\n'))
+                txn.message_stream.add_line(Buffer.from('\r\n'))
+                txn.message_stream.add_line(Buffer.from('body\r\n'))
+                await new Promise((r) => txn.message_stream.add_line_end(r))
+
+                // Start a pipe on the message_stream and fire a synchronous callback
+                // before it drains — this models what dkim_verify does.
+                const verifierFiredCb = new Promise((resolve) => {
+                    let scheduled = false
+                    const verifier = new stream.Writable({
+                        write(_chunk, _enc, cb) {
+                            if (!scheduled) {
+                                scheduled = true
+                                process.nextTick(resolve)
+                            }
+                            cb()
+                        },
+                    })
+                    txn.message_stream.pipe(verifier)
+                })
+                await verifierFiredCb
+
+                // Now invoke send_trans_email — its pre_send_trans_email_respond
+                // should yield (await setImmediate) before calling process_delivery,
+                // letting the verifier pipe drain so the new pipe can succeed.
+                await new Promise((resolve, reject) => {
+                    // Stub the heavy bits: we only care that the chain doesn't throw
+                    // "Cannot pipe while currently piping" before queuing happens.
+                    plugins.run_hooks = (hook, obj) => {
+                        if (hook === 'pre_send_trans_email') {
+                            // Mimic empty-hook synchronous callback (no plugins)
+                            obj.pre_send_trans_email_respond(constants.cont).catch(reject)
+                        } else {
+                            origRunHooks.call(plugins, hook, obj)
+                        }
+                    }
+
+                    outbound.send_trans_email(txn, (retval) => {
+                        if (retval === constants.ok) resolve()
+                        else reject(new Error(`unexpected retval ${retval}`))
+                    })
+                })
+            } finally {
+                plugins.run_hooks = origRunHooks
+                txn.message_stream.destroy()
+            }
+        })
+
+        it('adds missing Message-Id/Date and prepends Received before queueing', async () => {
+            process.env.HARAKA_TEST_DIR = path.resolve('test')
+            const Address = require('address-rfc2821').Address
+            const outbound = require('../../outbound')
+            const plugins = require('../../plugins')
+
+            const added = []
+            const leading = []
+            const queued = []
+            const transaction = {
+                uuid: 'txn-add-headers',
+                header: {
+                    get_all(_name) {
+                        return []
+                    },
+                    get() {
+                        return null
+                    },
+                },
+                rcpt_to: [new Address('<user@example.com>')],
+                notes: {},
+                add_header(name, value) {
+                    added.push([name, value])
+                },
+                remove_header() {},
+                add_leading_header(name, value) {
+                    leading.push([name, value])
+                },
+                results: {
+                    add() {},
+                },
+            }
+
+            const originalRunHooks = plugins.run_hooks
+            const originalProcessDelivery = outbound.process_delivery
+            const originalPush = outbound.delivery_queue.push
+            outbound.delivery_queue.push = (hmail) => {
+                queued.push(hmail)
+            }
+            outbound.process_delivery = async (_okPaths, _todo, hmails) => {
+                hmails.push({ queued: true })
+            }
+            plugins.run_hooks = (hook, conn) => {
+                if (hook === 'pre_send_trans_email') {
+                    conn.pre_send_trans_email_respond(constants.cont)
+                }
+            }
+
+            try {
+                const result = await new Promise((resolve) => {
+                    outbound.send_trans_email(transaction, (retval, msg) => resolve({ retval, msg }))
+                })
+
+                assert.equal(result.retval, constants.ok)
+                assert.match(result.msg, /Message Queued/)
+                assert.equal(queued.length, 1)
+                assert.equal(
+                    added.some(([name]) => name === 'Message-Id'),
+                    true,
+                )
+                assert.equal(
+                    added.some(([name]) => name === 'Date'),
+                    true,
+                )
+                assert.equal(leading[0][0], 'Received')
+            } finally {
+                plugins.run_hooks = originalRunHooks
+                outbound.process_delivery = originalProcessDelivery
+                outbound.delivery_queue.push = originalPush
+                delete process.env.HARAKA_TEST_DIR
+            }
+        })
     })
 
     describe('timer_queue', () => {

package/test/outbound/queue.js

@@ -230,4 +230,96 @@ describe('outbound/queue', () => {
             }
         })
     })
+
+    describe('queue maintenance', () => {
+        it('delete_dot_files removes leftover dot files only', async () => {
+            const tmpDir = path.join(os.tmpdir(), `haraka-dot-clean-${Date.now()}`)
+            fs.mkdirSync(tmpDir, { recursive: true })
+            const dotName = `${qfile.platformDOT}leftover`
+            const normalName = 'keep-me'
+            fs.writeFileSync(path.join(tmpDir, dotName), 'x')
+            fs.writeFileSync(path.join(tmpDir, normalName), 'x')
+
+            const originalQueueDir = queue.queue_dir
+            queue.queue_dir = tmpDir
+            try {
+                await queue.delete_dot_files()
+                assert.equal(fs.existsSync(path.join(tmpDir, dotName)), false)
+                assert.equal(fs.existsSync(path.join(tmpDir, normalName)), true)
+            } finally {
+                queue.queue_dir = originalQueueDir
+                fs.rmSync(tmpDir, { recursive: true, force: true })
+            }
+        })
+
+        it('_add_hmail pushes immediate items and schedules delayed ones', () => {
+            const originalPush = queue.delivery_queue.push
+            const originalAdd = queue.temp_fail_queue.add
+            const pushed = []
+            const delayed = []
+            let delayedCb
+
+            queue.delivery_queue.push = (item) => pushed.push(item)
+            queue.temp_fail_queue.add = (id, ms, cb) => {
+                delayed.push([id, ms])
+                delayedCb = cb
+            }
+            queue.cur_time = new Date()
+
+            const immediate = { filename: 'a', next_process: queue.cur_time - 1 }
+            const future = { filename: 'b', next_process: queue.cur_time.getTime() + 1000 }
+
+            try {
+                queue._add_hmail(immediate)
+                queue._add_hmail(future)
+                assert.equal(pushed.length, 1)
+                assert.equal(delayed.length, 1)
+                assert.equal(delayed[0][0], 'b')
+                delayedCb()
+                assert.equal(pushed.length, 2)
+            } finally {
+                queue.delivery_queue.push = originalPush
+                queue.temp_fail_queue.add = originalAdd
+            }
+        })
+
+        it('scan_queue_pids returns unique pids from queue files', async () => {
+            populateTestQueue()
+            const originalQueueDir = queue.queue_dir
+            queue.queue_dir = testQueueDir
+
+            try {
+                const pids = await queue.scan_queue_pids()
+                assert.ok(Array.isArray(pids))
+                assert.equal(pids.length >= 1, true)
+            } finally {
+                queue.queue_dir = originalQueueDir
+                clearTestQueue()
+            }
+        })
+
+        it('scan_queue_pids throws when queue dir cannot be read', async () => {
+            const originalQueueDir = queue.queue_dir
+            const badPath = path.join(os.tmpdir(), `queue-not-dir-${Date.now()}`)
+            fs.writeFileSync(badPath, 'x')
+            queue.queue_dir = badPath
+            try {
+                await assert.rejects(() => queue.scan_queue_pids())
+            } finally {
+                queue.queue_dir = originalQueueDir
+                fs.rmSync(badPath, { force: true })
+            }
+        })
+
+        it('delete_dot_files handles readdir errors without throwing', async () => {
+            const originalQueueDir = queue.queue_dir
+            queue.queue_dir = path.join(os.tmpdir(), `missing-dot-${Date.now()}`)
+            try {
+                await queue.delete_dot_files()
+                assert.ok(true)
+            } finally {
+                queue.queue_dir = originalQueueDir
+            }
+        })
+    })
 })