comq 0.15.7 → 0.15.8

package/source/connection.js

@@ -50,10 +50,7 @@ class Connection {
    * @param {string} url
    */
   constructor (url) {
-    this.#url = url
-
-    // EventEmitter throws on 'error' with no listeners
-    this.#diagnostics.on('error', noop)
+    this.#url = heartbeaten(url)
   }
 
   get connected () {
@@ -200,7 +197,14 @@ class Connection {
     }
 
     connection.removeAllListeners()
-    connection.connection?.stream?.destroy()
+
+    // amqplib keeps its heartbeater running on a connection it does not know is
+    // gone and emits 'error' on it, which throws once no listener is left
+    connection.on('error', noop)
+
+    // a socket destroyed without an error tells amqplib nothing, leaving its
+    // timers running and everything pending on it hanging forever
+    connection.connection?.stream?.destroy(silence())
   }
 
   #recover () {
@@ -215,7 +219,7 @@ class Connection {
    * @return {Promise<comq.amqp.Connection>}
    */
   async #connect () {
-    const connecting = amqp.connect(this.#url, { timeout: CONNECT_MS })
+    const connecting = amqp.connect(this.#url, SOCKET_OPTIONS)
 
     let timer
 
@@ -248,19 +252,23 @@ class Connection {
    * @param {comq.amqp.Connection} connection
    * @param {number} [timeoutMs]
    */
-  #armWatchdog (connection, timeoutMs = global.COMQ_TESTING_WATCHDOG_INTERVAL ?? WATCHDOG_MS) {
+  #armWatchdog (connection, timeoutMs = tolerance(connection)) {
+    this.#disarmWatchdog()
+
     const socket = connection.connection?.stream
 
-    if (socket === undefined) return
+    if (socket === undefined || timeoutMs === undefined) return
 
     const reset = () => {
       clearTimeout(this.#heartbeatTimer)
-      this.#heartbeatTimer = setTimeout(() => socket.destroy(), timeoutMs)
+      // destroying a socket without an error tells amqplib nothing: it only
+      // listens for 'error' and 'end', so a bare destroy() leaves the connection
+      // silently dead — no 'close' event, no recovery, every pending operation
+      // hanging forever
+      this.#heartbeatTimer = setTimeout(() => socket.destroy(silence()), timeoutMs)
       this.#heartbeatTimer.unref()
     }
 
-    this.#disarmWatchdog()
-
     this.#heartbeatSocket = socket
     this.#heartbeatReset = reset
 
@@ -289,7 +297,25 @@ class Connection {
   }
 }
 
+/**
+ * The heartbeat to ask for when the caller has not, in seconds. A broker is free
+ * to suggest one that leaves a connection lost for minutes before anything
+ * notices, and RabbitMQ suggests 60 by default.
+ *
+ * @type {number}
+ */
+const HEARTBEAT_S = 15
+
 /** @type {number} */
+const KEEPALIVE_MS = 10_000
+
+/** How many heartbeats a connection may miss before it is destroyed. */
+const MISSED_HEARTBEATS = 3
+
+/** @type {number} */
+const WATCHDOG_MIN_MS = 15_000
+
+/** The watchdog interval used when the negotiated heartbeat is unknown. */
 const WATCHDOG_MS = 60_000
 
 /** @type {number} */
@@ -298,6 +324,15 @@ const CONNECT_MS = 30_000
 /** @type {number} */
 const SHUTDOWN_MS = 5_000
 
+const SOCKET_OPTIONS = {
+  timeout: CONNECT_MS,
+  // a peer that went away without a word is noticed by the kernel as well
+  keepAlive: true,
+  keepAliveDelay: KEEPALIVE_MS
+}
+
+const HEARTBEAT_SET = /[?&]heartbeat=/
+
 const TRANSIENT_CODES = new Set([
   'ECONNREFUSED',
   'EAI_AGAIN',
@@ -323,6 +358,52 @@ function expiration () {
   return delay(timeoutMs, undefined, { ref: false })
 }
 
+/**
+ * amqplib reads the heartbeat from the URL only, hence it cannot be passed along
+ * with the socket options.
+ *
+ * @param {string} url
+ * @return {string}
+ */
+function heartbeaten (url) {
+  if (HEARTBEAT_SET.test(url)) return url
+
+  return url + (url.includes('?') ? '&' : '?') + 'heartbeat=' + HEARTBEAT_S
+}
+
+/**
+ * The watchdog measures silence, so it cannot be shorter than the interval at
+ * which a healthy broker is expected to say something, which is a heartbeat
+ * frame every half a heartbeat. A connection that has agreed to no heartbeats
+ * says nothing at all while it is idle, leaving nothing to measure.
+ *
+ * @param {comq.amqp.Connection} connection
+ * @return {number | undefined}
+ */
+function tolerance (connection) {
+  const override = global.COMQ_TESTING_WATCHDOG_INTERVAL
+
+  if (override !== undefined) return override
+
+  const heartbeat = connection.connection?.heartbeat
+
+  if (heartbeat === undefined) return WATCHDOG_MS
+  if (heartbeat === 0) return undefined
+
+  return Math.max(heartbeat * MISSED_HEARTBEATS * 1000, WATCHDOG_MIN_MS)
+}
+
+/**
+ * @return {Error}
+ */
+function silence () {
+  const exception = new Error('Connection is silent')
+
+  exception.code = 'ETIMEDOUT'
+
+  return exception
+}
+
 function noop () {}
 
 exports.Connection = Connection

package/source/emitter.js

@@ -2,8 +2,30 @@
 
 const { EventEmitter } = require('node:events')
 
+/**
+ * A listener must not be able to break what it observes: an exception thrown by
+ * one used to leave the reconnection it was reporting on unfinished, or take the
+ * process down through the emitter amqplib was calling.
+ */
+class Emitter extends EventEmitter {
+  emit (event, ...args) {
+    // raw listeners are used so that `once` still removes itself
+    const listeners = this.rawListeners(event)
+
+    for (const listener of listeners) {
+      try {
+        listener.apply(this, args)
+      } catch {
+        // a diagnostic listener has no one to report to
+      }
+    }
+
+    return listeners.length > 0
+  }
+}
+
 function create () {
-  const emitter = new EventEmitter()
+  const emitter = new Emitter()
 
   emitter.setMaxListeners(0)
 

package/source/encoders/index.js

@@ -1,9 +1,7 @@
 'use strict'
 
 const json = require('./json')
-const msgpack = require('./msgpack')
 const plain = require('./plain')
 
 exports['application/json'] = json
-exports['application/msgpack'] = msgpack
 exports['text/plain'] = plain

package/source/events.js

@@ -4,4 +4,4 @@
 exports.connection = ['open', 'close', 'error', 'reconnect']
 
 /** @type {comq.diagnostics.Event[]} */
-exports.channel = ['flow', 'drain', 'recover', 'discard', 'pause', 'resume', 'return', 'lost']
+exports.channel = ['flow', 'drain', 'recover', 'discard', 'pause', 'resume', 'return', 'lost', 'remove']

package/source/io.js

@@ -1,7 +1,6 @@
 'use strict'
 
 const stream = require('node:stream')
-const { EventEmitter } = require('node:events')
 const { randomBytes } = require('node:crypto')
 const { setTimeout } = require('node:timers/promises')
 const { Promex } = require('promex')
@@ -11,6 +10,7 @@ const { decode } = require('./decode')
 const { encode } = require('./encode')
 const { pipeline, transform } = require('./pipeline')
 const events = require('./events')
+const emitter = require('./emitter')
 const io = require('./.io')
 
 /**
@@ -44,7 +44,7 @@ class IO {
   /** @type {Set<comq.Destroyable>} */
   #replyPipes = new Set()
 
-  #diagnostics = new EventEmitter()
+  #diagnostics = emitter.create()
 
   /**
    * @param {comq.Connection} connection
@@ -52,9 +52,6 @@ class IO {
   constructor (connection) {
     this.#connection = connection
 
-    // EventEmitter throws on 'error' with no listeners
-    this.#diagnostics.on('error', noop)
-
     for (const event of events.connection) {
       this.#connection.diagnose(event, (...args) => this.#diagnostics.emit(event, ...args))
     }
@@ -220,6 +217,10 @@ class IO {
     } else {
       this.#requests.diagnose('recover', this.#retransmit)
     }
+
+    // a reply that could not be routed is dropped by the broker, and the queue
+    // it was addressed to only exists again once this channel has recovered
+    this.#replies.diagnose('recover', this.#retransmit)
   }
 
   /**

package/source/shards/channel.js

@@ -31,6 +31,9 @@ class Channel {
   /** @type {Map<comq.Channel, Promex>} */
   #bench = new Map()
 
+  /** @type {boolean} */
+  #paused = false
+
   /** @type {comq.topology.type} */
   #type
 
@@ -126,6 +129,9 @@ class Channel {
       this.#alive[index] = false
       this.#down[index].resolve()
 
+      // the channel stays in the pool, yet there is one less shard to publish to
+      this.#update()
+
       if (error !== undefined && connection.closed !== true) {
         this.#diagnostics.emit(LOST, index)
       }
@@ -134,6 +140,8 @@ class Channel {
     connection.diagnose('open', () => {
       this.#alive[index] = true
       this.#down[index] = new Promex()
+
+      this.#update()
     })
   }
 
@@ -159,6 +167,7 @@ class Channel {
     for (const event of events.channel) {
       if (event === RETURN) continue // returns are retried before being reported
       if (event === LOST) continue // a shard is lost with its connection, not its channel
+      if (event === REMOVE) continue // it is the pool that removes a channel, not the channel
 
       channel.diagnose(event, (...args) => this.#diagnostics.emit(event, ...args, channel.index))
     }
@@ -214,20 +223,23 @@ class Channel {
     this.#bench.set(channel, new Promex())
     this.#channels.delete(channel)
     this.#update()
-    this.#diagnostics.emit('remove', channel.index)
+    this.#diagnostics.emit(REMOVE, channel.index)
   }
 
+  /**
+   * A shard leaves the pool when it rejects a publish, and stops being reachable
+   * when its connection is lost, which the pool is not told about. Both leave
+   * `#one` waiting, so both are reported the same way.
+   */
   #update () {
-    const from = this.#pool?.length
-    const to = this.#channels.size
-
     this.#pool = Array.from(this.#channels)
 
-    if (from === undefined) return
+    const paused = this.#reachable().length === 0
 
-    if (from !== 0 && to === 0) { this.#diagnostics.emit('pause') }
+    if (paused === this.#paused) return
 
-    if (from === 0 && to !== 0) { this.#diagnostics.emit('resume') }
+    this.#paused = paused
+    this.#diagnostics.emit(paused ? 'pause' : 'resume')
   }
 
   /**
@@ -369,6 +381,7 @@ async function create (connections, type) {
 
 const DEFAULT = ''
 const RETURN = 'return'
+const REMOVE = 'remove'
 const RETURN_HEADER = 'x-return'
 const LOST = 'lost'
 

package/source/shards/connection.js

@@ -19,9 +19,6 @@ class Connection {
   constructor (connections) {
     this.#connections = connections
 
-    // EventEmitter throws on 'error' with no listeners
-    this.#diagnostics.on('error', noop)
-
     connections.map(this.#pipe)
   }
 
@@ -75,6 +72,4 @@ class Connection {
   }
 }
 
-function noop () {}
-
 exports.Connection = Connection

package/test/amqplib.mock.js

@@ -18,14 +18,26 @@ class Channel extends EventEmitter {
 }
 
 class Connection extends EventEmitter {
-  constructor () {
+  constructor (url = '') {
     super()
 
     const stream = new EventEmitter()
 
-    stream.destroy = jest.fn()
+    // amqplib listens for 'error' and 'end' on the socket only, so destroying it
+    // without an error leaves the connection unaware and silent forever
+    stream.destroy = jest.fn((error) => {
+      if (error === undefined || stream.destroyed === true) return
 
-    this.connection = { stream }
+      stream.destroyed = true
+
+      this.emit('error', error)
+      this.emit('close', error)
+    })
+
+    // a broker accepts the heartbeat the client asks for
+    const heartbeat = Number(/[?&]heartbeat=(\d+)/.exec(url)?.[1] ?? 60)
+
+    this.connection = { stream, heartbeat }
 
     // noinspection JSValidateTypes
     this.removeAllListeners = jest.spyOn(this, 'removeAllListeners')
@@ -39,7 +51,7 @@ class Connection extends EventEmitter {
   close = jest.fn(async () => undefined)
 }
 
-const connect = async () => new Connection()
+const connect = async (url) => new Connection(url)
 
 /** @type {jest.MockedObject<import('amqplib')>} */
 const amqplib = {

package/test/channel.fixtures.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { random, flip } = require('@toa.io/generic')
+const { random, flip } = require('./helpers')
 
 const preset = () => ({
   prefetch: random(10),

package/test/channel.test.js

@@ -4,7 +4,7 @@
 
 const { randomBytes } = require('node:crypto')
 const { generate } = require('randomstring')
-const { flip, random, timeout } = require('@toa.io/generic')
+const { flip, random, timeout } = require('./helpers')
 
 const backpressure = require('./backpressure')
 const { amqplib } = require('./amqplib.mock')

package/test/connection.test.js

@@ -4,7 +4,8 @@
 
 const { generate } = require('randomstring')
 
-const { timeout, promex, random } = require('@toa.io/generic')
+const { Promex } = require('promex')
+const { timeout, random } = require('./helpers')
 const { amqplib, connect } = require('./amqplib.mock')
 const { channel: create } = require('./connection.mock')
 const mock = { amqplib, channel: { create } }
@@ -37,7 +38,8 @@ describe('initial connection', () => {
   it('should connect', async () => {
     await connection.open()
 
-    expect(amqplib.connect).toHaveBeenCalledWith(url, { timeout: expect.any(Number) })
+    expect(amqplib.connect).toHaveBeenCalledWith(expect.stringContaining(url),
+      expect.objectContaining({ timeout: expect.any(Number) }))
   })
 
   it.each(/** @type {[string, Partial<Error>][]} */[
@@ -138,6 +140,41 @@ describe('reconnection', () => {
     expect(amqplib.connect.mock.calls.length).toBeGreaterThanOrEqual(3)
   }, 15000)
 
+  it('should keep an error sink on a dropped connection', async () => {
+    const channel = await connection.createChannel('request')
+
+    channel.recover
+      .mockRejectedValueOnce(new Error('Channel closed'))
+      .mockResolvedValue(undefined)
+
+    conn.emit('close', new Error('lost'))
+
+    await timeout(10)
+
+    const dropped = await amqplib.connect.mock.results[1].value
+
+    // amqplib goes on emitting on a connection it does not know is gone, and an
+    // 'error' with no listener left takes the process down
+    expect(() => dropped.emit('error', new Error('Heartbeat timeout'))).not.toThrow()
+
+    // let the attempt that follows the failed recovery settle
+    const start = Date.now()
+
+    while (channel.recover.mock.calls.length < 2 && Date.now() - start < 10000) await timeout(50)
+  }, 15000)
+
+  it('should reconnect despite a listener that throws', async () => {
+    // a diagnostic listener has no business breaking the reconnection it reports
+    connection.diagnose('reconnect', () => { throw new Error('listener') })
+    connection.diagnose('open', () => { throw new Error('listener') })
+
+    conn.emit('close', new Error('lost'))
+
+    await expect(connection.createChannel('event')).resolves.toBeDefined()
+
+    expect(amqplib.connect).toHaveBeenCalledTimes(2)
+  }, 10000)
+
   it('should emit error when reconnect open fails', async () => {
     const errors = []
     const unhandled = jest.fn()
@@ -339,7 +376,7 @@ describe('create channel', () => {
 
     jest.clearAllMocks()
 
-    const promise = promex()
+    const promise = new Promex()
 
     amqplib.connect.mockImplementationOnce(() => promise)
 
@@ -373,11 +410,23 @@ describe('watchdog', () => {
 
     const conn = await amqplib.connect.mock.results[0].value
 
-    await jest.advanceTimersByTimeAsync(60_000)
+    await jest.advanceTimersByTimeAsync(46_000)
 
     expect(conn.connection.stream.destroy).toHaveBeenCalled()
   })
 
+  it('should reconnect after the watchdog destroys a silent connection', async () => {
+    jest.useFakeTimers()
+
+    await connection.open()
+
+    await jest.advanceTimersByTimeAsync(46_000)
+
+    // a socket destroyed without an error is a socket amqplib never reports,
+    // which leaves the connection silently dead instead of recovering
+    expect(amqplib.connect).toHaveBeenCalledTimes(2)
+  })
+
   it('should not let a replaced socket disarm the watchdog', async () => {
     jest.useFakeTimers()
 
@@ -407,11 +456,11 @@ describe('watchdog', () => {
 
     const conn = await amqplib.connect.mock.results[0].value
 
-    await jest.advanceTimersByTimeAsync(50_000)
+    await jest.advanceTimersByTimeAsync(40_000)
 
     conn.connection.stream.emit('data', Buffer.alloc(0))
 
-    await jest.advanceTimersByTimeAsync(50_000)
+    await jest.advanceTimersByTimeAsync(40_000)
 
     expect(conn.connection.stream.destroy).not.toHaveBeenCalled()
 
@@ -419,6 +468,77 @@ describe('watchdog', () => {
 
     expect(conn.connection.stream.destroy).toHaveBeenCalled()
   })
+
+  it('should derive the watchdog from the negotiated heartbeat', async () => {
+    jest.useFakeTimers()
+
+    const connection = new Connection('amqp://localhost?heartbeat=30')
+
+    await connection.open()
+
+    const conn = await amqplib.connect.mock.results[0].value
+
+    // the broker is only expected to say something every 15 seconds
+    await jest.advanceTimersByTimeAsync(60_000)
+
+    expect(conn.connection.stream.destroy).not.toHaveBeenCalled()
+
+    await jest.advanceTimersByTimeAsync(31_000)
+
+    expect(conn.connection.stream.destroy).toHaveBeenCalled()
+  })
+
+  it('should not arm the watchdog when heartbeats are disabled', async () => {
+    jest.useFakeTimers()
+
+    const connection = new Connection('amqp://localhost?heartbeat=0')
+
+    await connection.open()
+
+    const conn = await amqplib.connect.mock.results[0].value
+
+    // an idle connection that has agreed to no heartbeats is silent by design
+    await jest.advanceTimersByTimeAsync(600_000)
+
+    expect(conn.connection.stream.destroy).not.toHaveBeenCalled()
+  })
+})
+
+describe('heartbeat', () => {
+  it('should ask for a heartbeat', async () => {
+    const connection = new Connection('amqp://developer@localhost')
+
+    await connection.open()
+
+    expect(amqplib.connect).toHaveBeenCalledWith('amqp://developer@localhost?heartbeat=15',
+      expect.anything())
+  })
+
+  it('should append the heartbeat to an existing query', async () => {
+    const connection = new Connection('amqp://localhost?frameMax=8192')
+
+    await connection.open()
+
+    expect(amqplib.connect).toHaveBeenCalledWith('amqp://localhost?frameMax=8192&heartbeat=15',
+      expect.anything())
+  })
+
+  it.each(['amqp://localhost?heartbeat=5', 'amqp://localhost?heartbeat=0'])(
+    'should keep the heartbeat requested by the caller (%s)',
+    async (url) => {
+      const connection = new Connection(url)
+
+      await connection.open()
+
+      expect(amqplib.connect).toHaveBeenCalledWith(url, expect.anything())
+    })
+
+  it('should keep the socket alive', async () => {
+    await connection.open()
+
+    expect(amqplib.connect)
+      .toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ keepAlive: true }))
+  })
 })
 
 describe('close', () => {

package/test/decode.test.js

@@ -2,7 +2,6 @@
 
 const { randomBytes } = require('node:crypto')
 const { generate } = require('randomstring')
-const { pack } = require('msgpackr')
 
 const { decode } = require('../source/decode')
 
@@ -23,18 +22,6 @@ it('should decode application/json', async () => {
   expect(decoded).toStrictEqual(object)
 })
 
-it('should decode application/msgpack', async () => {
-  const object = { [generate()]: generate() }
-  const content = pack(object)
-  const contentType = 'application/msgpack'
-  const properties = { contentType }
-  const message = /** @type {comq.amqp.Message} */ { content, properties }
-
-  const decoded = decode(message)
-
-  expect(decoded).toStrictEqual(object)
-})
-
 it('should return buffer if content type is not defined', async () => {
   const object = { [generate()]: generate() }
   const json = JSON.stringify(object)

package/test/emitter.test.js

@@ -0,0 +1,71 @@
+'use strict'
+
+const { generate } = require('randomstring')
+const emitter = require('../source/emitter')
+
+/** @type {import('node:events').EventEmitter} */
+let instance
+
+beforeEach(() => {
+  instance = emitter.create()
+})
+
+it('should be', async () => {
+  expect(emitter.create).toBeInstanceOf(Function)
+})
+
+it('should emit', async () => {
+  const listener = jest.fn()
+  const event = generate()
+  const args = [generate(), generate()]
+
+  instance.on(event, listener)
+
+  expect(instance.emit(event, ...args)).toStrictEqual(true)
+  expect(listener).toHaveBeenCalledWith(...args)
+})
+
+it('should report an event no one listens to', async () => {
+  expect(instance.emit(generate())).toStrictEqual(false)
+})
+
+it('should not throw on error without listeners', async () => {
+  expect(() => instance.emit('error', new Error(generate()))).not.toThrow()
+})
+
+it('should not let a listener break the emission', async () => {
+  const event = generate()
+  const listener = jest.fn()
+
+  instance.on(event, () => { throw new Error(generate()) })
+  instance.on(event, listener)
+
+  expect(() => instance.emit(event)).not.toThrow()
+
+  // the listeners that follow a broken one are still called
+  expect(listener).toHaveBeenCalled()
+})
+
+it('should call a `once` listener once', async () => {
+  const event = generate()
+  const listener = jest.fn()
+
+  instance.once(event, listener)
+
+  instance.emit(event)
+  instance.emit(event)
+
+  expect(listener).toHaveBeenCalledTimes(1)
+})
+
+it('should not call a listener that has been removed', async () => {
+  const event = generate()
+  const listener = jest.fn()
+
+  instance.on(event, listener)
+  instance.off(event, listener)
+
+  instance.emit(event)
+
+  expect(listener).not.toHaveBeenCalled()
+})