comq 0.15.7 → 0.16.0

package/readme.md

@@ -299,7 +299,6 @@ If the specified encoding format is not supported, an exception will be thrown.
 The following encoding formats are supported:
 
 - `application/json`
-- `application/msgpack`
 - `application/octet-stream`
 - `text/plain`
 
@@ -314,6 +313,12 @@ the connection is restored.
 
 When the established connection is lost, it will be automatically restored.
 Reconnection attempts will be made indefinitely, with intervals increasing up to 30 seconds.
+Unless the URL sets one, a 15 second heartbeat is requested, so that a connection that is gone
+without a word, such as after a machine wakes from sleep, is noticed within a minute instead of
+being left to whatever the broker suggests.
+A connection that stays silent for three heartbeats is destroyed regardless of what the broker and
+the operating system have reported, since neither is guaranteed to report anything at all.
+Requesting `heartbeat=0` disables both.
 If the broker rejects the connection, for example, due to access being denied, an exception will be thrown.
 Once reconnected, the topology will be recovered, and any unanswered requests and unconfirmed events will be
 retransmitted.
@@ -493,10 +498,13 @@ Subscribe to one of the diagnostic events:
 - `error`: an attempt to restore the connection has failed. The exception is passed as an argument.
   Attempts continue until one succeeds.
 - `reconnect`: an attempt to restore the connection has started. Followed by `open` or `error`.
+- `exhausted`: the connection has no channel left to allocate. The negotiated channel limit is
+  passed as an argument. Reconnecting does not free a channel, so the operation that asked for
+  one is rejected rather than retried.
 - `flow`: back pressure is applied to a channel. [Channel type](./types/topology.d.ts) is passed as
   an argument.
 - `drain`: back pressure is removed from a channel. Channel type is passed.
-- `remove`: channel is removed from the [pool](#sharded-connection).
+- `remove`: channel is removed from the [pool](#sharded-connection), having failed to publish.
 - `lost`: a shard has lost its connection, hence the requests awaiting their replies on it are
   re-sent. Channel type is passed.
 - `recover`: channel's topology is recovered. Channel type is passed.
@@ -504,7 +512,13 @@ Subscribe to one of the diagnostic events:
   exceptions. Channel type,
   raw [amqp message object](https://amqp-node.github.io/amqplib/channel_api.html#channel_consume)
   and the exception are passed as arguments.
+- `return`: message is returned by the broker as unroutable. Channel type and the raw
+  [amqp message object](https://amqp-node.github.io/amqplib/channel_api.html#channel_publish) are
+  passed as arguments. In the case of a [sharded connection](#sharded-connection), the message is
+  reported only once every shard has rejected it.
 - `pause`: channel is paused. Channel type is passed.
+  In the case of a [sharded connection](#sharded-connection), it means that there is no shard left
+  to publish to, be it because every one of them has rejected a publish or lost its connection.
 - `resume`: channel is resumed. Channel type is passed.
 
 In the case of a [sharded connection](#sharded-connection), an additional argument specifying the
@@ -522,6 +536,7 @@ established, there is no way to capture the initial `open` event.
 io.diagnose('flow', (type) => console.log(`Back pressure was applied to the ${type} channel`))
 
 io.diagnose('reconnect', (shard) => console.log('AMQP reconnecting', { shard }))
+io.diagnose('exhausted', (limit, shard) => console.log('AMQP channels exhausted', { limit, shard }))
 io.diagnose('open', (shard) => console.log('AMQP connection established', { shard }))
 io.diagnose('close', (error, shard) => console.log('AMQP connection closed', { message: error?.message, shard }))
 io.diagnose('error', (error, shard) => console.log('AMQP connection failed', { message: error.message, shard }))

package/source/attributes/lazy.test.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const { generate } = require('randomstring')
-const { timeout } = require('@toa.io/generic')
+const { timeout } = require('../../test/helpers')
 
 const { lazy } = require('./lazy')
 

package/source/attributes/track.test.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const { generate } = require('randomstring')
-const { promex } = require('@toa.io/generic')
+const { Promex } = require('promex')
 
 const { track } = require('./track')
 
@@ -35,8 +35,8 @@ let test
 beforeEach(() => {
   jest.clearAllMocks()
 
-  done = promex()
-  undone = promex()
+  done = new Promex()
+  undone = new Promex()
   test = new Test()
 })
 

package/source/channel.js

@@ -42,6 +42,8 @@ class Channel {
 
   #diagnostics = emitter.create()
 
+  #closed = false
+
   /**
    * @param {comq.amqp.Connection} connection
    * @param {comq.Topology} topology
@@ -126,6 +128,21 @@ class Channel {
     }
   }
 
+  /**
+   * Gives the channel back. A connection has a limited number of them, and one whose
+   * IO is done with it would otherwise be held until the whole connection goes.
+   */
+  async close () {
+    if (this.#closed) return
+
+    this.#closed = true
+
+    await this.seal()
+
+    // a channel that went down with its connection is the outcome this asks for
+    await this.#channel?.close().catch(noop)
+  }
+
   async seal () {
     this.#sealed = true
 
@@ -142,6 +159,10 @@ class Channel {
     this.#diagnostics.off(event, listener)
   }
 
+  get closed () {
+    return this.#closed
+  }
+
   async recover (connection) {
     this.#connection = connection
 

package/source/connection.js

@@ -50,10 +50,7 @@ class Connection {
    * @param {string} url
    */
   constructor (url) {
-    this.#url = url
-
-    // EventEmitter throws on 'error' with no listeners
-    this.#diagnostics.on('error', noop)
+    this.#url = heartbeaten(url)
   }
 
   get connected () {
@@ -102,6 +99,7 @@ class Connection {
       const topology = presets[type]
       const channel = await channels.create(this.#connection, topology, index)
 
+      this.#channels = this.#channels.filter((channel) => !channel.closed)
       this.#channels.push(channel)
 
       return channel
@@ -143,6 +141,8 @@ class Connection {
     this.#diagnostics.emit('open')
 
     try {
+      this.#channels = this.#channels.filter((channel) => !channel.closed)
+
       for (const channel of this.#channels) await channel.recover(connection)
     } catch (exception) {
       this.#diagnostics.emit('error', exception)
@@ -200,10 +200,30 @@ class Connection {
     }
 
     connection.removeAllListeners()
-    connection.connection?.stream?.destroy()
+
+    // amqplib keeps its heartbeater running on a connection it does not know is
+    // gone and emits 'error' on it, which throws once no listener is left
+    connection.on('error', noop)
+
+    // a socket destroyed without an error tells amqplib nothing, leaving its
+    // timers running and everything pending on it hanging forever
+    connection.connection?.stream?.destroy(silence())
   }
 
-  #recover () {
+  /**
+   * @param {Error} [exception]
+   * @return {Promise<void> | false}
+   */
+  #recover (exception) {
+    // a connection that has no channel left to give is not something reconnecting
+    // fixes, and `#recovery` on a connection that is perfectly well is a promise
+    // nobody ever resolves — returning `false` lets the caller see the refusal
+    if (exception?.message === EXHAUSTED) {
+      this.#diagnostics.emit('exhausted', this.#connection?.connection?.channelMax)
+
+      return false
+    }
+
     return this.#recovery
   }
 
@@ -215,7 +235,7 @@ class Connection {
    * @return {Promise<comq.amqp.Connection>}
    */
   async #connect () {
-    const connecting = amqp.connect(this.#url, { timeout: CONNECT_MS })
+    const connecting = amqp.connect(this.#url, SOCKET_OPTIONS)
 
     let timer
 
@@ -248,19 +268,23 @@ class Connection {
    * @param {comq.amqp.Connection} connection
    * @param {number} [timeoutMs]
    */
-  #armWatchdog (connection, timeoutMs = global.COMQ_TESTING_WATCHDOG_INTERVAL ?? WATCHDOG_MS) {
+  #armWatchdog (connection, timeoutMs = tolerance(connection)) {
+    this.#disarmWatchdog()
+
     const socket = connection.connection?.stream
 
-    if (socket === undefined) return
+    if (socket === undefined || timeoutMs === undefined) return
 
     const reset = () => {
       clearTimeout(this.#heartbeatTimer)
-      this.#heartbeatTimer = setTimeout(() => socket.destroy(), timeoutMs)
+      // destroying a socket without an error tells amqplib nothing: it only
+      // listens for 'error' and 'end', so a bare destroy() leaves the connection
+      // silently dead — no 'close' event, no recovery, every pending operation
+      // hanging forever
+      this.#heartbeatTimer = setTimeout(() => socket.destroy(silence()), timeoutMs)
       this.#heartbeatTimer.unref()
     }
 
-    this.#disarmWatchdog()
-
     this.#heartbeatSocket = socket
     this.#heartbeatReset = reset
 
@@ -289,7 +313,25 @@ class Connection {
   }
 }
 
+/**
+ * The heartbeat to ask for when the caller has not, in seconds. A broker is free
+ * to suggest one that leaves a connection lost for minutes before anything
+ * notices, and RabbitMQ suggests 60 by default.
+ *
+ * @type {number}
+ */
+const HEARTBEAT_S = 15
+
 /** @type {number} */
+const KEEPALIVE_MS = 10_000
+
+/** How many heartbeats a connection may miss before it is destroyed. */
+const MISSED_HEARTBEATS = 3
+
+/** @type {number} */
+const WATCHDOG_MIN_MS = 15_000
+
+/** The watchdog interval used when the negotiated heartbeat is unknown. */
 const WATCHDOG_MS = 60_000
 
 /** @type {number} */
@@ -298,6 +340,18 @@ const CONNECT_MS = 30_000
 /** @type {number} */
 const SHUTDOWN_MS = 5_000
 
+const SOCKET_OPTIONS = {
+  timeout: CONNECT_MS,
+  // a peer that went away without a word is noticed by the kernel as well
+  keepAlive: true,
+  keepAliveDelay: KEEPALIVE_MS
+}
+
+const HEARTBEAT_SET = /[?&]heartbeat=/
+
+/** What amqplib says when the negotiated channel limit leaves no identifier free. */
+const EXHAUSTED = 'No channels left to allocate'
+
 const TRANSIENT_CODES = new Set([
   'ECONNREFUSED',
   'EAI_AGAIN',
@@ -323,6 +377,52 @@ function expiration () {
   return delay(timeoutMs, undefined, { ref: false })
 }
 
+/**
+ * amqplib reads the heartbeat from the URL only, hence it cannot be passed along
+ * with the socket options.
+ *
+ * @param {string} url
+ * @return {string}
+ */
+function heartbeaten (url) {
+  if (HEARTBEAT_SET.test(url)) return url
+
+  return url + (url.includes('?') ? '&' : '?') + 'heartbeat=' + HEARTBEAT_S
+}
+
+/**
+ * The watchdog measures silence, so it cannot be shorter than the interval at
+ * which a healthy broker is expected to say something, which is a heartbeat
+ * frame every half a heartbeat. A connection that has agreed to no heartbeats
+ * says nothing at all while it is idle, leaving nothing to measure.
+ *
+ * @param {comq.amqp.Connection} connection
+ * @return {number | undefined}
+ */
+function tolerance (connection) {
+  const override = global.COMQ_TESTING_WATCHDOG_INTERVAL
+
+  if (override !== undefined) return override
+
+  const heartbeat = connection.connection?.heartbeat
+
+  if (heartbeat === undefined) return WATCHDOG_MS
+  if (heartbeat === 0) return undefined
+
+  return Math.max(heartbeat * MISSED_HEARTBEATS * 1000, WATCHDOG_MIN_MS)
+}
+
+/**
+ * @return {Error}
+ */
+function silence () {
+  const exception = new Error('Connection is silent')
+
+  exception.code = 'ETIMEDOUT'
+
+  return exception
+}
+
 function noop () {}
 
 exports.Connection = Connection

package/source/emitter.js

@@ -2,8 +2,30 @@
 
 const { EventEmitter } = require('node:events')
 
+/**
+ * A listener must not be able to break what it observes: an exception thrown by
+ * one used to leave the reconnection it was reporting on unfinished, or take the
+ * process down through the emitter amqplib was calling.
+ */
+class Emitter extends EventEmitter {
+  emit (event, ...args) {
+    // raw listeners are used so that `once` still removes itself
+    const listeners = this.rawListeners(event)
+
+    for (const listener of listeners) {
+      try {
+        listener.apply(this, args)
+      } catch {
+        // a diagnostic listener has no one to report to
+      }
+    }
+
+    return listeners.length > 0
+  }
+}
+
 function create () {
-  const emitter = new EventEmitter()
+  const emitter = new Emitter()
 
   emitter.setMaxListeners(0)
 

package/source/encoders/index.js

@@ -1,9 +1,7 @@
 'use strict'
 
 const json = require('./json')
-const msgpack = require('./msgpack')
 const plain = require('./plain')
 
 exports['application/json'] = json
-exports['application/msgpack'] = msgpack
 exports['text/plain'] = plain

package/source/events.js

@@ -1,7 +1,7 @@
 'use strict'
 
 /** @type {comq.diagnostics.Event[]} */
-exports.connection = ['open', 'close', 'error', 'reconnect']
+exports.connection = ['open', 'close', 'error', 'reconnect', 'exhausted']
 
 /** @type {comq.diagnostics.Event[]} */
-exports.channel = ['flow', 'drain', 'recover', 'discard', 'pause', 'resume', 'return', 'lost']
+exports.channel = ['flow', 'drain', 'recover', 'discard', 'pause', 'resume', 'return', 'lost', 'remove']

package/source/io.js

@@ -1,7 +1,6 @@
 'use strict'
 
 const stream = require('node:stream')
-const { EventEmitter } = require('node:events')
 const { randomBytes } = require('node:crypto')
 const { setTimeout } = require('node:timers/promises')
 const { Promex } = require('promex')
@@ -11,6 +10,7 @@ const { decode } = require('./decode')
 const { encode } = require('./encode')
 const { pipeline, transform } = require('./pipeline')
 const events = require('./events')
+const emitter = require('./emitter')
 const io = require('./.io')
 
 /**
@@ -44,7 +44,7 @@ class IO {
   /** @type {Set<comq.Destroyable>} */
   #replyPipes = new Set()
 
-  #diagnostics = new EventEmitter()
+  #diagnostics = emitter.create()
 
   /**
    * @param {comq.Connection} connection
@@ -52,9 +52,6 @@ class IO {
   constructor (connection) {
     this.#connection = connection
 
-    // EventEmitter throws on 'error' with no listeners
-    this.#diagnostics.on('error', noop)
-
     for (const event of events.connection) {
       this.#connection.diagnose(event, (...args) => this.#diagnostics.emit(event, ...args))
     }
@@ -166,6 +163,12 @@ class IO {
     await this.seal()
     await this.#destroyStreams(this.#replyPipes)
     await track(this)
+
+    // a connection is shared and outlives its IOs, so the channels are given back
+    // here — held to the end of the connection, they would run it out of them
+    await Promise.all([this.#requests, this.#replies, this.#events]
+      .map((channel) => channel?.close()))
+
     await this.#connection.close()
   })
 
@@ -220,6 +223,10 @@ class IO {
     } else {
       this.#requests.diagnose('recover', this.#retransmit)
     }
+
+    // a reply that could not be routed is dropped by the broker, and the queue
+    // it was addressed to only exists again once this channel has recovered
+    this.#replies.diagnose('recover', this.#retransmit)
   }
 
   /**

package/source/shards/channel.js

@@ -31,6 +31,9 @@ class Channel {
   /** @type {Map<comq.Channel, Promex>} */
   #bench = new Map()
 
+  /** @type {boolean} */
+  #paused = false
+
   /** @type {comq.topology.type} */
   #type
 
@@ -74,6 +77,14 @@ class Channel {
     return await this.#one((channel) => channel.fire(queue, buffer, options))
   }
 
+  async close () {
+    await this.#all((channel) => channel.close())
+  }
+
+  get closed () {
+    return [...this.#channels].every((channel) => channel.closed)
+  }
+
   async seal () {
     await this.#all((channel) => channel.seal())
   }
@@ -126,6 +137,9 @@ class Channel {
       this.#alive[index] = false
       this.#down[index].resolve()
 
+      // the channel stays in the pool, yet there is one less shard to publish to
+      this.#update()
+
       if (error !== undefined && connection.closed !== true) {
         this.#diagnostics.emit(LOST, index)
       }
@@ -134,6 +148,8 @@ class Channel {
     connection.diagnose('open', () => {
       this.#alive[index] = true
       this.#down[index] = new Promex()
+
+      this.#update()
     })
   }
 
@@ -159,6 +175,7 @@ class Channel {
     for (const event of events.channel) {
       if (event === RETURN) continue // returns are retried before being reported
       if (event === LOST) continue // a shard is lost with its connection, not its channel
+      if (event === REMOVE) continue // it is the pool that removes a channel, not the channel
 
       channel.diagnose(event, (...args) => this.#diagnostics.emit(event, ...args, channel.index))
     }
@@ -214,20 +231,23 @@ class Channel {
     this.#bench.set(channel, new Promex())
     this.#channels.delete(channel)
     this.#update()
-    this.#diagnostics.emit('remove', channel.index)
+    this.#diagnostics.emit(REMOVE, channel.index)
   }
 
+  /**
+   * A shard leaves the pool when it rejects a publish, and stops being reachable
+   * when its connection is lost, which the pool is not told about. Both leave
+   * `#one` waiting, so both are reported the same way.
+   */
   #update () {
-    const from = this.#pool?.length
-    const to = this.#channels.size
-
     this.#pool = Array.from(this.#channels)
 
-    if (from === undefined) return
+    const paused = this.#reachable().length === 0
 
-    if (from !== 0 && to === 0) { this.#diagnostics.emit('pause') }
+    if (paused === this.#paused) return
 
-    if (from === 0 && to !== 0) { this.#diagnostics.emit('resume') }
+    this.#paused = paused
+    this.#diagnostics.emit(paused ? 'pause' : 'resume')
   }
 
   /**
@@ -369,6 +389,7 @@ async function create (connections, type) {
 
 const DEFAULT = ''
 const RETURN = 'return'
+const REMOVE = 'remove'
 const RETURN_HEADER = 'x-return'
 const LOST = 'lost'
 

package/source/shards/connection.js

@@ -19,9 +19,6 @@ class Connection {
   constructor (connections) {
     this.#connections = connections
 
-    // EventEmitter throws on 'error' with no listeners
-    this.#diagnostics.on('error', noop)
-
     connections.map(this.#pipe)
   }
 
@@ -75,6 +72,4 @@ class Connection {
   }
 }
 
-function noop () {}
-
 exports.Connection = Connection

package/test/amqplib.mock.js

@@ -15,17 +15,33 @@ class Channel extends EventEmitter {
   assertExchange = jest.fn(async () => undefined)
   bindQueue = jest.fn(async () => undefined)
   publish = jest.fn((_0, _1, _2, _3, resolve) => resolve?.(null))
+  close = jest.fn(async () => undefined)
 }
 
 class Connection extends EventEmitter {
-  constructor () {
+  constructor (url = '') {
     super()
 
     const stream = new EventEmitter()
 
-    stream.destroy = jest.fn()
+    // amqplib listens for 'error' and 'end' on the socket only, so destroying it
+    // without an error leaves the connection unaware and silent forever
+    stream.destroy = jest.fn((error) => {
+      if (error === undefined || stream.destroyed === true) return
+
+      stream.destroyed = true
+
+      this.emit('error', error)
+      this.emit('close', error)
+    })
+
+    // a broker accepts the heartbeat the client asks for
+    const heartbeat = Number(/[?&]heartbeat=(\d+)/.exec(url)?.[1] ?? 60)
+
+    // and negotiates the channel limit down to whatever the client asked for
+    const channelMax = Number(/[?&]channelMax=(\d+)/.exec(url)?.[1] ?? 2047)
 
-    this.connection = { stream }
+    this.connection = { stream, heartbeat, channelMax }
 
     // noinspection JSValidateTypes
     this.removeAllListeners = jest.spyOn(this, 'removeAllListeners')
@@ -39,7 +55,7 @@ class Connection extends EventEmitter {
   close = jest.fn(async () => undefined)
 }
 
-const connect = async () => new Connection()
+const connect = async (url) => new Connection(url)
 
 /** @type {jest.MockedObject<import('amqplib')>} */
 const amqplib = {

package/test/channel.fixtures.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { random, flip } = require('@toa.io/generic')
+const { random, flip } = require('./helpers')
 
 const preset = () => ({
   prefetch: random(10),

package/test/channel.test.js

@@ -4,7 +4,7 @@
 
 const { randomBytes } = require('node:crypto')
 const { generate } = require('randomstring')
-const { flip, random, timeout } = require('@toa.io/generic')
+const { flip, random, timeout } = require('./helpers')
 
 const backpressure = require('./backpressure')
 const { amqplib } = require('./amqplib.mock')
@@ -503,6 +503,49 @@ describe('seal', () => {
   })
 })
 
+describe('close', () => {
+  const queue = generate()
+  const consumer = jest.fn()
+
+  beforeEach(async () => {
+    channel = await create(connection, topology)
+    chan = await getCreatedChannel()
+  })
+
+  it('should give the channel back', async () => {
+    await channel.close()
+
+    expect(chan.close).toHaveBeenCalled()
+    expect(channel.closed).toStrictEqual(true)
+  })
+
+  it('should stop consuming before giving it back', async () => {
+    await channel.consume(queue, consumer)
+
+    const { consumerTag: tag } = await chan.consume.mock.results[0].value
+
+    await channel.close()
+
+    expect(chan.cancel).toHaveBeenCalledWith(tag)
+    expect(chan.cancel.mock.invocationCallOrder[0])
+      .toBeLessThan(chan.close.mock.invocationCallOrder[0])
+  })
+
+  it('should give it back once', async () => {
+    await channel.close()
+    await channel.close()
+
+    expect(chan.close).toHaveBeenCalledTimes(1)
+  })
+
+  // one that went down with its connection is the outcome this asks for
+  it('should ignore one that is already gone', async () => {
+    chan.close.mockImplementationOnce(async () => { throw new Error() })
+
+    await expect(channel.close()).resolves.not.toThrow()
+  })
+})
+
 describe('back pressure', () => {
   const exchange = generate()
   const queue = generate()

package/test/connection.mock.js

@@ -12,6 +12,8 @@ const channel = /** @type {jest.MockedFunction<(sharded?: boolean, index?: numbe
     publish: jest.fn(async () => undefined),
     diagnose: jest.fn(async () => undefined),
     seal: jest.fn(async () => undefined),
+    close: jest.fn(async () => undefined),
+    closed: false,
     recover: jest.fn(async () => undefined)
   }))