comq 0.15.6 → 0.15.8

package/source/connection.js

@@ -50,10 +50,7 @@ class Connection {
    * @param {string} url
    */
   constructor (url) {
-    this.#url = url
-
-    // EventEmitter throws on 'error' with no listeners
-    this.#diagnostics.on('error', noop)
+    this.#url = heartbeaten(url)
   }
 
   get connected () {
@@ -73,6 +70,11 @@ class Connection {
 
     await this.#opening
 
+    if (this.#closed) return
+
+    // close may have landed after this attempt succeeded but before `#opening` was cleared
+    if (this.#connection === undefined) return this.open()
+
     this.#running = true
   }
 
@@ -109,16 +111,18 @@ class Connection {
   #open = async (retry) => {
     if (this.#closed) return
 
+    // the initial connect finishes before the caller can subscribe
+    if (this.#running) this.#diagnostics.emit('reconnect')
+
     /** @type {comq.amqp.Connection} */
     let connection
 
     try {
-      connection = await amqp.connect(this.#url, { timeout: CONNECT_MS })
+      connection = await this.#connect()
     } catch (exception) {
       if (this.#closed) return
       if (!this.#transient(exception)) throw exception
 
-      // attempts are made until one succeeds, so an outage is observable only here
       this.#diagnostics.emit('error', exception)
 
       return retry
@@ -193,30 +197,78 @@ class Connection {
     }
 
     connection.removeAllListeners()
-    connection.connection?.stream?.destroy()
+
+    // amqplib keeps its heartbeater running on a connection it does not know is
+    // gone and emits 'error' on it, which throws once no listener is left
+    connection.on('error', noop)
+
+    // a socket destroyed without an error tells amqplib nothing, leaving its
+    // timers running and everything pending on it hanging forever
+    connection.connection?.stream?.destroy(silence())
   }
 
   #recover () {
     return this.#recovery
   }
 
+  /**
+   * amqplib's `timeout` is a socket idle timer and starts only after DNS.
+   * After a machine wakes, `getaddrinfo` itself can hang, so the attempt is
+   * also bounded here.
+   *
+   * @return {Promise<comq.amqp.Connection>}
+   */
+  async #connect () {
+    const connecting = amqp.connect(this.#url, SOCKET_OPTIONS)
+
+    let timer
+
+    const expired = new Promise((_resolve, reject) => {
+      timer = setTimeout(() => {
+        const exception = new Error('connect ETIMEDOUT')
+
+        exception.code = 'ETIMEDOUT'
+
+        reject(exception)
+      }, CONNECT_MS)
+
+      timer.unref()
+    })
+
+    expired.catch(noop)
+
+    try {
+      return await Promise.race([connecting, expired])
+    } catch (exception) {
+      connecting.then((connection) => this.#shutdown(connection), noop)
+
+      throw exception
+    } finally {
+      clearTimeout(timer)
+    }
+  }
+
   /**
    * @param {comq.amqp.Connection} connection
    * @param {number} [timeoutMs]
    */
-  #armWatchdog (connection, timeoutMs = global.COMQ_TESTING_WATCHDOG_INTERVAL ?? WATCHDOG_MS) {
+  #armWatchdog (connection, timeoutMs = tolerance(connection)) {
+    this.#disarmWatchdog()
+
     const socket = connection.connection?.stream
 
-    if (socket === undefined) return
+    if (socket === undefined || timeoutMs === undefined) return
 
     const reset = () => {
       clearTimeout(this.#heartbeatTimer)
-      this.#heartbeatTimer = setTimeout(() => socket.destroy(), timeoutMs)
+      // destroying a socket without an error tells amqplib nothing: it only
+      // listens for 'error' and 'end', so a bare destroy() leaves the connection
+      // silently dead — no 'close' event, no recovery, every pending operation
+      // hanging forever
+      this.#heartbeatTimer = setTimeout(() => socket.destroy(silence()), timeoutMs)
       this.#heartbeatTimer.unref()
     }
 
-    this.#disarmWatchdog()
-
     this.#heartbeatSocket = socket
     this.#heartbeatReset = reset
 
@@ -245,7 +297,25 @@ class Connection {
   }
 }
 
+/**
+ * The heartbeat to ask for when the caller has not, in seconds. A broker is free
+ * to suggest one that leaves a connection lost for minutes before anything
+ * notices, and RabbitMQ suggests 60 by default.
+ *
+ * @type {number}
+ */
+const HEARTBEAT_S = 15
+
+/** @type {number} */
+const KEEPALIVE_MS = 10_000
+
+/** How many heartbeats a connection may miss before it is destroyed. */
+const MISSED_HEARTBEATS = 3
+
 /** @type {number} */
+const WATCHDOG_MIN_MS = 15_000
+
+/** The watchdog interval used when the negotiated heartbeat is unknown. */
 const WATCHDOG_MS = 60_000
 
 /** @type {number} */
@@ -254,6 +324,15 @@ const CONNECT_MS = 30_000
 /** @type {number} */
 const SHUTDOWN_MS = 5_000
 
+const SOCKET_OPTIONS = {
+  timeout: CONNECT_MS,
+  // a peer that went away without a word is noticed by the kernel as well
+  keepAlive: true,
+  keepAliveDelay: KEEPALIVE_MS
+}
+
+const HEARTBEAT_SET = /[?&]heartbeat=/
+
 const TRANSIENT_CODES = new Set([
   'ECONNREFUSED',
   'EAI_AGAIN',
@@ -279,6 +358,52 @@ function expiration () {
   return delay(timeoutMs, undefined, { ref: false })
 }
 
+/**
+ * amqplib reads the heartbeat from the URL only, hence it cannot be passed along
+ * with the socket options.
+ *
+ * @param {string} url
+ * @return {string}
+ */
+function heartbeaten (url) {
+  if (HEARTBEAT_SET.test(url)) return url
+
+  return url + (url.includes('?') ? '&' : '?') + 'heartbeat=' + HEARTBEAT_S
+}
+
+/**
+ * The watchdog measures silence, so it cannot be shorter than the interval at
+ * which a healthy broker is expected to say something, which is a heartbeat
+ * frame every half a heartbeat. A connection that has agreed to no heartbeats
+ * says nothing at all while it is idle, leaving nothing to measure.
+ *
+ * @param {comq.amqp.Connection} connection
+ * @return {number | undefined}
+ */
+function tolerance (connection) {
+  const override = global.COMQ_TESTING_WATCHDOG_INTERVAL
+
+  if (override !== undefined) return override
+
+  const heartbeat = connection.connection?.heartbeat
+
+  if (heartbeat === undefined) return WATCHDOG_MS
+  if (heartbeat === 0) return undefined
+
+  return Math.max(heartbeat * MISSED_HEARTBEATS * 1000, WATCHDOG_MIN_MS)
+}
+
+/**
+ * @return {Error}
+ */
+function silence () {
+  const exception = new Error('Connection is silent')
+
+  exception.code = 'ETIMEDOUT'
+
+  return exception
+}
+
 function noop () {}
 
 exports.Connection = Connection

package/source/emitter.js

@@ -2,8 +2,30 @@
 
 const { EventEmitter } = require('node:events')
 
+/**
+ * A listener must not be able to break what it observes: an exception thrown by
+ * one used to leave the reconnection it was reporting on unfinished, or take the
+ * process down through the emitter amqplib was calling.
+ */
+class Emitter extends EventEmitter {
+  emit (event, ...args) {
+    // raw listeners are used so that `once` still removes itself
+    const listeners = this.rawListeners(event)
+
+    for (const listener of listeners) {
+      try {
+        listener.apply(this, args)
+      } catch {
+        // a diagnostic listener has no one to report to
+      }
+    }
+
+    return listeners.length > 0
+  }
+}
+
 function create () {
-  const emitter = new EventEmitter()
+  const emitter = new Emitter()
 
   emitter.setMaxListeners(0)
 

package/source/encoders/index.js

@@ -1,9 +1,7 @@
 'use strict'
 
 const json = require('./json')
-const msgpack = require('./msgpack')
 const plain = require('./plain')
 
 exports['application/json'] = json
-exports['application/msgpack'] = msgpack
 exports['text/plain'] = plain

package/source/events.js

@@ -1,7 +1,7 @@
 'use strict'
 
 /** @type {comq.diagnostics.Event[]} */
-exports.connection = ['open', 'close', 'error']
+exports.connection = ['open', 'close', 'error', 'reconnect']
 
 /** @type {comq.diagnostics.Event[]} */
-exports.channel = ['flow', 'drain', 'recover', 'discard', 'pause', 'resume', 'return', 'lost']
+exports.channel = ['flow', 'drain', 'recover', 'discard', 'pause', 'resume', 'return', 'lost', 'remove']

package/source/io.js

@@ -1,7 +1,6 @@
 'use strict'
 
 const stream = require('node:stream')
-const { EventEmitter } = require('node:events')
 const { randomBytes } = require('node:crypto')
 const { setTimeout } = require('node:timers/promises')
 const { Promex } = require('promex')
@@ -11,6 +10,7 @@ const { decode } = require('./decode')
 const { encode } = require('./encode')
 const { pipeline, transform } = require('./pipeline')
 const events = require('./events')
+const emitter = require('./emitter')
 const io = require('./.io')
 
 /**
@@ -44,7 +44,7 @@ class IO {
   /** @type {Set<comq.Destroyable>} */
   #replyPipes = new Set()
 
-  #diagnostics = new EventEmitter()
+  #diagnostics = emitter.create()
 
   /**
    * @param {comq.Connection} connection
@@ -52,9 +52,6 @@ class IO {
   constructor (connection) {
     this.#connection = connection
 
-    // EventEmitter throws on 'error' with no listeners
-    this.#diagnostics.on('error', noop)
-
     for (const event of events.connection) {
       this.#connection.diagnose(event, (...args) => this.#diagnostics.emit(event, ...args))
     }
@@ -220,6 +217,10 @@ class IO {
     } else {
       this.#requests.diagnose('recover', this.#retransmit)
     }
+
+    // a reply that could not be routed is dropped by the broker, and the queue
+    // it was addressed to only exists again once this channel has recovered
+    this.#replies.diagnose('recover', this.#retransmit)
   }
 
   /**

package/source/shards/channel.js

@@ -31,6 +31,9 @@ class Channel {
   /** @type {Map<comq.Channel, Promex>} */
   #bench = new Map()
 
+  /** @type {boolean} */
+  #paused = false
+
   /** @type {comq.topology.type} */
   #type
 
@@ -126,6 +129,9 @@ class Channel {
       this.#alive[index] = false
       this.#down[index].resolve()
 
+      // the channel stays in the pool, yet there is one less shard to publish to
+      this.#update()
+
       if (error !== undefined && connection.closed !== true) {
         this.#diagnostics.emit(LOST, index)
       }
@@ -134,6 +140,8 @@ class Channel {
     connection.diagnose('open', () => {
       this.#alive[index] = true
       this.#down[index] = new Promex()
+
+      this.#update()
     })
   }
 
@@ -159,6 +167,7 @@ class Channel {
     for (const event of events.channel) {
       if (event === RETURN) continue // returns are retried before being reported
       if (event === LOST) continue // a shard is lost with its connection, not its channel
+      if (event === REMOVE) continue // it is the pool that removes a channel, not the channel
 
       channel.diagnose(event, (...args) => this.#diagnostics.emit(event, ...args, channel.index))
     }
@@ -214,20 +223,23 @@ class Channel {
     this.#bench.set(channel, new Promex())
     this.#channels.delete(channel)
     this.#update()
-    this.#diagnostics.emit('remove', channel.index)
+    this.#diagnostics.emit(REMOVE, channel.index)
   }
 
+  /**
+   * A shard leaves the pool when it rejects a publish, and stops being reachable
+   * when its connection is lost, which the pool is not told about. Both leave
+   * `#one` waiting, so both are reported the same way.
+   */
   #update () {
-    const from = this.#pool?.length
-    const to = this.#channels.size
-
     this.#pool = Array.from(this.#channels)
 
-    if (from === undefined) return
+    const paused = this.#reachable().length === 0
 
-    if (from !== 0 && to === 0) { this.#diagnostics.emit('pause') }
+    if (paused === this.#paused) return
 
-    if (from === 0 && to !== 0) { this.#diagnostics.emit('resume') }
+    this.#paused = paused
+    this.#diagnostics.emit(paused ? 'pause' : 'resume')
   }
 
   /**
@@ -318,26 +330,30 @@ class Channel {
   }
 
   /**
-   * The channels of the shards that are known to be connected, falling back to the
-   * whole pool while none is, so that a publish is attempted rather than dropped.
+   * The channels of the shards that are known to be connected.
+   * An empty result means every shard is down or benched — wait rather than
+   * publish: a destroyed socket accepts a write without complaining.
    *
    * @return {comq.Channel[]}
    */
   #reachable () {
-    const reachable = this.#pool.filter((channel) => this.#alive[channel.index] !== false)
-
-    return reachable.length === 0 ? this.#pool : reachable
+    return this.#pool.filter((channel) => this.#alive[channel.index] !== false)
   }
 
   /**
    * @param {(channel: comq.Channel) => void} fn
    */
   async #one (fn) {
-    if (this.#pool.length === 0) await this.#recovery
-
-    // publishing to a lost shard is silently accepted by the destroyed socket, so
-    // it must be avoided rather than retried: nothing would report the loss twice
+    // capture before the check: recover may replace `#recovery` in between
+    const waiting = this.#recovery
     const pool = this.#reachable()
+
+    if (pool.length === 0) {
+      await waiting
+
+      return this.#one(fn)
+    }
+
     const channel = pool[Math.floor(Math.random() * pool.length)]
 
     try {
@@ -345,7 +361,7 @@ class Channel {
     } catch {
       this.#remove(channel)
 
-      return await this.#one(fn)
+      return this.#one(fn)
     }
   }
 }
@@ -365,6 +381,7 @@ async function create (connections, type) {
 
 const DEFAULT = ''
 const RETURN = 'return'
+const REMOVE = 'remove'
 const RETURN_HEADER = 'x-return'
 const LOST = 'lost'
 

package/source/shards/connection.js

@@ -19,9 +19,6 @@ class Connection {
   constructor (connections) {
     this.#connections = connections
 
-    // EventEmitter throws on 'error' with no listeners
-    this.#diagnostics.on('error', noop)
-
     connections.map(this.#pipe)
   }
 
@@ -75,6 +72,4 @@ class Connection {
   }
 }
 
-function noop () {}
-
 exports.Connection = Connection

package/test/amqplib.mock.js

@@ -18,14 +18,26 @@ class Channel extends EventEmitter {
 }
 
 class Connection extends EventEmitter {
-  constructor () {
+  constructor (url = '') {
     super()
 
     const stream = new EventEmitter()
 
-    stream.destroy = jest.fn()
+    // amqplib listens for 'error' and 'end' on the socket only, so destroying it
+    // without an error leaves the connection unaware and silent forever
+    stream.destroy = jest.fn((error) => {
+      if (error === undefined || stream.destroyed === true) return
 
-    this.connection = { stream }
+      stream.destroyed = true
+
+      this.emit('error', error)
+      this.emit('close', error)
+    })
+
+    // a broker accepts the heartbeat the client asks for
+    const heartbeat = Number(/[?&]heartbeat=(\d+)/.exec(url)?.[1] ?? 60)
+
+    this.connection = { stream, heartbeat }
 
     // noinspection JSValidateTypes
     this.removeAllListeners = jest.spyOn(this, 'removeAllListeners')
@@ -39,7 +51,7 @@ class Connection extends EventEmitter {
   close = jest.fn(async () => undefined)
 }
 
-const connect = async () => new Connection()
+const connect = async (url) => new Connection(url)
 
 /** @type {jest.MockedObject<import('amqplib')>} */
 const amqplib = {

package/test/channel.fixtures.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { random, flip } = require('@toa.io/generic')
+const { random, flip } = require('./helpers')
 
 const preset = () => ({
   prefetch: random(10),

package/test/channel.test.js

@@ -4,7 +4,7 @@
 
 const { randomBytes } = require('node:crypto')
 const { generate } = require('randomstring')
-const { flip, random, timeout } = require('@toa.io/generic')
+const { flip, random, timeout } = require('./helpers')
 
 const backpressure = require('./backpressure')
 const { amqplib } = require('./amqplib.mock')