comq 0.15.4 → 0.15.5

package/CHANGELOG.md

@@ -1,3 +1,11 @@
+## [0.15.5](https://github.com/toa-io/comq/compare/v0.15.4...v0.15.5) (2026-08-25)
+
+
+### Bug Fixes
+
+* recover from a lost connection reliably ([59b1fab](https://github.com/toa-io/comq/commit/59b1fab61d4f27ab6eb608693da6493b5b1f4ae5))
+* recover from a lost connection reliably ([#247](https://github.com/toa-io/comq/issues/247)) ([e4701cf](https://github.com/toa-io/comq/commit/e4701cfa384d5451c15d0a6bf53c82ce1da91ac5))
+
 ## [0.15.4](https://github.com/toa-io/comq/compare/v0.15.3...v0.15.4) (2026-08-16)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.15.4",
+  "version": "0.15.5",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",

package/readme.md

@@ -490,10 +490,14 @@ Subscribe to one of the diagnostic events:
 - `close`: connection is closed.
   Optional [`error`](https://amqp-node.github.io/amqplib/channel_api.html#model_events) is passed
   as an argument.
+- `error`: an attempt to restore the connection has failed. The exception is passed as an argument.
+  Attempts continue, so this event is the only way to observe an ongoing outage.
 - `flow`: back pressure is applied to a channel. [Channel type](./types/topology.d.ts) is passed as
   an argument.
 - `drain`: back pressure is removed from a channel. Channel type is passed.
 - `remove`: channel is removed from the [pool](#sharded-connection).
+- `lost`: a shard has lost its connection, hence the requests awaiting their replies on it are
+  re-sent. Channel type is passed.
 - `recover`: channel's topology is recovered. Channel type is passed.
 - `discard`: message is [discarded](#messages) as it repeatedly caused
   exceptions. Channel type,
@@ -515,6 +519,10 @@ established, there is no way to capture the initial `open` event.
 
 ```javascript
 io.diagnose('flow', (type) => console.log(`Back pressure was applied to the ${type} channel`))
+
+io.diagnose('open', () => console.log('AMQP connection established'))
+io.diagnose('close', (error) => console.log('AMQP connection closed', error?.message))
+io.diagnose('error', (error) => console.log('AMQP connection failed', error.message))
 ```
 
 # Gratitude

package/source/connection.js

@@ -1,5 +1,6 @@
 'use strict'
 
+const { setTimeout: delay } = require('node:timers/promises')
 const amqp = require('amqplib')
 const { Promex } = require('promex')
 const { retry } = require('reretry')
@@ -31,9 +32,18 @@ class Connection {
   /** @type {boolean} */
   #running = false
 
+  /** @type {boolean} */
+  #closed = false
+
   /** @type {NodeJS.Timeout | null} */
   #heartbeatTimer = null
 
+  /** @type {import('node:net').Socket | null} */
+  #heartbeatSocket = null
+
+  /** @type {(() => void) | null} */
+  #heartbeatReset = null
+
   #diagnostics = emitter.create()
 
   /**
@@ -51,6 +61,8 @@ class Connection {
   }
 
   async open () {
+    this.#closed = false
+
     if (this.#opening !== null) return this.#opening
 
     this.#opening = retry(this.#open).finally(() => { this.#opening = null })
@@ -61,9 +73,12 @@ class Connection {
   }
 
   async close () {
-    if (this.#connection === undefined) await this.#recovery
+    this.#closed = true
 
-    await this.#connection.close()
+    // a connection that is about to be established must not be left open, yet an
+    // attempt that is being retried must not hold up the shutdown
+    if (this.#opening !== null) await Promise.race([this.#opening.catch(noop), expiration()])
+    if (this.#connection !== undefined) await this.#shutdown(this.#connection)
   }
 
   createChannel = failsafe(this, this.#recover,
@@ -88,16 +103,25 @@ class Connection {
   }
 
   #open = async (retry) => {
+    if (this.#closed) return
+
     /** @type {comq.amqp.Connection} */
     let connection
 
     try {
-      connection = await amqp.connect(this.#url)
+      connection = await amqp.connect(this.#url, { timeout: CONNECT_MS })
     } catch (exception) {
-      if (this.#transient(exception)) return retry
-      else throw exception
+      if (this.#closed) return
+      if (!this.#transient(exception)) throw exception
+
+      // attempts are made until one succeeds, so an outage is observable only here
+      this.#diagnostics.emit('error', exception)
+
+      return retry
     }
 
+    if (this.#closed) return await this.#shutdown(connection)
+
     // This prevents the process from crashing; 'close' will be emitted next.
     // https://amqp-node.github.io/amqplib/channel_api.html#model_events
     connection.on('error', noop)
@@ -110,17 +134,16 @@ class Connection {
     try {
       for (const channel of this.#channels) await channel.recover(connection)
     } catch (exception) {
-      this.#disarmWatchdog()
       this.#diagnostics.emit('error', exception)
-      connection.removeAllListeners()
-
-      await connection.close().catch(noop)
-
-      if (this.#connection === connection) this.#connection = undefined
+      this.#drop(connection)
 
       return retry
     }
 
+    // the connection may have been lost while the topology was being recovered,
+    // in which case 'close' has left the reconnection to this very attempt
+    if (this.#connection !== connection) return retry
+
     this.#recovery.resolve()
     this.#recovery = new Promex()
   }
@@ -130,19 +153,45 @@ class Connection {
    * @param {Error} [error]
    */
   #close = (connection, error) => {
-    this.#disarmWatchdog()
-
     if (this.#connection !== connection) return
 
+    this.#disarmWatchdog()
     this.#diagnostics.emit('close', error)
     connection.removeAllListeners()
     this.#connection = undefined
 
-    if (error !== undefined) {
+    if (error !== undefined && !this.#closed) {
       this.open().catch((exception) => this.#diagnostics.emit('error', exception))
     }
   }
 
+  /**
+   * An AMQP connection is only closed once the broker has replied with Close-Ok,
+   * which never happens on a connection that has already been lost.
+   *
+   * @param {comq.amqp.Connection} connection
+   */
+  async #shutdown (connection) {
+    const closing = connection.close().catch(noop)
+
+    await Promise.race([closing, expiration()])
+
+    this.#drop(connection)
+  }
+
+  /**
+   * @param {comq.amqp.Connection} connection
+   */
+  #drop (connection) {
+    if (this.#connection === connection) {
+      this.#disarmWatchdog()
+      this.#connection = undefined
+    }
+
+    connection.removeAllListeners()
+    connection.connection?.stream?.destroy()
+  }
+
   #recover () {
     return this.#recovery
   }
@@ -162,6 +211,11 @@ class Connection {
       this.#heartbeatTimer.unref()
     }
 
+    this.#disarmWatchdog()
+
+    this.#heartbeatSocket = socket
+    this.#heartbeatReset = reset
+
     reset()
     socket.on('data', reset)
   }
@@ -169,6 +223,13 @@ class Connection {
   #disarmWatchdog () {
     clearTimeout(this.#heartbeatTimer)
     this.#heartbeatTimer = null
+
+    // otherwise a byte arriving on a replaced socket rearms the watchdog of a
+    // connection that is already gone, leaving the current one unguarded
+    this.#heartbeatSocket?.off('data', this.#heartbeatReset)
+
+    this.#heartbeatSocket = null
+    this.#heartbeatReset = null
   }
 
   #transient (exception) {
@@ -183,6 +244,12 @@ class Connection {
 /** @type {number} */
 const WATCHDOG_MS = 60_000
 
+/** @type {number} */
+const CONNECT_MS = 30_000
+
+/** @type {number} */
+const SHUTDOWN_MS = 5_000
+
 const TRANSIENT_CODES = new Set([
   'ECONNREFUSED',
   'EAI_AGAIN',
@@ -195,9 +262,19 @@ const TRANSIENT_CODES = new Set([
 
 const TRANSIENT_MESSAGES = new Set([
   'Socket closed abruptly during opening handshake',
-  'Client network socket disconnected before secure TLS connection was established'
+  'Client network socket disconnected before secure TLS connection was established',
+  'connect ETIMEDOUT' // amqplib reports the `timeout` option without a code
 ])
 
+/**
+ * @return {Promise<void>}
+ */
+function expiration () {
+  const timeoutMs = global.COMQ_TESTING_SHUTDOWN_TIMEOUT ?? SHUTDOWN_MS
+
+  return delay(timeoutMs, undefined, { ref: false })
+}
+
 function noop () {}
 
 exports.Connection = Connection

package/source/io.js

@@ -52,6 +52,9 @@ class IO {
   constructor (connection) {
     this.#connection = connection
 
+    // EventEmitter throws on 'error' with no listeners
+    this.#diagnostics.on('error', noop)
+
     for (const event of events.connection) {
       this.#connection.diagnose(event, (...args) => this.#diagnostics.emit(event, ...args))
     }

package/source/shards/connection.js

@@ -19,6 +19,9 @@ class Connection {
   constructor (connections) {
     this.#connections = connections
 
+    // EventEmitter throws on 'error' with no listeners
+    this.#diagnostics.on('error', noop)
+
     connections.map(this.#pipe)
   }
 
@@ -72,4 +75,6 @@ class Connection {
   }
 }
 
+function noop () {}
+
 exports.Connection = Connection

package/test/amqplib.mock.js

@@ -39,9 +39,12 @@ class Connection extends EventEmitter {
   close = jest.fn(async () => undefined)
 }
 
+const connect = async () => new Connection()
+
 /** @type {jest.MockedObject<import('amqplib')>} */
 const amqplib = {
-  connect: jest.fn(async () => new Connection())
+  connect: jest.fn(connect)
 }
 
 exports.amqplib = amqplib
+exports.connect = connect

package/test/connection.test.js

@@ -5,7 +5,7 @@
 const { generate } = require('randomstring')
 
 const { timeout, promex, random } = require('@toa.io/generic')
-const { amqplib } = require('./amqplib.mock')
+const { amqplib, connect } = require('./amqplib.mock')
 const { channel: create } = require('./connection.mock')
 const mock = { amqplib, channel: { create } }
 
@@ -26,6 +26,7 @@ const url = generate()
 
 beforeEach(() => {
   jest.clearAllMocks()
+  amqplib.connect.mockImplementation(connect)
 
   connection = new Connection(url)
 })
@@ -36,12 +37,13 @@ describe('initial connection', () => {
   it('should connect', async () => {
     await connection.open()
 
-    expect(amqplib.connect).toHaveBeenCalledWith(url)
+    expect(amqplib.connect).toHaveBeenCalledWith(url, { timeout: expect.any(Number) })
   })
 
   it.each(/** @type {[string, Partial<Error>][]} */[
     ['Socket closed', { message: 'Socket closed abruptly during opening handshake' }],
     ['TLS disconnect', { message: 'Client network socket disconnected before secure TLS connection was established' }],
+    ['connect timeout', { message: 'connect ETIMEDOUT' }],
     ['ECONNREFUSED', { code: 'ECONNREFUSED' }],
     ['EAI_AGAIN', { code: 'EAI_AGAIN' }],
     ['ENOTFOUND', { code: 'ENOTFOUND' }],
@@ -157,6 +159,58 @@ describe('reconnection', () => {
     expect(unhandled).not.toHaveBeenCalled()
   })
 
+  it('should reconnect when connection is lost while channels recover', async () => {
+    const channel = await connection.createChannel('request')
+
+    // the connection is lost right after the topology has been recovered on it
+    channel.recover.mockImplementationOnce(async (connection) => {
+      connection.emit('close', new Error('lost again'))
+    })
+
+    conn.emit('close', new Error('lost'))
+
+    await expect(connection.createChannel('event')).resolves.toBeDefined()
+
+    expect(amqplib.connect).toHaveBeenCalledTimes(3)
+  }, 10000)
+
+  it('should not wait for a connection that failed to recover to close', async () => {
+    const channel = await connection.createChannel('request')
+
+    channel.recover
+      .mockRejectedValueOnce(new Error('Channel closed'))
+      .mockResolvedValue(undefined)
+
+    // Close-Ok never arrives on a connection that has already been lost
+    conn.close.mockImplementation(() => new Promise(() => undefined))
+    conn.emit('close', new Error('lost'))
+
+    await timeout(0)
+
+    const failed = await amqplib.connect.mock.results[1].value
+
+    failed.close.mockImplementation(() => new Promise(() => undefined))
+
+    await expect(connection.createChannel('event')).resolves.toBeDefined()
+
+    expect(failed.connection.stream.destroy).toHaveBeenCalled()
+  }, 10000)
+
+  it('should emit error on a failed connection attempt', async () => {
+    const errors = []
+    const exception = { code: 'ECONNREFUSED' }
+
+    connection.diagnose('error', (error) => errors.push(error))
+    amqplib.connect.mockImplementationOnce(async () => { throw exception })
+
+    conn.emit('close', new Error('lost'))
+
+    // the attempt is retried, hence the channel is created on the next connection
+    await connection.createChannel('event')
+
+    expect(errors).toContain(exception)
+  }, 10000)
+
   it('should ignore close from a stale connection', async () => {
     const stale = conn
     const closeHandler = stale.on.mock.calls.find(([event]) => event === 'close')[1]
@@ -289,6 +343,28 @@ describe('watchdog', () => {
     expect(conn.connection.stream.destroy).toHaveBeenCalled()
   })
 
+  it('should not let a replaced socket disarm the watchdog', async () => {
+    jest.useFakeTimers()
+
+    await connection.open()
+
+    const stale = await amqplib.connect.mock.results[0].value
+
+    stale.emit('close', new Error('lost'))
+
+    await jest.advanceTimersByTimeAsync(1)
+
+    const live = await amqplib.connect.mock.results[1].value
+
+    await jest.advanceTimersByTimeAsync(30_000)
+
+    stale.connection.stream.emit('data', Buffer.alloc(0))
+
+    await jest.advanceTimersByTimeAsync(30_000)
+
+    expect(live.connection.stream.destroy).toHaveBeenCalled()
+  })
+
   it('should reset watchdog on socket data', async () => {
     jest.useFakeTimers()
 
@@ -311,6 +387,14 @@ describe('watchdog', () => {
 })
 
 describe('close', () => {
+  beforeEach(() => {
+    global.COMQ_TESTING_SHUTDOWN_TIMEOUT = 10
+  })
+
+  afterEach(() => {
+    delete global.COMQ_TESTING_SHUTDOWN_TIMEOUT
+  })
+
   it('should close connection', async () => {
     await connection.open()
     await connection.close()
@@ -331,6 +415,49 @@ describe('close', () => {
 
     expect(conn.close).toHaveBeenCalled()
   })
+
+  it('should not wait for Close-Ok that never arrives', async () => {
+    await connection.open()
+
+    /** @type {jest.MockedObject<comq.amqp.Connection>} */
+    const conn = await amqplib.connect.mock.results[0].value
+
+    conn.close.mockImplementation(() => new Promise(() => undefined))
+
+    await connection.close()
+
+    expect(conn.connection.stream.destroy).toHaveBeenCalled()
+  })
+
+  it('should open again after close', async () => {
+    await connection.open()
+    await connection.close()
+    await connection.open()
+
+    await expect(connection.createChannel('request')).resolves.toBeDefined()
+
+    expect(amqplib.connect).toHaveBeenCalledTimes(2)
+  })
+
+  it('should stop reconnecting', async () => {
+    await connection.open()
+
+    const conn = await amqplib.connect.mock.results[0].value
+    const attempts = []
+
+    connection.diagnose('error', (exception) => attempts.push(exception))
+    amqplib.connect.mockImplementation(async () => { throw { code: 'ECONNREFUSED' } }) // eslint-disable-line
+
+    conn.emit('close', new Error('lost'))
+
+    await connection.close()
+
+    const failed = attempts.length
+
+    await timeout(1500)
+
+    expect(attempts.length).toStrictEqual(failed)
+  }, 10000)
 })
 
 describe('diagnostics', () => {

package/test/io.diagnostics.test.js

@@ -54,6 +54,13 @@ describe.each(['request', 'reply', 'event'])('%s channel events',
       })
   })
 
+it('should not throw on error without listeners', async () => {
+  const call = connection.diagnose.mock.calls.find(([event]) => event === 'error')
+  const emit = call[1]
+
+  expect(() => emit(new Error(generate()))).not.toThrow()
+})
+
 it.each(['open', 'close'])('should re-emit %s from connection',
   /**
    * @param {comq.diagnostics.Event} event

package/types/io.d.ts

@@ -59,7 +59,7 @@ declare namespace comq {
 
     diagnose(event: 'open', listener: (index?: number) => void): void
 
-    diagnose(event: 'close', listener: (index?: number) => void): void
+    diagnose(event: 'close', listener: (error?: Error, index?: number) => void): void
 
     diagnose(event: 'error', listener: (error: Error, index?: number) => void): void
 
@@ -69,6 +69,8 @@ declare namespace comq {
 
     diagnose(event: 'remove', listener: (index?: number) => void): void
 
+    diagnose(event: 'lost', listener: (channel: _topology.type, index?: number) => void): void
+
     diagnose(event: 'recover', listener: (channel: _topology.type, index?: number) => void): void
 
     diagnose(event: 'discard', listener: (channel: _topology.type, message: any, index?: number) => void): void