comq 0.15.3 → 0.15.5

package/CHANGELOG.md

@@ -1,3 +1,18 @@
+## [0.15.5](https://github.com/toa-io/comq/compare/v0.15.4...v0.15.5) (2026-08-25)
+
+
+### Bug Fixes
+
+* recover from a lost connection reliably ([59b1fab](https://github.com/toa-io/comq/commit/59b1fab61d4f27ab6eb608693da6493b5b1f4ae5))
+* recover from a lost connection reliably ([#247](https://github.com/toa-io/comq/issues/247)) ([e4701cf](https://github.com/toa-io/comq/commit/e4701cfa384d5451c15d0a6bf53c82ce1da91ac5))
+
+## [0.15.4](https://github.com/toa-io/comq/compare/v0.15.3...v0.15.4) (2026-08-16)
+
+
+### Bug Fixes
+
+* remove request() timeout ([#246](https://github.com/toa-io/comq/issues/246)) ([d432388](https://github.com/toa-io/comq/commit/d4323889bcad867a870d0ed351b92098f6aad551))
+
 ## [0.15.3](https://github.com/toa-io/comq/compare/v0.15.2...v0.15.3) (2026-08-04)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.15.3",
+  "version": "0.15.5",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",

package/readme.md

@@ -103,11 +103,10 @@ await io.reply('add_numbers', ({ a, b }) => (a + b))
 
 ## Request
 
-`async IO.request(queue: string, payload: any, encoding?: string, timeout?: number): any`
+`async IO.request(queue: string, payload: any, encoding?: string): any`
 
 Send encoded Request message with `replyTo` and `correlationId` properties set and
-return decoded Reply content. Optional `timeout` (ms) rejects with `ETIMEDOUT` if no
-Reply arrives in time; encoding may be omitted when only `timeout` is passed.
+return decoded Reply content. The promise stays pending until the Reply arrives.
 
 On the initial call, queues for Requests and Replies are asserted.
 
@@ -115,7 +114,6 @@ On the initial call, queues for Requests and Replies are asserted.
 
 ```javascript
 const sum = await io.request('add_numbers', { a: 1, b: 2 })
-const sumOrTimeout = await io.request('add_numbers', { a: 1, b: 2 }, 5000)
 ```
 
 ## Consumption
@@ -332,8 +330,7 @@ Outgoing messages are sent to a single connection chosen at random from the shar
 underlying connection or experience channel [back pressure](#flow-control) on a corresponding channel are removed from
 the pool until the issue is resolved. Pending messages meeting these conditions are immediately routed among the
 remaining shards in the pool. If no shards are available, `send` / `publish` / `request` wait until a shard's
-connection is re-established (unbounded). For RPC, pass a [`request` timeout](#request) so callers fail with
-`ETIMEDOUT` instead of hanging while the pool is empty or a Reply never arrives.
+connection is re-established.
 
 Incoming messages are consumed from all shards.
 
@@ -493,10 +490,14 @@ Subscribe to one of the diagnostic events:
 - `close`: connection is closed.
   Optional [`error`](https://amqp-node.github.io/amqplib/channel_api.html#model_events) is passed
   as an argument.
+- `error`: an attempt to restore the connection has failed. The exception is passed as an argument.
+  Attempts continue, so this event is the only way to observe an ongoing outage.
 - `flow`: back pressure is applied to a channel. [Channel type](./types/topology.d.ts) is passed as
   an argument.
 - `drain`: back pressure is removed from a channel. Channel type is passed.
 - `remove`: channel is removed from the [pool](#sharded-connection).
+- `lost`: a shard has lost its connection, hence the requests awaiting their replies on it are
+  re-sent. Channel type is passed.
 - `recover`: channel's topology is recovered. Channel type is passed.
 - `discard`: message is [discarded](#messages) as it repeatedly caused
   exceptions. Channel type,
@@ -518,6 +519,10 @@ established, there is no way to capture the initial `open` event.
 
 ```javascript
 io.diagnose('flow', (type) => console.log(`Back pressure was applied to the ${type} channel`))
+
+io.diagnose('open', () => console.log('AMQP connection established'))
+io.diagnose('close', (error) => console.log('AMQP connection closed', error?.message))
+io.diagnose('error', (error) => console.log('AMQP connection failed', error.message))
 ```
 
 # Gratitude

package/source/connection.js

@@ -1,5 +1,6 @@
 'use strict'
 
+const { setTimeout: delay } = require('node:timers/promises')
 const amqp = require('amqplib')
 const { Promex } = require('promex')
 const { retry } = require('reretry')
@@ -31,9 +32,18 @@ class Connection {
   /** @type {boolean} */
   #running = false
 
+  /** @type {boolean} */
+  #closed = false
+
   /** @type {NodeJS.Timeout | null} */
   #heartbeatTimer = null
 
+  /** @type {import('node:net').Socket | null} */
+  #heartbeatSocket = null
+
+  /** @type {(() => void) | null} */
+  #heartbeatReset = null
+
   #diagnostics = emitter.create()
 
   /**
@@ -51,6 +61,8 @@ class Connection {
   }
 
   async open () {
+    this.#closed = false
+
     if (this.#opening !== null) return this.#opening
 
     this.#opening = retry(this.#open).finally(() => { this.#opening = null })
@@ -61,9 +73,12 @@ class Connection {
   }
 
   async close () {
-    if (this.#connection === undefined) await this.#recovery
+    this.#closed = true
 
-    await this.#connection.close()
+    // a connection that is about to be established must not be left open, yet an
+    // attempt that is being retried must not hold up the shutdown
+    if (this.#opening !== null) await Promise.race([this.#opening.catch(noop), expiration()])
+    if (this.#connection !== undefined) await this.#shutdown(this.#connection)
   }
 
   createChannel = failsafe(this, this.#recover,
@@ -88,16 +103,25 @@ class Connection {
   }
 
   #open = async (retry) => {
+    if (this.#closed) return
+
     /** @type {comq.amqp.Connection} */
     let connection
 
     try {
-      connection = await amqp.connect(this.#url)
+      connection = await amqp.connect(this.#url, { timeout: CONNECT_MS })
     } catch (exception) {
-      if (this.#transient(exception)) return retry
-      else throw exception
+      if (this.#closed) return
+      if (!this.#transient(exception)) throw exception
+
+      // attempts are made until one succeeds, so an outage is observable only here
+      this.#diagnostics.emit('error', exception)
+
+      return retry
     }
 
+    if (this.#closed) return await this.#shutdown(connection)
+
     // This prevents the process from crashing; 'close' will be emitted next.
     // https://amqp-node.github.io/amqplib/channel_api.html#model_events
     connection.on('error', noop)
@@ -110,17 +134,16 @@ class Connection {
     try {
       for (const channel of this.#channels) await channel.recover(connection)
     } catch (exception) {
-      this.#disarmWatchdog()
       this.#diagnostics.emit('error', exception)
-      connection.removeAllListeners()
-
-      await connection.close().catch(noop)
-
-      if (this.#connection === connection) this.#connection = undefined
+      this.#drop(connection)
 
       return retry
     }
 
+    // the connection may have been lost while the topology was being recovered,
+    // in which case 'close' has left the reconnection to this very attempt
+    if (this.#connection !== connection) return retry
+
     this.#recovery.resolve()
     this.#recovery = new Promex()
   }
@@ -130,19 +153,45 @@ class Connection {
    * @param {Error} [error]
    */
   #close = (connection, error) => {
-    this.#disarmWatchdog()
-
     if (this.#connection !== connection) return
 
+    this.#disarmWatchdog()
     this.#diagnostics.emit('close', error)
     connection.removeAllListeners()
     this.#connection = undefined
 
-    if (error !== undefined) {
+    if (error !== undefined && !this.#closed) {
       this.open().catch((exception) => this.#diagnostics.emit('error', exception))
     }
   }
 
+  /**
+   * An AMQP connection is only closed once the broker has replied with Close-Ok,
+   * which never happens on a connection that has already been lost.
+   *
+   * @param {comq.amqp.Connection} connection
+   */
+  async #shutdown (connection) {
+    const closing = connection.close().catch(noop)
+
+    await Promise.race([closing, expiration()])
+
+    this.#drop(connection)
+  }
+
+  /**
+   * @param {comq.amqp.Connection} connection
+   */
+  #drop (connection) {
+    if (this.#connection === connection) {
+      this.#disarmWatchdog()
+      this.#connection = undefined
+    }
+
+    connection.removeAllListeners()
+    connection.connection?.stream?.destroy()
+  }
+
   #recover () {
     return this.#recovery
   }
@@ -162,6 +211,11 @@ class Connection {
       this.#heartbeatTimer.unref()
     }
 
+    this.#disarmWatchdog()
+
+    this.#heartbeatSocket = socket
+    this.#heartbeatReset = reset
+
     reset()
     socket.on('data', reset)
   }
@@ -169,6 +223,13 @@ class Connection {
   #disarmWatchdog () {
     clearTimeout(this.#heartbeatTimer)
     this.#heartbeatTimer = null
+
+    // otherwise a byte arriving on a replaced socket rearms the watchdog of a
+    // connection that is already gone, leaving the current one unguarded
+    this.#heartbeatSocket?.off('data', this.#heartbeatReset)
+
+    this.#heartbeatSocket = null
+    this.#heartbeatReset = null
   }
 
   #transient (exception) {
@@ -183,6 +244,12 @@ class Connection {
 /** @type {number} */
 const WATCHDOG_MS = 60_000
 
+/** @type {number} */
+const CONNECT_MS = 30_000
+
+/** @type {number} */
+const SHUTDOWN_MS = 5_000
+
 const TRANSIENT_CODES = new Set([
   'ECONNREFUSED',
   'EAI_AGAIN',
@@ -195,9 +262,19 @@ const TRANSIENT_CODES = new Set([
 
 const TRANSIENT_MESSAGES = new Set([
   'Socket closed abruptly during opening handshake',
-  'Client network socket disconnected before secure TLS connection was established'
+  'Client network socket disconnected before secure TLS connection was established',
+  'connect ETIMEDOUT' // amqplib reports the `timeout` option without a code
 ])
 
+/**
+ * @return {Promise<void>}
+ */
+function expiration () {
+  const timeoutMs = global.COMQ_TESTING_SHUTDOWN_TIMEOUT ?? SHUTDOWN_MS
+
+  return delay(timeoutMs, undefined, { ref: false })
+}
+
 function noop () {}
 
 exports.Connection = Connection

package/source/io.js

@@ -3,7 +3,7 @@
 const stream = require('node:stream')
 const { EventEmitter } = require('node:events')
 const { randomBytes } = require('node:crypto')
-const { setTimeout: delay } = require('node:timers/promises')
+const { setTimeout } = require('node:timers/promises')
 const { Promex } = require('promex')
 const { memo, failsafe, lazy, track } = require('./attributes')
 
@@ -52,6 +52,9 @@ class IO {
   constructor (connection) {
     this.#connection = connection
 
+    // EventEmitter throws on 'error' with no listeners
+    this.#diagnostics.on('error', noop)
+
     for (const event of events.connection) {
       this.#connection.diagnose(event, (...args) => this.#diagnostics.emit(event, ...args))
     }
@@ -75,45 +78,24 @@ class IO {
       /**
        * @param {string} queue
        * @param {any | Readable} payload
-       * @param {comq.Encoding | number} [encoding]
-       * @param {number} [timeout]
+       * @param {comq.Encoding} [encoding]
        * @returns {Promise<any | Readable>}
        */
-      async (queue, payload, encoding, timeout) => {
-        if (typeof encoding === 'number') {
-          timeout = encoding
-          encoding = undefined
-        }
-
+      async (queue, payload, encoding) => {
         if (payload instanceof stream.Readable) {
           return pipeline(
             payload,
-            (payload) => this.request(queue, payload, encoding, timeout),
+            (payload) => this.request(queue, payload, encoding),
             this.#requests
           )
         }
 
-        const request = this.#createRequest(queue, payload, /** @type {comq.Encoding} */ encoding)
+        const request = this.#createRequest(queue, payload, encoding)
         const reply = this.#createReply(request)
 
-        const done = this.#requests.send(queue, request.buffer, request.properties)
-          .then(() => reply)
+        await this.#requests.send(queue, request.buffer, request.properties)
 
-        if (typeof timeout !== 'number') return done
-
-        try {
-          return await Promise.race([
-            done,
-            delay(timeout).then(() => Promise.reject(timeoutError(timeout)))
-          ])
-        } catch (exception) {
-          if (exception.code === 'ETIMEDOUT') {
-            request.emitter.removeAllListeners(request.properties.correlationId)
-            reply.reject(exception)
-          }
-
-          throw exception
-        }
+        return reply
       }))
 
   consume = lazy(this, this.#createEventChannel,
@@ -428,7 +410,7 @@ class IO {
     Even if these messages are lost, the reply stream will be closed anyway,
     either due to missing heartbeat or the deletion of the stream queue.
     */
-    await delay(50)
+    await setTimeout(50)
   }
 
   /**
@@ -495,19 +477,6 @@ const DEFAULT = 'application/json'
 
 const RETRANSMISSION = /** @type {Error} */ Symbol('retransmission')
 
-/**
- * @param {number} timeout
- * @returns {Error}
- */
-function timeoutError (timeout) {
-  const error = /** @type {Error & { code: string }} */ new Error(
-    `Request timed out after ${timeout}ms`)
-
-  error.code = 'ETIMEDOUT'
-
-  return error
-}
-
 function noop () {}
 
 exports.IO = IO

package/source/shards/connection.js

@@ -19,6 +19,9 @@ class Connection {
   constructor (connections) {
     this.#connections = connections
 
+    // EventEmitter throws on 'error' with no listeners
+    this.#diagnostics.on('error', noop)
+
     connections.map(this.#pipe)
   }
 
@@ -72,4 +75,6 @@ class Connection {
   }
 }
 
+function noop () {}
+
 exports.Connection = Connection

package/test/amqplib.mock.js

@@ -39,9 +39,12 @@ class Connection extends EventEmitter {
   close = jest.fn(async () => undefined)
 }
 
+const connect = async () => new Connection()
+
 /** @type {jest.MockedObject<import('amqplib')>} */
 const amqplib = {
-  connect: jest.fn(async () => new Connection())
+  connect: jest.fn(connect)
 }
 
 exports.amqplib = amqplib
+exports.connect = connect

package/test/connection.test.js

@@ -5,7 +5,7 @@
 const { generate } = require('randomstring')
 
 const { timeout, promex, random } = require('@toa.io/generic')
-const { amqplib } = require('./amqplib.mock')
+const { amqplib, connect } = require('./amqplib.mock')
 const { channel: create } = require('./connection.mock')
 const mock = { amqplib, channel: { create } }
 
@@ -26,6 +26,7 @@ const url = generate()
 
 beforeEach(() => {
   jest.clearAllMocks()
+  amqplib.connect.mockImplementation(connect)
 
   connection = new Connection(url)
 })
@@ -36,12 +37,13 @@ describe('initial connection', () => {
   it('should connect', async () => {
     await connection.open()
 
-    expect(amqplib.connect).toHaveBeenCalledWith(url)
+    expect(amqplib.connect).toHaveBeenCalledWith(url, { timeout: expect.any(Number) })
   })
 
   it.each(/** @type {[string, Partial<Error>][]} */[
     ['Socket closed', { message: 'Socket closed abruptly during opening handshake' }],
     ['TLS disconnect', { message: 'Client network socket disconnected before secure TLS connection was established' }],
+    ['connect timeout', { message: 'connect ETIMEDOUT' }],
     ['ECONNREFUSED', { code: 'ECONNREFUSED' }],
     ['EAI_AGAIN', { code: 'EAI_AGAIN' }],
     ['ENOTFOUND', { code: 'ENOTFOUND' }],
@@ -157,6 +159,58 @@ describe('reconnection', () => {
     expect(unhandled).not.toHaveBeenCalled()
   })
 
+  it('should reconnect when connection is lost while channels recover', async () => {
+    const channel = await connection.createChannel('request')
+
+    // the connection is lost right after the topology has been recovered on it
+    channel.recover.mockImplementationOnce(async (connection) => {
+      connection.emit('close', new Error('lost again'))
+    })
+
+    conn.emit('close', new Error('lost'))
+
+    await expect(connection.createChannel('event')).resolves.toBeDefined()
+
+    expect(amqplib.connect).toHaveBeenCalledTimes(3)
+  }, 10000)
+
+  it('should not wait for a connection that failed to recover to close', async () => {
+    const channel = await connection.createChannel('request')
+
+    channel.recover
+      .mockRejectedValueOnce(new Error('Channel closed'))
+      .mockResolvedValue(undefined)
+
+    // Close-Ok never arrives on a connection that has already been lost
+    conn.close.mockImplementation(() => new Promise(() => undefined))
+    conn.emit('close', new Error('lost'))
+
+    await timeout(0)
+
+    const failed = await amqplib.connect.mock.results[1].value
+
+    failed.close.mockImplementation(() => new Promise(() => undefined))
+
+    await expect(connection.createChannel('event')).resolves.toBeDefined()
+
+    expect(failed.connection.stream.destroy).toHaveBeenCalled()
+  }, 10000)
+
+  it('should emit error on a failed connection attempt', async () => {
+    const errors = []
+    const exception = { code: 'ECONNREFUSED' }
+
+    connection.diagnose('error', (error) => errors.push(error))
+    amqplib.connect.mockImplementationOnce(async () => { throw exception })
+
+    conn.emit('close', new Error('lost'))
+
+    // the attempt is retried, hence the channel is created on the next connection
+    await connection.createChannel('event')
+
+    expect(errors).toContain(exception)
+  }, 10000)
+
   it('should ignore close from a stale connection', async () => {
     const stale = conn
     const closeHandler = stale.on.mock.calls.find(([event]) => event === 'close')[1]
@@ -289,6 +343,28 @@ describe('watchdog', () => {
     expect(conn.connection.stream.destroy).toHaveBeenCalled()
   })
 
+  it('should not let a replaced socket disarm the watchdog', async () => {
+    jest.useFakeTimers()
+
+    await connection.open()
+
+    const stale = await amqplib.connect.mock.results[0].value
+
+    stale.emit('close', new Error('lost'))
+
+    await jest.advanceTimersByTimeAsync(1)
+
+    const live = await amqplib.connect.mock.results[1].value
+
+    await jest.advanceTimersByTimeAsync(30_000)
+
+    stale.connection.stream.emit('data', Buffer.alloc(0))
+
+    await jest.advanceTimersByTimeAsync(30_000)
+
+    expect(live.connection.stream.destroy).toHaveBeenCalled()
+  })
+
   it('should reset watchdog on socket data', async () => {
     jest.useFakeTimers()
 
@@ -311,6 +387,14 @@ describe('watchdog', () => {
 })
 
 describe('close', () => {
+  beforeEach(() => {
+    global.COMQ_TESTING_SHUTDOWN_TIMEOUT = 10
+  })
+
+  afterEach(() => {
+    delete global.COMQ_TESTING_SHUTDOWN_TIMEOUT
+  })
+
   it('should close connection', async () => {
     await connection.open()
     await connection.close()
@@ -331,6 +415,49 @@ describe('close', () => {
 
     expect(conn.close).toHaveBeenCalled()
   })
+
+  it('should not wait for Close-Ok that never arrives', async () => {
+    await connection.open()
+
+    /** @type {jest.MockedObject<comq.amqp.Connection>} */
+    const conn = await amqplib.connect.mock.results[0].value
+
+    conn.close.mockImplementation(() => new Promise(() => undefined))
+
+    await connection.close()
+
+    expect(conn.connection.stream.destroy).toHaveBeenCalled()
+  })
+
+  it('should open again after close', async () => {
+    await connection.open()
+    await connection.close()
+    await connection.open()
+
+    await expect(connection.createChannel('request')).resolves.toBeDefined()
+
+    expect(amqplib.connect).toHaveBeenCalledTimes(2)
+  })
+
+  it('should stop reconnecting', async () => {
+    await connection.open()
+
+    const conn = await amqplib.connect.mock.results[0].value
+    const attempts = []
+
+    connection.diagnose('error', (exception) => attempts.push(exception))
+    amqplib.connect.mockImplementation(async () => { throw { code: 'ECONNREFUSED' } }) // eslint-disable-line
+
+    conn.emit('close', new Error('lost'))
+
+    await connection.close()
+
+    const failed = attempts.length
+
+    await timeout(1500)
+
+    expect(attempts.length).toStrictEqual(failed)
+  }, 10000)
 })
 
 describe('diagnostics', () => {

package/test/io.diagnostics.test.js

@@ -54,6 +54,13 @@ describe.each(['request', 'reply', 'event'])('%s channel events',
       })
   })
 
+it('should not throw on error without listeners', async () => {
+  const call = connection.diagnose.mock.calls.find(([event]) => event === 'error')
+  const emit = call[1]
+
+  expect(() => emit(new Error(generate()))).not.toThrow()
+})
+
 it.each(['open', 'close'])('should re-emit %s from connection',
   /**
    * @param {comq.diagnostics.Event} event

package/types/io.d.ts

@@ -27,11 +27,9 @@ declare namespace comq {
   interface IO extends _diagnostics.Diagnosable {
     reply(queue: string, produce: Producer): Promise<void>
 
-    request<Reply = any, Request = any>(queue: string, payload: Request, encoding?: _encoding.Encoding, timeout?: number): Promise<Reply> | Promise<Readable>
+    request<Reply = any, Request = any>(queue: string, payload: Request, encoding?: _encoding.Encoding): Promise<Reply> | Promise<Readable>
 
-    request<Reply = any, Request = any>(queue: string, payload: Request, timeout: number): Promise<Reply> | Promise<Readable>
-
-    request(queue: string, stream: Readable, encoding?: _encoding.Encoding, timeout?: number): Promise<Readable>
+    request(queue: string, stream: Readable, encoding?: _encoding.Encoding): Promise<Readable>
 
     consume<T = any>(exchange: string, group: string, consumer: Consumer<T>): Promise<void>
 
@@ -61,7 +59,7 @@ declare namespace comq {
 
     diagnose(event: 'open', listener: (index?: number) => void): void
 
-    diagnose(event: 'close', listener: (index?: number) => void): void
+    diagnose(event: 'close', listener: (error?: Error, index?: number) => void): void
 
     diagnose(event: 'error', listener: (error: Error, index?: number) => void): void
 
@@ -71,6 +69,8 @@ declare namespace comq {
 
     diagnose(event: 'remove', listener: (index?: number) => void): void
 
+    diagnose(event: 'lost', listener: (channel: _topology.type, index?: number) => void): void
+
     diagnose(event: 'recover', listener: (channel: _topology.type, index?: number) => void): void
 
     diagnose(event: 'discard', listener: (channel: _topology.type, message: any, index?: number) => void): void

package/test/io.request.timeout.test.js

@@ -1,61 +0,0 @@
-'use strict'
-
-const { generate } = require('randomstring')
-
-const mock = require('./connection.mock')
-const { IO } = require('../source/io')
-
-/** @type {comq.IO} */
-let io
-
-/** @type {jest.MockedObject<comq.Connection>} */
-let connection
-
-const queue = generate()
-const payload = { [generate()]: generate() }
-
-beforeEach(() => {
-  jest.clearAllMocks()
-
-  connection = mock.connection()
-  io = new IO(connection)
-})
-
-it('should reject when request times out', async () => {
-  await expect(io.request(queue, payload, 50)).rejects.toMatchObject({
-    message: expect.stringContaining('timed out'),
-    code: 'ETIMEDOUT'
-  })
-})
-
-it('should reject when request times out with encoding', async () => {
-  await expect(io.request(queue, payload, 'application/json', 50)).rejects.toMatchObject({
-    code: 'ETIMEDOUT'
-  })
-})
-
-it('should time out while send is still pending', async () => {
-  connection.createChannel.mockImplementation(async () => hangingChannel())
-
-  await expect(io.request(queue, payload, 50)).rejects.toMatchObject({
-    code: 'ETIMEDOUT'
-  })
-})
-
-/**
- * @returns {jest.MockedObject<comq.Channel>}
- */
-function hangingChannel () {
-  return /** @type {jest.MockedObject<comq.Channel>} */ ({
-    sharded: false,
-    consume: jest.fn(async () => undefined),
-    send: jest.fn(() => new Promise(() => {})),
-    deliver: jest.fn(async () => undefined),
-    fire: jest.fn(async () => undefined),
-    subscribe: jest.fn(async () => undefined),
-    publish: jest.fn(async () => undefined),
-    diagnose: jest.fn(),
-    seal: jest.fn(async () => undefined),
-    recover: jest.fn(async () => undefined)
-  })
-}