comq 0.15.2 → 0.15.3

package/CHANGELOG.md

@@ -1,3 +1,11 @@
+## [0.15.3](https://github.com/toa-io/comq/compare/v0.15.2...v0.15.3) (2026-08-04)
+
+
+### Bug Fixes
+
+* re-send requests left unanswered by a lost shard ([c534a01](https://github.com/toa-io/comq/commit/c534a0136e5f0189cdb6976cd9cffb9a2ca5a148))
+* re-send requests left unanswered by a lost shard ([#242](https://github.com/toa-io/comq/issues/242)) ([c821da8](https://github.com/toa-io/comq/commit/c821da895049bd78a907c6705477ebe74c160e1b))
+
 ## [0.15.2](https://github.com/toa-io/comq/compare/v0.15.1...v0.15.2) (2026-08-04)
 
 

package/features/shards.feature

@@ -40,6 +40,22 @@ Feature: Sharded Connection
     And all replies have been received
     Then the broker is up
 
+  Scenario: Shard crashes while requests await their replies
+    Given an active sharded connection
+    And a producer replying `slow` queue in 3000ms
+    When the consumer sends 10 requests to the `slow` queue
+    And one of the brokers has crashed
+    Then all replies have been received
+    Then the broker is up
+
+  Scenario: Shard stops responding while requests are sent
+    Given watchdog interval is set to 3000ms with 1s AMQP heartbeat
+    And an active sharded connection
+    And a producer replying `stalled` queue
+    When the consumer sends 10 requests to the `stalled` queue as a broker freezes
+    Then all replies have been received
+    Then the broker is up
+
   Scenario: Shard crashes while publishing events
     Given an active sharded connection
     And events are exclusively consumed from the `flood` exchange

package/features/steps/brokers.js

@@ -103,6 +103,9 @@ async function healthy (n = 0) {
 
 const actions = {
   up: async (n = 0) => {
+    // a paused broker is neither stopped nor usable, and the hooks bring every
+    // broker up between scenarios, so unpausing belongs here
+    await docker('docker', ['unpause', `comq-rmq-${n}`]).catch(() => undefined)
     await docker('docker', ['start', `comq-rmq-${n}`])
     await healthy(n)
   },
@@ -111,6 +114,11 @@ const actions = {
   },
   crashed: async (n = 0) => {
     await docker('docker', ['kill', `comq-rmq-${n}`])
+  },
+  // a frozen broker keeps its connections open while answering nothing, which is
+  // what a publisher cannot tell from a working one until the watchdog fires
+  frozen: async (n = 0) => {
+    await docker('docker', ['pause', `comq-rmq-${n}`])
   }
 }
 

package/features/steps/context.js

@@ -89,7 +89,12 @@ class Context extends World {
   }
 
   #url (i, user, password) {
-    return PROTOCOL + user + ':' + password + '@' + getAddress(i)
+    const url = PROTOCOL + user + ':' + password + '@' + getAddress(i)
+    const heartbeat = global.COMQ_TESTING_AMQP_HEARTBEAT
+
+    // the watchdog measures silence, so it may only be shortened along with the
+    // interval at which a healthy broker is expected to say something
+    return heartbeat === undefined ? url : url + '?heartbeat=' + heartbeat
   }
 }
 

package/features/steps/hooks.js

@@ -12,6 +12,9 @@ After(
    * @this {comq.features.Context}
    */
   async function () {
+    delete global.COMQ_TESTING_WATCHDOG_INTERVAL
+    delete global.COMQ_TESTING_AMQP_HEARTBEAT
+
     await Promise.all(
       Array.from({ length: BROKERS_AMOUNT }, (_, n) => actions.up(n))
     )

package/features/steps/parameters.js

@@ -28,7 +28,7 @@ defineParameterType({
 
 defineParameterType({
   name: 'status',
-  regexp: /(up|down|crashed)/,
+  regexp: /(up|down|crashed|frozen)/,
   transformer: (value) => value
 })
 

package/features/steps/rpc.js

@@ -4,9 +4,10 @@ const stream = require('node:stream')
 const assert = require('node:assert')
 const { randomBytes } = require('node:crypto')
 const { parse } = require('./yaml')
-const { match, timeout, quantity } = require('@toa.io/generic')
+const { match, timeout, quantity, random } = require('@toa.io/generic')
 const { Given, When, Then } = require('@cucumber/cucumber')
 const { Readable } = require('node:stream')
+const { actions, BROKERS_AMOUNT } = require('./brokers')
 
 Given('function replying {token} queue:',
   /**
@@ -85,6 +86,12 @@ Given('idle timeout is set to {number}ms',
     global.COMQ_TESTING_IDLE_INTERVAL = value
   })
 
+Given('watchdog interval is set to {number}ms with {number}s AMQP heartbeat',
+  function (watchdog, heartbeat) {
+    global.COMQ_TESTING_WATCHDOG_INTERVAL = watchdog
+    global.COMQ_TESTING_AMQP_HEARTBEAT = heartbeat
+  })
+
 Given('a producer replying {token} queue',
   /**
    * @param {string} queue
@@ -100,6 +107,22 @@ Given('a producer replying {token} queue',
     await this.io.reply(queue, producer)
   })
 
+Given('a producer replying {token} queue in {number}ms',
+  /**
+   * @param {string} queue
+   * @param {number} delay
+   * @this {comq.features.Context}
+   */
+  async function (queue, delay) {
+    const producer = async (request) => {
+      await timeout(delay)
+
+      return request
+    }
+
+    await this.io.reply(queue, producer)
+  })
+
 Given('function replying {token} queue is expected:',
   /**
    * @param {string} queue
@@ -136,6 +159,32 @@ When('the consumer sends a request to the {token} queue',
     await send.call(this, queue, payload)
   })
 
+When('the consumer sends {number} requests to the {token} queue',
+  /**
+   * @param {number} amount
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  async function (amount, queue) {
+    sendMany.call(this, amount, queue)
+  })
+
+// the requests must be published while the frozen shard still looks reachable,
+// which is a window too short to spend on a step boundary
+When('the consumer sends {number} requests to the {token} queue as a broker freezes',
+  /**
+   * @param {number} amount
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  async function (amount, queue) {
+    this.shard = random(BROKERS_AMOUNT)
+
+    await actions.frozen(this.shard)
+
+    sendMany.call(this, amount, queue)
+  })
+
 When('the consumer requests a stream with the following request to the {token} queue:',
   /**
    * @param {string} queue
@@ -425,6 +474,17 @@ Then('all replies have been received',
     await Promise.all(this.requestsSent)
   })
 
+/**
+ * @param {number} amount
+ * @param {string} queue
+ * @this {comq.features.Context}
+ */
+function sendMany (amount, queue) {
+  for (let i = 0; i < amount; i++) {
+    this.requestsSent.push(this.io.request(queue, randomBytes(8)))
+  }
+}
+
 async function send (queue, payload) {
   if (this.expected) await this.expected
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.15.2",
+  "version": "0.15.3",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",

package/source/connection.js

@@ -151,7 +151,7 @@ class Connection {
    * @param {comq.amqp.Connection} connection
    * @param {number} [timeoutMs]
    */
-  #armWatchdog (connection, timeoutMs = WATCHDOG_MS) {
+  #armWatchdog (connection, timeoutMs = global.COMQ_TESTING_WATCHDOG_INTERVAL ?? WATCHDOG_MS) {
     const socket = connection.connection?.stream
 
     if (socket === undefined) return

package/source/events.js

@@ -4,4 +4,4 @@
 exports.connection = ['open', 'close', 'error']
 
 /** @type {comq.diagnostics.Event[]} */
-exports.channel = ['flow', 'drain', 'recover', 'discard', 'pause', 'resume', 'return']
+exports.channel = ['flow', 'drain', 'recover', 'discard', 'pause', 'resume', 'return', 'lost']

package/source/io.js

@@ -35,8 +35,8 @@ class IO {
   /** @type {comq.ReplyEmitter | null} */
   #control = null
 
-  /** @type {Set<Promex>} */
-  #pendingReplies = new Set()
+  /** @type {Map<Promex, comq.Request>} */
+  #pendingReplies = new Map()
 
   /** @type {Set<comq.Destroyable>} */
   #replyStreams = new Set()
@@ -230,9 +230,14 @@ class IO {
   }
 
   #setupRetransmission () {
-    const event = (this.#requests.sharded === true) ? 'remove' : 'recover'
-
-    this.#requests.diagnose(event, this.#retransmit)
+    if (this.#requests.sharded === true) {
+      // a shard leaves the pool when it rejects a publish, and is lost when its
+      // connection drops, which leaves an already sent request unanswered
+      this.#requests.diagnose('remove', this.#retransmit)
+      this.#requests.diagnose('lost', this.#retransmit)
+    } else {
+      this.#requests.diagnose('recover', this.#retransmit)
+    }
   }
 
   /**
@@ -316,7 +321,7 @@ class IO {
    * @return {Promex<any>}
    */
   #createReply (request) {
-    const reply = this.#createPendingReply()
+    const reply = this.#createPendingReply(request)
 
     request.emitter.once(request.properties.correlationId, this.#getReplyResolver(request, reply))
 
@@ -324,12 +329,13 @@ class IO {
   }
 
   /**
+   * @param {comq.Request} request
    * @return {Promex}
    */
-  #createPendingReply () {
+  #createPendingReply (request) {
     const reply = new Promex()
 
-    this.#pendingReplies.add(reply)
+    this.#pendingReplies.set(reply, request)
 
     reply
       .catch(noop)
@@ -455,10 +461,14 @@ class IO {
   }
 
   #retransmit = () => {
-    for (const emitter of this.#emitters.values()) emitter.removeAllListeners()
+    for (const [reply, request] of this.#pendingReplies) {
+      // detaching this attempt alone leaves the listeners of the reply streams
+      // that are still flowing over the other shards in place
+      request.emitter.removeAllListeners(request.properties.correlationId)
 
-    // trigger failsafe attribute
-    for (const reply of this.#pendingReplies) reply.reject(RETRANSMISSION)
+      // trigger failsafe attribute
+      reply.reject(RETRANSMISSION)
+    }
   }
 
   /**

package/source/shards/channel.js

@@ -25,6 +25,9 @@ class Channel {
   /** @type {Promex[]} */
   #down = []
 
+  /** @type {boolean[]} */
+  #alive = []
+
   /** @type {Map<comq.Channel, Promex>} */
   #bench = new Map()
 
@@ -106,18 +109,28 @@ class Channel {
    * Tracks whether a shard is reachable, so that subscribing does not wait for
    * one that is not. The pool itself is left alone: a channel is only benched
    * when it actually fails to publish, otherwise a lost connection would
-   * interrupt the streams it is carrying.
+   * interrupt the streams it is carrying. Losing a shard is reported instead,
+   * since a request awaiting its reply on it will never be answered.
    *
    * @param {comq.Connection} connection
    * @param {number} index
    */
   #watch (connection, index) {
     this.#down[index] = new Promex()
+    this.#alive[index] = connection.connected !== false
 
     if (connection.connected === false) this.#down[index].resolve()
 
-    connection.diagnose('close', () => this.#down[index].resolve())
-    connection.diagnose('open', () => (this.#down[index] = new Promex()))
+    connection.diagnose('close', () => {
+      this.#alive[index] = false
+      this.#down[index].resolve()
+      this.#diagnostics.emit(LOST, index)
+    })
+
+    connection.diagnose('open', () => {
+      this.#alive[index] = true
+      this.#down[index] = new Promex()
+    })
   }
 
   /**
@@ -141,6 +154,7 @@ class Channel {
   #pipe (channel) {
     for (const event of events.channel) {
       if (event === RETURN) continue // returns are retried before being reported
+      if (event === LOST) continue // a shard is lost with its connection, not its channel
 
       channel.diagnose(event, (...args) => this.#diagnostics.emit(event, ...args, channel.index))
     }
@@ -299,13 +313,28 @@ class Channel {
     return promises
   }
 
+  /**
+   * The channels of the shards that are known to be connected, falling back to the
+   * whole pool while none is, so that a publish is attempted rather than dropped.
+   *
+   * @return {comq.Channel[]}
+   */
+  #reachable () {
+    const reachable = this.#pool.filter((channel) => this.#alive[channel.index] !== false)
+
+    return reachable.length === 0 ? this.#pool : reachable
+  }
+
   /**
    * @param {(channel: comq.Channel) => void} fn
    */
   async #one (fn) {
     if (this.#pool.length === 0) await this.#recovery
 
-    const channel = this.#pool[Math.floor(Math.random() * this.#pool.length)]
+    // publishing to a lost shard is silently accepted by the destroyed socket, so
+    // it must be avoided rather than retried: nothing would report the loss twice
+    const pool = this.#reachable()
+    const channel = pool[Math.floor(Math.random() * pool.length)]
 
     try {
       return await fn(channel)
@@ -333,6 +362,7 @@ async function create (connections, type) {
 const DEFAULT = ''
 const RETURN = 'return'
 const RETURN_HEADER = 'x-return'
+const LOST = 'lost'
 
 function noop () {}
 

package/test/io.request.test.js

@@ -207,8 +207,125 @@ describe('send', () => {
 
     expect(requests.send).toHaveBeenCalledTimes(2)
   })
+
+  it('should resend unanswered Requests when a shard is lost', async () => {
+    jest.clearAllMocks()
+
+    connection = mock.connection(true)
+    io = new IO(connection)
+
+    promise = io.request(queue, payload)
+
+    // allows initializers to run
+    await immediate()
+
+    requests = await findChannel('request')
+
+    expect(requests.diagnose).toHaveBeenCalledWith('lost', expect.any(Function))
+
+    await lose()
+
+    expect(requests.send).toHaveBeenCalledTimes(2)
+  })
+
+  it('should not interrupt a confirmed reply stream when a shard is lost', async () => {
+    jest.clearAllMocks()
+
+    connection = mock.connection(true)
+    io = new IO(connection)
+
+    promise = io.request(queue, payload)
+
+    await immediate()
+
+    requests = await findChannel('request')
+    replies = await findChannel('reply')
+
+    const correlationId = requests.send.mock.calls[0][2].correlationId
+    const callback = replies.consume.mock.calls[0][1]
+
+    await callback(message(correlationId, 0, 'ok', 'control'))
+
+    const output = await promise
+
+    expect(output).toBeInstanceOf(stream.Readable)
+
+    await lose()
+
+    // the stream is carried by whichever shard replied, so it keeps flowing
+    const chunk = randomBytes(8)
+
+    await callback(message(correlationId, 1, chunk))
+
+    expect(output.read()).toStrictEqual(chunk)
+
+    output.destroy()
+  })
+
+  it('should not resend an answered Request when a shard is lost', async () => {
+    jest.clearAllMocks()
+
+    connection = mock.connection(true)
+    io = new IO(connection)
+
+    promise = io.request(queue, payload)
+
+    await immediate()
+
+    requests = await findChannel('request')
+    replies = await findChannel('reply')
+
+    const correlationId = requests.send.mock.calls[0][2].correlationId
+    const callback = replies.consume.mock.calls[0][1]
+
+    const answer = /** @type {comq.amqp.Message} */
+      { content: randomBytes(8), properties: { correlationId } }
+
+    await callback(answer)
+    await promise
+
+    await lose()
+
+    expect(requests.send).toHaveBeenCalledTimes(1)
+  })
 })
 
+/**
+ * Fires the listeners the IO has attached to the shard loss diagnostic.
+ *
+ * @return {Promise<void>}
+ */
+async function lose () {
+  const calls = requests.diagnose.mock.calls.filter((call) => call[0] === 'lost')
+
+  for (const [, listener] of calls) listener(0)
+
+  await immediate()
+  await immediate()
+}
+
+/**
+ * @param {string} correlationId
+ * @param {number} index
+ * @param {any} content
+ * @param {string} [type]
+ * @return {comq.amqp.Message}
+ */
+function message (correlationId, index, content, type) {
+  const raw = Buffer.isBuffer(content)
+
+  return /** @type {comq.amqp.Message} */ {
+    content: raw ? content : encode(content, 'application/json'),
+    properties: {
+      correlationId,
+      type,
+      replyTo: queue,
+      contentType: raw ? 'application/octet-stream' : 'application/json',
+      headers: { index }
+    }
+  }
+}
+
 describe('reply', () => {
   it.each([undefined, 'application/octet-stream'])('should return raw content if encoding is %s',
     async (contentType) => {

package/test/shards/channel.test.js

@@ -29,6 +29,58 @@ it('should expose `sharded`', async () => {
   expect(channel.sharded).toStrictEqual(true)
 })
 
+it('should report a lost shard', async () => {
+  channel = await create(connections, type)
+
+  const lost = /** @type {jest.MockedFunction} */ jest.fn()
+
+  channel.diagnose('lost', lost)
+
+  const calls = connections[1].diagnose.mock.calls.filter((call) => call[0] === 'close')
+
+  expect(calls.length).toBeGreaterThan(0)
+
+  for (const call of calls) call[1]()
+
+  expect(lost).toHaveBeenCalledWith(1)
+})
+
+it('should not publish to a lost shard', async () => {
+  channel = await create(connections, type)
+
+  const channels = await getCreatedChannels()
+  const calls = connections[1].diagnose.mock.calls.filter((call) => call[0] === 'close')
+
+  for (const call of calls) call[1]()
+
+  const buffer = randomBytes(8)
+  const label = generate()
+
+  for (let i = 0; i < 20; i++) await channel.send(label, buffer)
+
+  expect(channels[0].send).toHaveBeenCalledTimes(20)
+  expect(channels[1].send).not.toHaveBeenCalled()
+})
+
+// a lost shard is not benched, so it is used again as soon as it reconnects
+it('should publish to a recovered shard', async () => {
+  channel = await create(connections, type)
+
+  const channels = await getCreatedChannels()
+  const close = connections[1].diagnose.mock.calls.filter((call) => call[0] === 'close')
+  const open = connections[1].diagnose.mock.calls.filter((call) => call[0] === 'open')
+
+  for (const call of close) call[1]()
+  for (const call of open) call[1]()
+
+  const buffer = randomBytes(8)
+  const label = generate()
+
+  for (let i = 0; i < 20; i++) await channel.send(label, buffer)
+
+  expect(channels[1].send).toHaveBeenCalled()
+})
+
 it('should resolve when one of the connections has created a channel', async () => {
   /** @type {toa.generic.Promex[]} */
   const promises = []