comq 0.15.1 → 0.15.2

package/CHANGELOG.md

@@ -1,3 +1,11 @@
+## [0.15.2](https://github.com/toa-io/comq/compare/v0.15.1...v0.15.2) (2026-08-04)
+
+
+### Bug Fixes
+
+* report and recover undeliverable replies ([b2d2d0f](https://github.com/toa-io/comq/commit/b2d2d0f11d7714c2d32d3d85894fa78e2203f8c6))
+* report and recover undeliverable replies ([#241](https://github.com/toa-io/comq/issues/241)) ([73e5592](https://github.com/toa-io/comq/commit/73e5592762634fbead29d825be68a3da3136729b))
+
 ## [0.15.1](https://github.com/toa-io/comq/compare/v0.15.0...v0.15.1) (2026-08-02)
 
 

package/features/shards.subscriptions.feature

@@ -0,0 +1,25 @@
+Feature: Sharded subscriptions
+
+  A subscription is only complete once every shard has it, otherwise a message
+  published to a randomly picked shard has nowhere to be routed.
+
+  Background:
+    Given an active sharded connection
+
+  Scenario: Events published right after subscribing
+    Given events are exclusively consumed from the `numbers_added` exchange
+    When a stream of 100 events is emitted to the `numbers_added` exchange
+    Then 100 events have been received
+
+  Scenario: Requesting streams repeatedly
+    Given a generator replying `get_numbers` queue:
+      """
+      function * ({ amount }) {
+        for (let i = 0; i < amount; i++) yield i
+      }
+      """
+    When the consumer requests 20 streams with the following request to the `get_numbers` queue:
+      """yaml
+      amount: 5
+      """
+    Then every stream has been received

package/features/steps/rpc.js

@@ -167,6 +167,38 @@ When('the consumer{number} requests a stream with request to the {token} queue',
     await fetch.call(this, queue, null, number)
   })
 
+When('the consumer requests {number} streams with the following request to the {token} queue:',
+  /**
+   * @param {number} amount
+   * @param {string} queue
+   * @param {string} yaml
+   * @this {comq.features.Context}
+   */
+  async function (amount, queue, yaml) {
+    const payload = parse(yaml)
+
+    for (let number = 0; number < amount; number++) {
+      await fetch.call(this, queue, payload, number)
+
+      for await (const value of this.streams[number]) this.streamsValues[number].push(value)
+
+      this.streamsEnded[number] = true
+    }
+  })
+
+Then('every stream has been received',
+  /**
+   * @this {comq.features.Context}
+   */
+  async function () {
+    for (const number of Object.keys(this.streams)) {
+      assert.equal(this.streamsEnded[number], true, `Stream ${number} was not closed`)
+
+      assert.notEqual(this.streamsValues[number].length, 0,
+        `Stream ${number} is empty`)
+    }
+  })
+
 Then('the consumer receives the reply:',
   /**
    * @param {string} yaml

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.15.1",
+  "version": "0.15.2",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",

package/source/.io/ReplyStream.js

@@ -44,13 +44,21 @@ class ReplyStream extends Readable {
     this.#maxBufferSize = global['COMQ_TESTING_MAX_BUFFER_SIZE'] || MAX_BUFFER_SIZE
     this.#reply = reply
 
+    this.confirmation.catch(noop) // it is not awaited until the stream is handed over
+
     this.#emitter.on(this.#correlationId, this.arrange.bind(this))
+
+    // control.ok may never arrive, hence the watchdog is armed before the first message
+    this._heartbeat()
   }
 
   _destroy (error, callback) {
     this._clear()
     this.push(null)
 
+    // a no-op once control.ok has been received
+    this.confirmation.reject(error ?? new Error(UNCONFIRMED))
+
     if (this.#control !== undefined)
       void this.#reply(this.#control, control.end)
 
@@ -145,4 +153,9 @@ class ReplyStream extends Readable {
 
 const MAX_BUFFER_SIZE = 1000
 
+const UNCONFIRMED = 'Reply stream has been destroyed before confirmation'
+
+function noop () {}
+
 exports.ReplyStream = ReplyStream
+exports.UNCONFIRMED = UNCONFIRMED

package/source/connection.js

@@ -46,6 +46,10 @@ class Connection {
     this.#diagnostics.on('error', noop)
   }
 
+  get connected () {
+    return this.#connection !== undefined
+  }
+
   async open () {
     if (this.#opening !== null) return this.#opening
 

package/source/events.js

@@ -4,4 +4,4 @@
 exports.connection = ['open', 'close', 'error']
 
 /** @type {comq.diagnostics.Event[]} */
-exports.channel = ['flow', 'drain', 'recover', 'discard', 'pause', 'resume']
+exports.channel = ['flow', 'drain', 'recover', 'discard', 'pause', 'resume', 'return']

package/source/io.js

@@ -349,7 +349,12 @@ class IO {
       if (isStream) {
         const stream = this.#createReplyStream(request, payload, properties)
 
-        await stream.confirmation
+        try {
+          await stream.confirmation
+        } catch {
+          // the stream has never started, hence the request is re-sent
+          return reply.reject(RETRANSMISSION)
+        }
 
         reply.resolve(stream)
       } else {
@@ -439,6 +444,9 @@ class IO {
     properties.contentType = contentType
     properties.correlationId = request.properties.correlationId
 
+    // an unroutable reply must be returned by the broker rather than dropped
+    properties.mandatory = true
+
     return await this.#replies.fire(replyTo, buffer, properties)
   }
 

package/source/shards/channel.js

@@ -19,8 +19,11 @@ class Channel {
   /** @type {comq.Channel[]} */
   #pool
 
-  /** @type {Set<Promise<comq.Channel>>} */
-  #pending = new Set()
+  /** @type {Map<Promise<comq.Channel>, number>} */
+  #pending = new Map()
+
+  /** @type {Promex[]} */
+  #down = []
 
   /** @type {Map<comq.Channel, Promex>} */
   #bench = new Map()
@@ -48,11 +51,11 @@ class Channel {
   }
 
   async consume (queue, consumer) {
-    return await this.#any((channel) => channel.consume(queue, consumer))
+    return await this.#every((channel) => channel.consume(queue, consumer))
   }
 
   async subscribe (queue, group, consumer) {
-    await this.#any((channel) => channel.subscribe(queue, group, consumer))
+    await this.#every((channel) => channel.subscribe(queue, group, consumer))
   }
 
   async send (queue, buffer, options) {
@@ -86,8 +89,10 @@ class Channel {
    * @return {Promise<void>}
    */
   #create = async (connection, index) => {
+    this.#watch(connection, index)
+
     const pending = connection.createChannel(this.#type, index)
-    const channel = await this.#pend(pending)
+    const channel = await this.#pend(pending, index)
 
     this.#add(channel)
     this.#pipe(channel)
@@ -97,12 +102,31 @@ class Channel {
     channel.diagnose('recover', () => this.#recover(channel))
   }
 
+  /**
+   * Tracks whether a shard is reachable, so that subscribing does not wait for
+   * one that is not. The pool itself is left alone: a channel is only benched
+   * when it actually fails to publish, otherwise a lost connection would
+   * interrupt the streams it is carrying.
+   *
+   * @param {comq.Connection} connection
+   * @param {number} index
+   */
+  #watch (connection, index) {
+    this.#down[index] = new Promex()
+
+    if (connection.connected === false) this.#down[index].resolve()
+
+    connection.diagnose('close', () => this.#down[index].resolve())
+    connection.diagnose('open', () => (this.#down[index] = new Promex()))
+  }
+
   /**
    * @param {Promise<comq.Channel>} pending
+   * @param {number} index
    * @return {Promise<comq.Channel>}
    */
-  async #pend (pending) {
-    this.#pending.add(pending)
+  async #pend (pending, index) {
+    this.#pending.set(pending, index)
 
     const channel = await pending
 
@@ -116,8 +140,43 @@ class Channel {
    */
   #pipe (channel) {
     for (const event of events.channel) {
+      if (event === RETURN) continue // returns are retried before being reported
+
       channel.diagnose(event, (...args) => this.#diagnostics.emit(event, ...args, channel.index))
     }
+
+    channel.diagnose(RETURN, (message) => this.#returned(message, channel))
+  }
+
+  /**
+   * An unroutable message is retried on the shards that have not seen it yet,
+   * since a queue may be declared on some of them only. The return is reported
+   * once every shard has rejected the message.
+   *
+   * @param {comq.amqp.Message} message
+   * @param {comq.Channel} channel
+   */
+  #returned (message, channel) {
+    const report = () => this.#diagnostics.emit(RETURN, message, channel.index)
+    const attempt = (message.properties.headers?.[RETURN_HEADER] ?? 0) + 1
+    const rest = this.#pool.filter((one) => one !== channel)
+
+    // only replies are published to the default exchange, and only they are mandatory
+    const exhausted = message.fields.exchange !== DEFAULT ||
+      attempt >= this.#connections.length ||
+      rest.length === 0
+
+    if (exhausted) return report()
+
+    const properties = {
+      ...message.properties,
+      mandatory: true,
+      headers: { ...message.properties.headers, [RETURN_HEADER]: attempt }
+    }
+
+    const next = rest[Math.floor(Math.random() * rest.length)]
+
+    next.fire(message.fields.routingKey, message.content, properties).catch(report)
   }
 
   /**
@@ -171,12 +230,49 @@ class Channel {
   }
 
   /**
+   * Resolves once every available shard has settled, requiring at least one to
+   * succeed. Benched shards are applied when they come back, which must not
+   * hold up the caller.
+   *
    * @param {(channel: comq.Channel) => void} fn
+   * @return {Promise<any>}
    */
-  async #any (fn) {
-    const promises = this.#apply(fn)
+  async #every (fn) {
+    const promises = []
+
+    for (const channel of this.#channels) {
+      promises.push(this.#unless(fn(channel), channel.index))
+    }
+
+    for (const [pending, index] of this.#pending) {
+      promises.push(this.#unless(pending.then(fn), index))
+    }
+
+    for (const recover of this.#bench.values()) recover.then(fn).catch(noop)
+
+    const results = await Promise.allSettled(promises)
+    const fulfilled = results.find((result) => result.status === 'fulfilled')
+
+    if (fulfilled === undefined) {
+      const reasons = results.map((result) => result.reason)
 
-    return await Promise.any(promises)
+      throw new AggregateError(reasons, 'No shard is available')
+    }
+
+    return fulfilled.value
+  }
+
+  /**
+   * Gives up on a shard as soon as its connection is lost.
+   *
+   * @param {Promise<any>} promise
+   * @param {number} index
+   * @return {Promise<any>}
+   */
+  async #unless (promise, index) {
+    const down = this.#down[index]
+
+    return down === undefined ? await promise : await Promise.race([promise, down])
   }
 
   /**
@@ -197,7 +293,7 @@ class Channel {
     const promises = []
 
     for (const channel of this.#channels) promises.push(fn(channel))
-    for (const pending of this.#pending) promises.push(pending.then(fn))
+    for (const pending of this.#pending.keys()) promises.push(pending.then(fn))
     for (const recover of this.#bench.values()) promises.push(recover.then(fn))
 
     return promises
@@ -234,4 +330,10 @@ async function create (connections, type) {
   return channel
 }
 
+const DEFAULT = ''
+const RETURN = 'return'
+const RETURN_HEADER = 'x-return'
+
+function noop () {}
+
 exports.create = create

package/test/ReplyStream.test.js

@@ -2,7 +2,8 @@
 
 const { EventEmitter } = require('node:events')
 const { once } = require('node:events')
-const { ReplyStream } = require('../source/.io/ReplyStream')
+const { ReplyStream, UNCONFIRMED } = require('../source/.io/ReplyStream')
+const { control } = require('../source/.io/const')
 
 /** @type {jest.Mock} */
 let reply
@@ -43,3 +44,40 @@ it('should destroy without calling reply when buffer overflows before control.ok
   expect(stream.destroyed).toBe(true)
   expect(reply).not.toHaveBeenCalled()
 })
+
+it('should reject confirmation when buffer overflows before control.ok', async () => {
+  for (let index = 1; index <= 5; index++) {
+    stream.arrange(index, { headers: { index } })
+  }
+
+  await expect(stream.confirmation).rejects.toThrow(UNCONFIRMED)
+})
+
+it('should reject confirmation when destroyed before control.ok', async () => {
+  stream.destroy()
+
+  await expect(stream.confirmation).rejects.toThrow(UNCONFIRMED)
+})
+
+it('should reject confirmation when nothing arrives within the idle interval', async () => {
+  global.COMQ_TESTING_IDLE_INTERVAL = 10
+
+  const request = {
+    emitter: new EventEmitter(),
+    properties: { correlationId: 'idle-correlation' }
+  }
+
+  const idle = new ReplyStream(request, reply)
+
+  await expect(idle.confirmation).rejects.toThrow(UNCONFIRMED)
+})
+
+it('should keep confirmation resolved after control.ok', async () => {
+  stream.arrange(control.ok, { headers: { index: 0 }, type: 'control' })
+
+  await expect(stream.confirmation).resolves.toBeUndefined()
+
+  stream.destroy()
+
+  await expect(stream.confirmation).resolves.toBeUndefined()
+})

package/test/connection.mock.js

@@ -19,6 +19,7 @@ const channel = /** @type {jest.MockedFunction<(sharded?: boolean, index?: numbe
  * @returns {jest.MockedObject<comq.Connection>}
  */
 const connection = (sharded = false) => (/** @type {jest.MockedObject<comq.Connection>} */ {
+  connected: true,
   createChannel: jest.fn(async (type, index) => channel(sharded, index)),
   open: jest.fn(async () => undefined),
   close: jest.fn(async () => undefined),

package/test/io.reply.test.js

@@ -185,7 +185,8 @@ describe('encoding', () => {
 
     const props = {
       correlationId: properties.correlationId,
-      contentType: properties.contentType
+      contentType: properties.contentType,
+      mandatory: true
     }
 
     expect(replies.fire).toHaveBeenCalledWith(properties.replyTo, reply, props)

package/test/io.request.test.js

@@ -237,6 +237,30 @@ describe('reply', () => {
 
       expect(output).toStrictEqual(value)
     })
+
+  it('should re-send the Request if a reply stream is never confirmed', async () => {
+    global.COMQ_TESTING_MAX_BUFFER_SIZE = 3
+
+    const correlationId = requests.send.mock.calls[0][2].correlationId
+    const callback = replies.consume.mock.calls[0][1]
+
+    try {
+      // out of order chunks overflow the buffer, so the stream dies before control.ok
+      for (let index = 1; index <= 5; index++) {
+        const properties = { correlationId, headers: { index } }
+        const message = /** @type {comq.amqp.Message} */ { content: randomBytes(8), properties }
+
+        await callback(message)
+      }
+
+      await immediate()
+      await immediate()
+
+      expect(requests.send).toHaveBeenCalledTimes(2)
+    } finally {
+      delete global.COMQ_TESTING_MAX_BUFFER_SIZE
+    }
+  })
 })
 
 const reply = async (content = randomBytes(8), contentType = undefined) => {

package/test/shards/channel.test.js

@@ -87,28 +87,75 @@ describe.each(['consume', 'subscribe'])('%s', (method) => {
     for (const chan of channels) expect(chan[method]).toHaveBeenCalledWith(...args)
   })
 
-  it(`should should ${method} using available and pending channels`, async () => {
+  it(`should not resolve ${method} until pending channels are created`, async () => {
     const promise = promex()
 
     connections[0].createChannel.mockImplementationOnce(() => promise)
 
     channel = await create(connections, type)
 
-    await channel[method](...args)
+    let resolved = false
+
+    const subscription = channel[method](...args).then(() => { resolved = true })
+
+    await immediate()
 
     /** @type {jest.MockedObject<comq.Channel>} */
     const chan1 = await connections[1].createChannel.mock.results[0].value
 
     expect(chan1[method]).toHaveBeenCalled()
+    expect(resolved).toStrictEqual(false)
 
     /** @type {jest.MockedObject<comq.Channel>} */
     const chan0 = mock.channel()
 
     promise.resolve(chan0)
 
-    await immediate()
+    await subscription
 
     expect(chan0[method]).toHaveBeenCalled()
+    expect(resolved).toStrictEqual(true)
+  })
+
+  it(`should resolve ${method} without a disconnected shard`, async () => {
+    const promise = promex()
+
+    connections[0].createChannel.mockImplementationOnce(() => promise)
+    connections[0].connected = false
+
+    try {
+      channel = await create(connections, type)
+
+      // must not wait for the channel that is never going to be created
+      await channel[method](...args)
+
+      /** @type {jest.MockedObject<comq.Channel>} */
+      const chan1 = await connections[1].createChannel.mock.results[0].value
+
+      expect(chan1[method]).toHaveBeenCalled()
+    } finally {
+      connections[0].connected = true
+      promise.resolve(mock.channel())
+    }
+  })
+
+  it(`should resolve ${method} when a shard disconnects`, async () => {
+    const promise = promex()
+
+    connections[0].createChannel.mockImplementationOnce(() => promise)
+
+    channel = await create(connections, type)
+
+    const subscription = channel[method](...args)
+    const calls = connections[0].diagnose.mock.calls.filter((call) => call[0] === 'close')
+
+    expect(calls.length).toBeGreaterThan(0)
+
+    for (const call of calls) call[1]()
+
+    await subscription
+
+    promise.resolve(mock.channel())
   })
 
   it(`should ${method} using a channel removed from the pool once it is recovered`, async () => {
@@ -339,6 +386,43 @@ describe('diagnose', () => {
     expect(listener).toHaveBeenCalledWith(expect.any(Number))
   })
 
+  it('should re-fire a returned message on another shard', async () => {
+    const listener = /** @type {jest.MockedFunction} */ jest.fn()
+    const index = random(channels.length)
+    const chan = channels[index]
+    const message = returned()
+
+    channel.diagnose('return', listener)
+
+    emitReturn(chan, message)
+
+    const rest = channels.filter((one) => one !== chan)
+
+    const properties = expect.objectContaining(
+      { mandatory: true, headers: { 'x-return': 1 } })
+
+    for (const one of rest) {
+      expect(one.fire).toHaveBeenCalledWith(message.fields.routingKey, message.content, properties)
+    }
+
+    expect(listener).not.toHaveBeenCalled()
+  })
+
+  it('should emit `return` event once every shard has returned the message', async () => {
+    const listener = /** @type {jest.MockedFunction} */ jest.fn()
+    const index = random(channels.length)
+    const chan = channels[index]
+    const message = returned({ 'x-return': channels.length - 1 })
+
+    channel.diagnose('return', listener)
+
+    emitReturn(chan, message)
+
+    for (const one of channels) expect(one.fire).not.toHaveBeenCalled()
+
+    expect(listener).toHaveBeenCalledWith(message, index)
+  })
+
   it('should emit `pause` and `unpause` events', async () => {
     const queue = generate()
     const buffer = randomBytes(8)
@@ -428,6 +512,30 @@ it('should remove channel with back pressure from the pool', async () => {
   expect(chan.send).toHaveBeenCalled()
 })
 
+/**
+ * @param {Record<string, any>} [headers]
+ * @return {comq.amqp.Message}
+ */
+function returned (headers = {}) {
+  return /** @type {comq.amqp.Message} */ {
+    content: randomBytes(8),
+    fields: { exchange: '', routingKey: generate() },
+    properties: { correlationId: generate(), headers }
+  }
+}
+
+/**
+ * @param {jest.MockedObject<comq.Channel>} channel
+ * @param {comq.amqp.Message} message
+ */
+function emitReturn (channel, message) {
+  const calls = channel.diagnose.mock.calls.filter((call) => call[0] === 'return')
+
+  expect(calls.length).toBeGreaterThan(0)
+
+  for (const call of calls) call[1](message)
+}
+
 /**
  * @return {Promise<jest.MockedObject<comq.Channel>[]>}
  */

package/types/connection.d.ts

@@ -6,6 +6,8 @@ import * as _topology from './topology'
 declare namespace comq {
 
   interface Connection {
+    readonly connected?: boolean
+
     open(): Promise<void>
 
     close(): Promise<void>