comq 0.15.1 → 0.15.3

package/CHANGELOG.md

@@ -1,3 +1,19 @@
+## [0.15.3](https://github.com/toa-io/comq/compare/v0.15.2...v0.15.3) (2026-08-04)
+
+
+### Bug Fixes
+
+* re-send requests left unanswered by a lost shard ([c534a01](https://github.com/toa-io/comq/commit/c534a0136e5f0189cdb6976cd9cffb9a2ca5a148))
+* re-send requests left unanswered by a lost shard ([#242](https://github.com/toa-io/comq/issues/242)) ([c821da8](https://github.com/toa-io/comq/commit/c821da895049bd78a907c6705477ebe74c160e1b))
+
+## [0.15.2](https://github.com/toa-io/comq/compare/v0.15.1...v0.15.2) (2026-08-04)
+
+
+### Bug Fixes
+
+* report and recover undeliverable replies ([b2d2d0f](https://github.com/toa-io/comq/commit/b2d2d0f11d7714c2d32d3d85894fa78e2203f8c6))
+* report and recover undeliverable replies ([#241](https://github.com/toa-io/comq/issues/241)) ([73e5592](https://github.com/toa-io/comq/commit/73e5592762634fbead29d825be68a3da3136729b))
+
 ## [0.15.1](https://github.com/toa-io/comq/compare/v0.15.0...v0.15.1) (2026-08-02)
 
 

package/features/shards.feature

@@ -40,6 +40,22 @@ Feature: Sharded Connection
     And all replies have been received
     Then the broker is up
 
+  Scenario: Shard crashes while requests await their replies
+    Given an active sharded connection
+    And a producer replying `slow` queue in 3000ms
+    When the consumer sends 10 requests to the `slow` queue
+    And one of the brokers has crashed
+    Then all replies have been received
+    Then the broker is up
+
+  Scenario: Shard stops responding while requests are sent
+    Given watchdog interval is set to 3000ms with 1s AMQP heartbeat
+    And an active sharded connection
+    And a producer replying `stalled` queue
+    When the consumer sends 10 requests to the `stalled` queue as a broker freezes
+    Then all replies have been received
+    Then the broker is up
+
   Scenario: Shard crashes while publishing events
     Given an active sharded connection
     And events are exclusively consumed from the `flood` exchange

package/features/shards.subscriptions.feature

@@ -0,0 +1,25 @@
+Feature: Sharded subscriptions
+
+  A subscription is only complete once every shard has it, otherwise a message
+  published to a randomly picked shard has nowhere to be routed.
+
+  Background:
+    Given an active sharded connection
+
+  Scenario: Events published right after subscribing
+    Given events are exclusively consumed from the `numbers_added` exchange
+    When a stream of 100 events is emitted to the `numbers_added` exchange
+    Then 100 events have been received
+
+  Scenario: Requesting streams repeatedly
+    Given a generator replying `get_numbers` queue:
+      """
+      function * ({ amount }) {
+        for (let i = 0; i < amount; i++) yield i
+      }
+      """
+    When the consumer requests 20 streams with the following request to the `get_numbers` queue:
+      """yaml
+      amount: 5
+      """
+    Then every stream has been received

package/features/steps/brokers.js

@@ -103,6 +103,9 @@ async function healthy (n = 0) {
 
 const actions = {
   up: async (n = 0) => {
+    // a paused broker is neither stopped nor usable, and the hooks bring every
+    // broker up between scenarios, so unpausing belongs here
+    await docker('docker', ['unpause', `comq-rmq-${n}`]).catch(() => undefined)
     await docker('docker', ['start', `comq-rmq-${n}`])
     await healthy(n)
   },
@@ -111,6 +114,11 @@ const actions = {
   },
   crashed: async (n = 0) => {
     await docker('docker', ['kill', `comq-rmq-${n}`])
+  },
+  // a frozen broker keeps its connections open while answering nothing, which is
+  // what a publisher cannot tell from a working one until the watchdog fires
+  frozen: async (n = 0) => {
+    await docker('docker', ['pause', `comq-rmq-${n}`])
   }
 }
 

package/features/steps/context.js

@@ -89,7 +89,12 @@ class Context extends World {
   }
 
   #url (i, user, password) {
-    return PROTOCOL + user + ':' + password + '@' + getAddress(i)
+    const url = PROTOCOL + user + ':' + password + '@' + getAddress(i)
+    const heartbeat = global.COMQ_TESTING_AMQP_HEARTBEAT
+
+    // the watchdog measures silence, so it may only be shortened along with the
+    // interval at which a healthy broker is expected to say something
+    return heartbeat === undefined ? url : url + '?heartbeat=' + heartbeat
   }
 }
 

package/features/steps/hooks.js

@@ -12,6 +12,9 @@ After(
    * @this {comq.features.Context}
    */
   async function () {
+    delete global.COMQ_TESTING_WATCHDOG_INTERVAL
+    delete global.COMQ_TESTING_AMQP_HEARTBEAT
+
     await Promise.all(
       Array.from({ length: BROKERS_AMOUNT }, (_, n) => actions.up(n))
     )

package/features/steps/parameters.js

@@ -28,7 +28,7 @@ defineParameterType({
 
 defineParameterType({
   name: 'status',
-  regexp: /(up|down|crashed)/,
+  regexp: /(up|down|crashed|frozen)/,
   transformer: (value) => value
 })
 

package/features/steps/rpc.js

@@ -4,9 +4,10 @@ const stream = require('node:stream')
 const assert = require('node:assert')
 const { randomBytes } = require('node:crypto')
 const { parse } = require('./yaml')
-const { match, timeout, quantity } = require('@toa.io/generic')
+const { match, timeout, quantity, random } = require('@toa.io/generic')
 const { Given, When, Then } = require('@cucumber/cucumber')
 const { Readable } = require('node:stream')
+const { actions, BROKERS_AMOUNT } = require('./brokers')
 
 Given('function replying {token} queue:',
   /**
@@ -85,6 +86,12 @@ Given('idle timeout is set to {number}ms',
     global.COMQ_TESTING_IDLE_INTERVAL = value
   })
 
+Given('watchdog interval is set to {number}ms with {number}s AMQP heartbeat',
+  function (watchdog, heartbeat) {
+    global.COMQ_TESTING_WATCHDOG_INTERVAL = watchdog
+    global.COMQ_TESTING_AMQP_HEARTBEAT = heartbeat
+  })
+
 Given('a producer replying {token} queue',
   /**
    * @param {string} queue
@@ -100,6 +107,22 @@ Given('a producer replying {token} queue',
     await this.io.reply(queue, producer)
   })
 
+Given('a producer replying {token} queue in {number}ms',
+  /**
+   * @param {string} queue
+   * @param {number} delay
+   * @this {comq.features.Context}
+   */
+  async function (queue, delay) {
+    const producer = async (request) => {
+      await timeout(delay)
+
+      return request
+    }
+
+    await this.io.reply(queue, producer)
+  })
+
 Given('function replying {token} queue is expected:',
   /**
    * @param {string} queue
@@ -136,6 +159,32 @@ When('the consumer sends a request to the {token} queue',
     await send.call(this, queue, payload)
   })
 
+When('the consumer sends {number} requests to the {token} queue',
+  /**
+   * @param {number} amount
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  async function (amount, queue) {
+    sendMany.call(this, amount, queue)
+  })
+
+// the requests must be published while the frozen shard still looks reachable,
+// which is a window too short to spend on a step boundary
+When('the consumer sends {number} requests to the {token} queue as a broker freezes',
+  /**
+   * @param {number} amount
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  async function (amount, queue) {
+    this.shard = random(BROKERS_AMOUNT)
+
+    await actions.frozen(this.shard)
+
+    sendMany.call(this, amount, queue)
+  })
+
 When('the consumer requests a stream with the following request to the {token} queue:',
   /**
    * @param {string} queue
@@ -167,6 +216,38 @@ When('the consumer{number} requests a stream with request to the {token} queue',
     await fetch.call(this, queue, null, number)
   })
 
+When('the consumer requests {number} streams with the following request to the {token} queue:',
+  /**
+   * @param {number} amount
+   * @param {string} queue
+   * @param {string} yaml
+   * @this {comq.features.Context}
+   */
+  async function (amount, queue, yaml) {
+    const payload = parse(yaml)
+
+    for (let number = 0; number < amount; number++) {
+      await fetch.call(this, queue, payload, number)
+
+      for await (const value of this.streams[number]) this.streamsValues[number].push(value)
+
+      this.streamsEnded[number] = true
+    }
+  })
+
+Then('every stream has been received',
+  /**
+   * @this {comq.features.Context}
+   */
+  async function () {
+    for (const number of Object.keys(this.streams)) {
+      assert.equal(this.streamsEnded[number], true, `Stream ${number} was not closed`)
+
+      assert.notEqual(this.streamsValues[number].length, 0,
+        `Stream ${number} is empty`)
+    }
+  })
+
 Then('the consumer receives the reply:',
   /**
    * @param {string} yaml
@@ -393,6 +474,17 @@ Then('all replies have been received',
     await Promise.all(this.requestsSent)
   })
 
+/**
+ * @param {number} amount
+ * @param {string} queue
+ * @this {comq.features.Context}
+ */
+function sendMany (amount, queue) {
+  for (let i = 0; i < amount; i++) {
+    this.requestsSent.push(this.io.request(queue, randomBytes(8)))
+  }
+}
+
 async function send (queue, payload) {
   if (this.expected) await this.expected
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.15.1",
+  "version": "0.15.3",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",

package/source/.io/ReplyStream.js

@@ -44,13 +44,21 @@ class ReplyStream extends Readable {
     this.#maxBufferSize = global['COMQ_TESTING_MAX_BUFFER_SIZE'] || MAX_BUFFER_SIZE
     this.#reply = reply
 
+    this.confirmation.catch(noop) // it is not awaited until the stream is handed over
+
     this.#emitter.on(this.#correlationId, this.arrange.bind(this))
+
+    // control.ok may never arrive, hence the watchdog is armed before the first message
+    this._heartbeat()
   }
 
   _destroy (error, callback) {
     this._clear()
     this.push(null)
 
+    // a no-op once control.ok has been received
+    this.confirmation.reject(error ?? new Error(UNCONFIRMED))
+
     if (this.#control !== undefined)
       void this.#reply(this.#control, control.end)
 
@@ -145,4 +153,9 @@ class ReplyStream extends Readable {
 
 const MAX_BUFFER_SIZE = 1000
 
+const UNCONFIRMED = 'Reply stream has been destroyed before confirmation'
+
+function noop () {}
+
 exports.ReplyStream = ReplyStream
+exports.UNCONFIRMED = UNCONFIRMED

package/source/connection.js

@@ -46,6 +46,10 @@ class Connection {
     this.#diagnostics.on('error', noop)
   }
 
+  get connected () {
+    return this.#connection !== undefined
+  }
+
   async open () {
     if (this.#opening !== null) return this.#opening
 
@@ -147,7 +151,7 @@ class Connection {
    * @param {comq.amqp.Connection} connection
    * @param {number} [timeoutMs]
    */
-  #armWatchdog (connection, timeoutMs = WATCHDOG_MS) {
+  #armWatchdog (connection, timeoutMs = global.COMQ_TESTING_WATCHDOG_INTERVAL ?? WATCHDOG_MS) {
     const socket = connection.connection?.stream
 
     if (socket === undefined) return

package/source/events.js

@@ -4,4 +4,4 @@
 exports.connection = ['open', 'close', 'error']
 
 /** @type {comq.diagnostics.Event[]} */
-exports.channel = ['flow', 'drain', 'recover', 'discard', 'pause', 'resume']
+exports.channel = ['flow', 'drain', 'recover', 'discard', 'pause', 'resume', 'return', 'lost']

package/source/io.js

@@ -35,8 +35,8 @@ class IO {
   /** @type {comq.ReplyEmitter | null} */
   #control = null
 
-  /** @type {Set<Promex>} */
-  #pendingReplies = new Set()
+  /** @type {Map<Promex, comq.Request>} */
+  #pendingReplies = new Map()
 
   /** @type {Set<comq.Destroyable>} */
   #replyStreams = new Set()
@@ -230,9 +230,14 @@ class IO {
   }
 
   #setupRetransmission () {
-    const event = (this.#requests.sharded === true) ? 'remove' : 'recover'
-
-    this.#requests.diagnose(event, this.#retransmit)
+    if (this.#requests.sharded === true) {
+      // a shard leaves the pool when it rejects a publish, and is lost when its
+      // connection drops, which leaves an already sent request unanswered
+      this.#requests.diagnose('remove', this.#retransmit)
+      this.#requests.diagnose('lost', this.#retransmit)
+    } else {
+      this.#requests.diagnose('recover', this.#retransmit)
+    }
   }
 
   /**
@@ -316,7 +321,7 @@ class IO {
    * @return {Promex<any>}
    */
   #createReply (request) {
-    const reply = this.#createPendingReply()
+    const reply = this.#createPendingReply(request)
 
     request.emitter.once(request.properties.correlationId, this.#getReplyResolver(request, reply))
 
@@ -324,12 +329,13 @@ class IO {
   }
 
   /**
+   * @param {comq.Request} request
    * @return {Promex}
    */
-  #createPendingReply () {
+  #createPendingReply (request) {
     const reply = new Promex()
 
-    this.#pendingReplies.add(reply)
+    this.#pendingReplies.set(reply, request)
 
     reply
       .catch(noop)
@@ -349,7 +355,12 @@ class IO {
       if (isStream) {
         const stream = this.#createReplyStream(request, payload, properties)
 
-        await stream.confirmation
+        try {
+          await stream.confirmation
+        } catch {
+          // the stream has never started, hence the request is re-sent
+          return reply.reject(RETRANSMISSION)
+        }
 
         reply.resolve(stream)
       } else {
@@ -439,6 +450,9 @@ class IO {
     properties.contentType = contentType
     properties.correlationId = request.properties.correlationId
 
+    // an unroutable reply must be returned by the broker rather than dropped
+    properties.mandatory = true
+
     return await this.#replies.fire(replyTo, buffer, properties)
   }
 
@@ -447,10 +461,14 @@ class IO {
   }
 
   #retransmit = () => {
-    for (const emitter of this.#emitters.values()) emitter.removeAllListeners()
+    for (const [reply, request] of this.#pendingReplies) {
+      // detaching this attempt alone leaves the listeners of the reply streams
+      // that are still flowing over the other shards in place
+      request.emitter.removeAllListeners(request.properties.correlationId)
 
-    // trigger failsafe attribute
-    for (const reply of this.#pendingReplies) reply.reject(RETRANSMISSION)
+      // trigger failsafe attribute
+      reply.reject(RETRANSMISSION)
+    }
   }
 
   /**

package/source/shards/channel.js

@@ -19,8 +19,14 @@ class Channel {
   /** @type {comq.Channel[]} */
   #pool
 
-  /** @type {Set<Promise<comq.Channel>>} */
-  #pending = new Set()
+  /** @type {Map<Promise<comq.Channel>, number>} */
+  #pending = new Map()
+
+  /** @type {Promex[]} */
+  #down = []
+
+  /** @type {boolean[]} */
+  #alive = []
 
   /** @type {Map<comq.Channel, Promex>} */
   #bench = new Map()
@@ -48,11 +54,11 @@ class Channel {
   }
 
   async consume (queue, consumer) {
-    return await this.#any((channel) => channel.consume(queue, consumer))
+    return await this.#every((channel) => channel.consume(queue, consumer))
   }
 
   async subscribe (queue, group, consumer) {
-    await this.#any((channel) => channel.subscribe(queue, group, consumer))
+    await this.#every((channel) => channel.subscribe(queue, group, consumer))
   }
 
   async send (queue, buffer, options) {
@@ -86,8 +92,10 @@ class Channel {
    * @return {Promise<void>}
    */
   #create = async (connection, index) => {
+    this.#watch(connection, index)
+
     const pending = connection.createChannel(this.#type, index)
-    const channel = await this.#pend(pending)
+    const channel = await this.#pend(pending, index)
 
     this.#add(channel)
     this.#pipe(channel)
@@ -97,12 +105,41 @@ class Channel {
     channel.diagnose('recover', () => this.#recover(channel))
   }
 
+  /**
+   * Tracks whether a shard is reachable, so that subscribing does not wait for
+   * one that is not. The pool itself is left alone: a channel is only benched
+   * when it actually fails to publish, otherwise a lost connection would
+   * interrupt the streams it is carrying. Losing a shard is reported instead,
+   * since a request awaiting its reply on it will never be answered.
+   *
+   * @param {comq.Connection} connection
+   * @param {number} index
+   */
+  #watch (connection, index) {
+    this.#down[index] = new Promex()
+    this.#alive[index] = connection.connected !== false
+
+    if (connection.connected === false) this.#down[index].resolve()
+
+    connection.diagnose('close', () => {
+      this.#alive[index] = false
+      this.#down[index].resolve()
+      this.#diagnostics.emit(LOST, index)
+    })
+
+    connection.diagnose('open', () => {
+      this.#alive[index] = true
+      this.#down[index] = new Promex()
+    })
+  }
+
   /**
    * @param {Promise<comq.Channel>} pending
+   * @param {number} index
    * @return {Promise<comq.Channel>}
    */
-  async #pend (pending) {
-    this.#pending.add(pending)
+  async #pend (pending, index) {
+    this.#pending.set(pending, index)
 
     const channel = await pending
 
@@ -116,8 +153,44 @@ class Channel {
    */
   #pipe (channel) {
     for (const event of events.channel) {
+      if (event === RETURN) continue // returns are retried before being reported
+      if (event === LOST) continue // a shard is lost with its connection, not its channel
+
       channel.diagnose(event, (...args) => this.#diagnostics.emit(event, ...args, channel.index))
     }
+
+    channel.diagnose(RETURN, (message) => this.#returned(message, channel))
+  }
+
+  /**
+   * An unroutable message is retried on the shards that have not seen it yet,
+   * since a queue may be declared on some of them only. The return is reported
+   * once every shard has rejected the message.
+   *
+   * @param {comq.amqp.Message} message
+   * @param {comq.Channel} channel
+   */
+  #returned (message, channel) {
+    const report = () => this.#diagnostics.emit(RETURN, message, channel.index)
+    const attempt = (message.properties.headers?.[RETURN_HEADER] ?? 0) + 1
+    const rest = this.#pool.filter((one) => one !== channel)
+
+    // only replies are published to the default exchange, and only they are mandatory
+    const exhausted = message.fields.exchange !== DEFAULT ||
+      attempt >= this.#connections.length ||
+      rest.length === 0
+
+    if (exhausted) return report()
+
+    const properties = {
+      ...message.properties,
+      mandatory: true,
+      headers: { ...message.properties.headers, [RETURN_HEADER]: attempt }
+    }
+
+    const next = rest[Math.floor(Math.random() * rest.length)]
+
+    next.fire(message.fields.routingKey, message.content, properties).catch(report)
   }
 
   /**
@@ -171,12 +244,49 @@ class Channel {
   }
 
   /**
+   * Resolves once every available shard has settled, requiring at least one to
+   * succeed. Benched shards are applied when they come back, which must not
+   * hold up the caller.
+   *
    * @param {(channel: comq.Channel) => void} fn
+   * @return {Promise<any>}
    */
-  async #any (fn) {
-    const promises = this.#apply(fn)
+  async #every (fn) {
+    const promises = []
 
-    return await Promise.any(promises)
+    for (const channel of this.#channels) {
+      promises.push(this.#unless(fn(channel), channel.index))
+    }
+
+    for (const [pending, index] of this.#pending) {
+      promises.push(this.#unless(pending.then(fn), index))
+    }
+
+    for (const recover of this.#bench.values()) recover.then(fn).catch(noop)
+
+    const results = await Promise.allSettled(promises)
+    const fulfilled = results.find((result) => result.status === 'fulfilled')
+
+    if (fulfilled === undefined) {
+      const reasons = results.map((result) => result.reason)
+
+      throw new AggregateError(reasons, 'No shard is available')
+    }
+
+    return fulfilled.value
+  }
+
+  /**
+   * Gives up on a shard as soon as its connection is lost.
+   *
+   * @param {Promise<any>} promise
+   * @param {number} index
+   * @return {Promise<any>}
+   */
+  async #unless (promise, index) {
+    const down = this.#down[index]
+
+    return down === undefined ? await promise : await Promise.race([promise, down])
   }
 
   /**
@@ -197,19 +307,34 @@ class Channel {
     const promises = []
 
     for (const channel of this.#channels) promises.push(fn(channel))
-    for (const pending of this.#pending) promises.push(pending.then(fn))
+    for (const pending of this.#pending.keys()) promises.push(pending.then(fn))
     for (const recover of this.#bench.values()) promises.push(recover.then(fn))
 
     return promises
   }
 
+  /**
+   * The channels of the shards that are known to be connected, falling back to the
+   * whole pool while none is, so that a publish is attempted rather than dropped.
+   *
+   * @return {comq.Channel[]}
+   */
+  #reachable () {
+    const reachable = this.#pool.filter((channel) => this.#alive[channel.index] !== false)
+
+    return reachable.length === 0 ? this.#pool : reachable
+  }
+
   /**
    * @param {(channel: comq.Channel) => void} fn
    */
   async #one (fn) {
     if (this.#pool.length === 0) await this.#recovery
 
-    const channel = this.#pool[Math.floor(Math.random() * this.#pool.length)]
+    // publishing to a lost shard is silently accepted by the destroyed socket, so
+    // it must be avoided rather than retried: nothing would report the loss twice
+    const pool = this.#reachable()
+    const channel = pool[Math.floor(Math.random() * pool.length)]
 
     try {
       return await fn(channel)
@@ -234,4 +359,11 @@ async function create (connections, type) {
   return channel
 }
 
+const DEFAULT = ''
+const RETURN = 'return'
+const RETURN_HEADER = 'x-return'
+const LOST = 'lost'
+
+function noop () {}
+
 exports.create = create

package/test/ReplyStream.test.js

@@ -2,7 +2,8 @@
 
 const { EventEmitter } = require('node:events')
 const { once } = require('node:events')
-const { ReplyStream } = require('../source/.io/ReplyStream')
+const { ReplyStream, UNCONFIRMED } = require('../source/.io/ReplyStream')
+const { control } = require('../source/.io/const')
 
 /** @type {jest.Mock} */
 let reply
@@ -43,3 +44,40 @@ it('should destroy without calling reply when buffer overflows before control.ok
   expect(stream.destroyed).toBe(true)
   expect(reply).not.toHaveBeenCalled()
 })
+
+it('should reject confirmation when buffer overflows before control.ok', async () => {
+  for (let index = 1; index <= 5; index++) {
+    stream.arrange(index, { headers: { index } })
+  }
+
+  await expect(stream.confirmation).rejects.toThrow(UNCONFIRMED)
+})
+
+it('should reject confirmation when destroyed before control.ok', async () => {
+  stream.destroy()
+
+  await expect(stream.confirmation).rejects.toThrow(UNCONFIRMED)
+})
+
+it('should reject confirmation when nothing arrives within the idle interval', async () => {
+  global.COMQ_TESTING_IDLE_INTERVAL = 10
+
+  const request = {
+    emitter: new EventEmitter(),
+    properties: { correlationId: 'idle-correlation' }
+  }
+
+  const idle = new ReplyStream(request, reply)
+
+  await expect(idle.confirmation).rejects.toThrow(UNCONFIRMED)
+})
+
+it('should keep confirmation resolved after control.ok', async () => {
+  stream.arrange(control.ok, { headers: { index: 0 }, type: 'control' })
+
+  await expect(stream.confirmation).resolves.toBeUndefined()
+
+  stream.destroy()
+
+  await expect(stream.confirmation).resolves.toBeUndefined()
+})

package/test/connection.mock.js

@@ -19,6 +19,7 @@ const channel = /** @type {jest.MockedFunction<(sharded?: boolean, index?: numbe
  * @returns {jest.MockedObject<comq.Connection>}
  */
 const connection = (sharded = false) => (/** @type {jest.MockedObject<comq.Connection>} */ {
+  connected: true,
   createChannel: jest.fn(async (type, index) => channel(sharded, index)),
   open: jest.fn(async () => undefined),
   close: jest.fn(async () => undefined),

package/test/io.reply.test.js

@@ -185,7 +185,8 @@ describe('encoding', () => {
 
     const props = {
       correlationId: properties.correlationId,
-      contentType: properties.contentType
+      contentType: properties.contentType,
+      mandatory: true
     }
 
     expect(replies.fire).toHaveBeenCalledWith(properties.replyTo, reply, props)

package/test/io.request.test.js

@@ -207,8 +207,125 @@ describe('send', () => {
 
     expect(requests.send).toHaveBeenCalledTimes(2)
   })
+
+  it('should resend unanswered Requests when a shard is lost', async () => {
+    jest.clearAllMocks()
+
+    connection = mock.connection(true)
+    io = new IO(connection)
+
+    promise = io.request(queue, payload)
+
+    // allows initializers to run
+    await immediate()
+
+    requests = await findChannel('request')
+
+    expect(requests.diagnose).toHaveBeenCalledWith('lost', expect.any(Function))
+
+    await lose()
+
+    expect(requests.send).toHaveBeenCalledTimes(2)
+  })
+
+  it('should not interrupt a confirmed reply stream when a shard is lost', async () => {
+    jest.clearAllMocks()
+
+    connection = mock.connection(true)
+    io = new IO(connection)
+
+    promise = io.request(queue, payload)
+
+    await immediate()
+
+    requests = await findChannel('request')
+    replies = await findChannel('reply')
+
+    const correlationId = requests.send.mock.calls[0][2].correlationId
+    const callback = replies.consume.mock.calls[0][1]
+
+    await callback(message(correlationId, 0, 'ok', 'control'))
+
+    const output = await promise
+
+    expect(output).toBeInstanceOf(stream.Readable)
+
+    await lose()
+
+    // the stream is carried by whichever shard replied, so it keeps flowing
+    const chunk = randomBytes(8)
+
+    await callback(message(correlationId, 1, chunk))
+
+    expect(output.read()).toStrictEqual(chunk)
+
+    output.destroy()
+  })
+
+  it('should not resend an answered Request when a shard is lost', async () => {
+    jest.clearAllMocks()
+
+    connection = mock.connection(true)
+    io = new IO(connection)
+
+    promise = io.request(queue, payload)
+
+    await immediate()
+
+    requests = await findChannel('request')
+    replies = await findChannel('reply')
+
+    const correlationId = requests.send.mock.calls[0][2].correlationId
+    const callback = replies.consume.mock.calls[0][1]
+
+    const answer = /** @type {comq.amqp.Message} */
+      { content: randomBytes(8), properties: { correlationId } }
+
+    await callback(answer)
+    await promise
+
+    await lose()
+
+    expect(requests.send).toHaveBeenCalledTimes(1)
+  })
 })
 
+/**
+ * Fires the listeners the IO has attached to the shard loss diagnostic.
+ *
+ * @return {Promise<void>}
+ */
+async function lose () {
+  const calls = requests.diagnose.mock.calls.filter((call) => call[0] === 'lost')
+
+  for (const [, listener] of calls) listener(0)
+
+  await immediate()
+  await immediate()
+}
+
+/**
+ * @param {string} correlationId
+ * @param {number} index
+ * @param {any} content
+ * @param {string} [type]
+ * @return {comq.amqp.Message}
+ */
+function message (correlationId, index, content, type) {
+  const raw = Buffer.isBuffer(content)
+
+  return /** @type {comq.amqp.Message} */ {
+    content: raw ? content : encode(content, 'application/json'),
+    properties: {
+      correlationId,
+      type,
+      replyTo: queue,
+      contentType: raw ? 'application/octet-stream' : 'application/json',
+      headers: { index }
+    }
+  }
+}
+
 describe('reply', () => {
   it.each([undefined, 'application/octet-stream'])('should return raw content if encoding is %s',
     async (contentType) => {
@@ -237,6 +354,30 @@ describe('reply', () => {
 
       expect(output).toStrictEqual(value)
     })
+
+  it('should re-send the Request if a reply stream is never confirmed', async () => {
+    global.COMQ_TESTING_MAX_BUFFER_SIZE = 3
+
+    const correlationId = requests.send.mock.calls[0][2].correlationId
+    const callback = replies.consume.mock.calls[0][1]
+
+    try {
+      // out of order chunks overflow the buffer, so the stream dies before control.ok
+      for (let index = 1; index <= 5; index++) {
+        const properties = { correlationId, headers: { index } }
+        const message = /** @type {comq.amqp.Message} */ { content: randomBytes(8), properties }
+
+        await callback(message)
+      }
+
+      await immediate()
+      await immediate()
+
+      expect(requests.send).toHaveBeenCalledTimes(2)
+    } finally {
+      delete global.COMQ_TESTING_MAX_BUFFER_SIZE
+    }
+  })
 })
 
 const reply = async (content = randomBytes(8), contentType = undefined) => {

package/test/shards/channel.test.js

@@ -29,6 +29,58 @@ it('should expose `sharded`', async () => {
   expect(channel.sharded).toStrictEqual(true)
 })
 
+it('should report a lost shard', async () => {
+  channel = await create(connections, type)
+
+  const lost = /** @type {jest.MockedFunction} */ jest.fn()
+
+  channel.diagnose('lost', lost)
+
+  const calls = connections[1].diagnose.mock.calls.filter((call) => call[0] === 'close')
+
+  expect(calls.length).toBeGreaterThan(0)
+
+  for (const call of calls) call[1]()
+
+  expect(lost).toHaveBeenCalledWith(1)
+})
+
+it('should not publish to a lost shard', async () => {
+  channel = await create(connections, type)
+
+  const channels = await getCreatedChannels()
+  const calls = connections[1].diagnose.mock.calls.filter((call) => call[0] === 'close')
+
+  for (const call of calls) call[1]()
+
+  const buffer = randomBytes(8)
+  const label = generate()
+
+  for (let i = 0; i < 20; i++) await channel.send(label, buffer)
+
+  expect(channels[0].send).toHaveBeenCalledTimes(20)
+  expect(channels[1].send).not.toHaveBeenCalled()
+})
+
+// a lost shard is not benched, so it is used again as soon as it reconnects
+it('should publish to a recovered shard', async () => {
+  channel = await create(connections, type)
+
+  const channels = await getCreatedChannels()
+  const close = connections[1].diagnose.mock.calls.filter((call) => call[0] === 'close')
+  const open = connections[1].diagnose.mock.calls.filter((call) => call[0] === 'open')
+
+  for (const call of close) call[1]()
+  for (const call of open) call[1]()
+
+  const buffer = randomBytes(8)
+  const label = generate()
+
+  for (let i = 0; i < 20; i++) await channel.send(label, buffer)
+
+  expect(channels[1].send).toHaveBeenCalled()
+})
+
 it('should resolve when one of the connections has created a channel', async () => {
   /** @type {toa.generic.Promex[]} */
   const promises = []
@@ -87,28 +139,75 @@ describe.each(['consume', 'subscribe'])('%s', (method) => {
     for (const chan of channels) expect(chan[method]).toHaveBeenCalledWith(...args)
   })
 
-  it(`should should ${method} using available and pending channels`, async () => {
+  it(`should not resolve ${method} until pending channels are created`, async () => {
     const promise = promex()
 
     connections[0].createChannel.mockImplementationOnce(() => promise)
 
     channel = await create(connections, type)
 
-    await channel[method](...args)
+    let resolved = false
+
+    const subscription = channel[method](...args).then(() => { resolved = true })
+
+    await immediate()
 
     /** @type {jest.MockedObject<comq.Channel>} */
     const chan1 = await connections[1].createChannel.mock.results[0].value
 
     expect(chan1[method]).toHaveBeenCalled()
+    expect(resolved).toStrictEqual(false)
 
     /** @type {jest.MockedObject<comq.Channel>} */
     const chan0 = mock.channel()
 
     promise.resolve(chan0)
 
-    await immediate()
+    await subscription
 
     expect(chan0[method]).toHaveBeenCalled()
+    expect(resolved).toStrictEqual(true)
+  })
+
+  it(`should resolve ${method} without a disconnected shard`, async () => {
+    const promise = promex()
+
+    connections[0].createChannel.mockImplementationOnce(() => promise)
+    connections[0].connected = false
+
+    try {
+      channel = await create(connections, type)
+
+      // must not wait for the channel that is never going to be created
+      await channel[method](...args)
+
+      /** @type {jest.MockedObject<comq.Channel>} */
+      const chan1 = await connections[1].createChannel.mock.results[0].value
+
+      expect(chan1[method]).toHaveBeenCalled()
+    } finally {
+      connections[0].connected = true
+      promise.resolve(mock.channel())
+    }
+  })
+
+  it(`should resolve ${method} when a shard disconnects`, async () => {
+    const promise = promex()
+
+    connections[0].createChannel.mockImplementationOnce(() => promise)
+
+    channel = await create(connections, type)
+
+    const subscription = channel[method](...args)
+    const calls = connections[0].diagnose.mock.calls.filter((call) => call[0] === 'close')
+
+    expect(calls.length).toBeGreaterThan(0)
+
+    for (const call of calls) call[1]()
+
+    await subscription
+
+    promise.resolve(mock.channel())
   })
 
   it(`should ${method} using a channel removed from the pool once it is recovered`, async () => {
@@ -339,6 +438,43 @@ describe('diagnose', () => {
     expect(listener).toHaveBeenCalledWith(expect.any(Number))
   })
 
+  it('should re-fire a returned message on another shard', async () => {
+    const listener = /** @type {jest.MockedFunction} */ jest.fn()
+    const index = random(channels.length)
+    const chan = channels[index]
+    const message = returned()
+
+    channel.diagnose('return', listener)
+
+    emitReturn(chan, message)
+
+    const rest = channels.filter((one) => one !== chan)
+
+    const properties = expect.objectContaining(
+      { mandatory: true, headers: { 'x-return': 1 } })
+
+    for (const one of rest) {
+      expect(one.fire).toHaveBeenCalledWith(message.fields.routingKey, message.content, properties)
+    }
+
+    expect(listener).not.toHaveBeenCalled()
+  })
+
+  it('should emit `return` event once every shard has returned the message', async () => {
+    const listener = /** @type {jest.MockedFunction} */ jest.fn()
+    const index = random(channels.length)
+    const chan = channels[index]
+    const message = returned({ 'x-return': channels.length - 1 })
+
+    channel.diagnose('return', listener)
+
+    emitReturn(chan, message)
+
+    for (const one of channels) expect(one.fire).not.toHaveBeenCalled()
+
+    expect(listener).toHaveBeenCalledWith(message, index)
+  })
+
   it('should emit `pause` and `unpause` events', async () => {
     const queue = generate()
     const buffer = randomBytes(8)
@@ -428,6 +564,30 @@ it('should remove channel with back pressure from the pool', async () => {
   expect(chan.send).toHaveBeenCalled()
 })
 
+/**
+ * @param {Record<string, any>} [headers]
+ * @return {comq.amqp.Message}
+ */
+function returned (headers = {}) {
+  return /** @type {comq.amqp.Message} */ {
+    content: randomBytes(8),
+    fields: { exchange: '', routingKey: generate() },
+    properties: { correlationId: generate(), headers }
+  }
+}
+
+/**
+ * @param {jest.MockedObject<comq.Channel>} channel
+ * @param {comq.amqp.Message} message
+ */
+function emitReturn (channel, message) {
+  const calls = channel.diagnose.mock.calls.filter((call) => call[0] === 'return')
+
+  expect(calls.length).toBeGreaterThan(0)
+
+  for (const call of calls) call[1](message)
+}
+
 /**
  * @return {Promise<jest.MockedObject<comq.Channel>[]>}
  */

package/types/connection.d.ts

@@ -6,6 +6,8 @@ import * as _topology from './topology'
 declare namespace comq {
 
   interface Connection {
+    readonly connected?: boolean
+
     open(): Promise<void>
 
     close(): Promise<void>