comq 0.15.8 → 0.16.1

package/.releaserc

@@ -3,8 +3,24 @@
     "release"
   ],
   "plugins": [
-    "@semantic-release/commit-analyzer",
-    "@semantic-release/release-notes-generator",
+    [
+      "@semantic-release/commit-analyzer",
+      {
+        "preset": "conventionalcommits",
+        "releaseRules": [
+          {
+            "breaking": true,
+            "release": "minor"
+          }
+        ]
+      }
+    ],
+    [
+      "@semantic-release/release-notes-generator",
+      {
+        "preset": "conventionalcommits"
+      }
+    ],
     [
       "@semantic-release/npm"
     ],

package/features/channels.feature

@@ -0,0 +1,14 @@
+Feature: Channel exhaustion
+
+  A connection allocates channels from a range the broker negotiates, and once the range
+  is spent there is nothing a reconnection can do about it. The refusal is reported and
+  passed on rather than retried, so that whoever asked for a channel is told instead of
+  waiting for a recovery that is not coming.
+
+  Scenario: Reporting a connection with no channel left to allocate
+    Given the AMQP channel limit is set to 2
+    And an active connection to the broker
+    And a producer replying `echo` queue
+    When emitting an event to the `exhaustion` exchange is attempted
+    Then the exception is thrown: "No channels left to allocate"
+    And the `exhausted` event is emitted

package/features/connection.feature

@@ -18,6 +18,20 @@ Feature: Connection Tolerance
     When the broker is up
     Then the connection is restored
 
+  Scenario: Restoring singleton connection after broker restart
+    Given an active singleton connection
+    When the broker is down
+    Then the connection is lost
+    When the broker is up
+    Then the connection is restored
+
+  Scenario: Restoring sharded singleton connection after broker crash
+    Given an active sharded singleton connection
+    When the broker has crashed
+    Then the connection is lost
+    When the broker is up
+    Then the connection is restored
+
   Scenario: Connecting with wrong credentials
     When I attempt to connect to the broker as "someone" with password "whatever"
     Then an exception is thrown: "Handshake terminated by server: 403 (ACCESS-REFUSED)"

package/features/silence.feature

@@ -24,6 +24,22 @@ Feature: Silent Connection Tolerance
     When the consumer sends a request to the `echo` queue
     Then the consumer receives the reply
 
+  Scenario: Restoring a singleton connection that has gone silent
+    Given an active singleton connection
+    And a producer replying `echo` queue
+    When the network goes silent
+    Then the connection is lost within 10 seconds
+    When the consumer sends a request to the `echo` queue
+    Then the consumer receives the reply
+
+  Scenario: Restoring a sharded singleton connection whose every shard has gone silent
+    Given an active sharded singleton connection
+    And a producer replying `echo` queue
+    When the network goes silent
+    Then the connection is lost within 10 seconds
+    When the consumer sends a request to the `echo` queue
+    Then the consumer receives the reply
+
   Scenario: Answering a request sent as the connection goes silent
     Given an active connection to the broker
     And a producer replying `echo` queue

package/features/steps/channels.js

@@ -0,0 +1,45 @@
+'use strict'
+
+const assert = require('node:assert')
+const { randomBytes } = require('node:crypto')
+
+const { Given, When, Then } = require('@cucumber/cucumber')
+
+Given('the AMQP channel limit is set to {number}',
+  /**
+   * @param {number} limit
+   */
+  function (limit) {
+    global.COMQ_TESTING_AMQP_CHANNEL_MAX = limit
+  })
+
+When('emitting an event to the {token} exchange is attempted',
+  /**
+   * @param {string} exchange
+   * @this {comq.features.Context}
+   */
+  async function (exchange) {
+    this.exception = undefined
+
+    await this.io.emit(exchange, randomBytes(8))
+      .catch((exception) => { this.exception = exception })
+  })
+
+Then('the exception is thrown: {string}',
+  /**
+   * @param {string} message
+   * @this {comq.features.Context}
+   */
+  function (message) {
+    assert.notEqual(this.exception, undefined, 'no exception was thrown')
+    assert.equal(this.exception.message, message)
+  })
+
+Then('the {token} event is emitted',
+  /**
+   * @param {comq.diagnostics.Event} event
+   * @this {comq.features.Context}
+   */
+  function (event) {
+    assert.equal(this.events[event], true, `the '${event}' event was not emitted`)
+  })

package/features/steps/context.js

@@ -100,17 +100,27 @@ class Context extends World {
   #url (i, user, password) {
     const address = this.networks[i]?.address ?? getAddress(i)
     const url = PROTOCOL + user + ':' + password + '@' + address
-    const heartbeat = global.COMQ_TESTING_AMQP_HEARTBEAT
+    const query = []
 
     // the watchdog measures silence, so it may only be shortened along with the
     // interval at which a healthy broker is expected to say something
-    return heartbeat === undefined ? url : url + '?heartbeat=' + heartbeat
+    if (global.COMQ_TESTING_AMQP_HEARTBEAT !== undefined) {
+      query.push('heartbeat=' + global.COMQ_TESTING_AMQP_HEARTBEAT)
+    }
+
+    // a broker negotiates the channel limit down to what the client asks for,
+    // which is how exhaustion is reached without opening two thousand channels
+    if (global.COMQ_TESTING_AMQP_CHANNEL_MAX !== undefined) {
+      query.push('channelMax=' + global.COMQ_TESTING_AMQP_CHANNEL_MAX)
+    }
+
+    return query.length === 0 ? url : url + '?' + query.join('&')
   }
 }
 
 const PROTOCOL = 'amqp://'
 
 /** @type {comq.diagnostics.Event[]} */
-const EVENTS = ['open', 'close', 'flow', 'discard', 'pause', 'resume']
+const EVENTS = ['open', 'close', 'flow', 'discard', 'pause', 'resume', 'exhausted']
 
 exports.Context = Context

package/features/steps/hooks.js

@@ -14,6 +14,7 @@ After(
   async function () {
     delete global.COMQ_TESTING_WATCHDOG_INTERVAL
     delete global.COMQ_TESTING_AMQP_HEARTBEAT
+    delete global.COMQ_TESTING_AMQP_CHANNEL_MAX
 
     await Promise.all(
       Array.from({ length: BROKERS_AMOUNT }, (_, n) => actions.up(n))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.15.8",
+  "version": "0.16.1",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",
@@ -33,15 +33,17 @@
     "@semantic-release/github": "^12.0.2",
     "@semantic-release/npm": "^13.1.2",
     "@semantic-release/release-notes-generator": "^14.1.0",
-    "@testcontainers/rabbitmq": "^12.0.4",
+    "@testcontainers/rabbitmq": "^12.1.0",
     "@types/amqplib": "^0.10.8",
     "@types/jest": "^30.0.0",
+    "conventional-changelog-conventionalcommits": "^8.0.0",
     "fast-glob": "^3.3.3",
-    "fuzzysort": "3.1.0",
+    "fuzzysort": "4.0.2",
     "husky": "^9.1.7",
     "jest": "^30.4.2",
-    "js-yaml": "^5.2.3",
+    "js-yaml": "^5.4.1",
     "randomstring": "^1.3.1",
+    "semantic-release": "^25.0.9",
     "snazzy": "^9.0.0",
     "standard": "^17.1.2",
     "testcontainers": "^12.0.4"

package/readme.md

@@ -498,6 +498,9 @@ Subscribe to one of the diagnostic events:
 - `error`: an attempt to restore the connection has failed. The exception is passed as an argument.
   Attempts continue until one succeeds.
 - `reconnect`: an attempt to restore the connection has started. Followed by `open` or `error`.
+- `exhausted`: the connection has no channel left to allocate. The negotiated channel limit is
+  passed as an argument. Reconnecting does not free a channel, so the operation that asked for
+  one is rejected rather than retried.
 - `flow`: back pressure is applied to a channel. [Channel type](./types/topology.d.ts) is passed as
   an argument.
 - `drain`: back pressure is removed from a channel. Channel type is passed.
@@ -533,6 +536,7 @@ established, there is no way to capture the initial `open` event.
 io.diagnose('flow', (type) => console.log(`Back pressure was applied to the ${type} channel`))
 
 io.diagnose('reconnect', (shard) => console.log('AMQP reconnecting', { shard }))
+io.diagnose('exhausted', (limit, shard) => console.log('AMQP channels exhausted', { limit, shard }))
 io.diagnose('open', (shard) => console.log('AMQP connection established', { shard }))
 io.diagnose('close', (error, shard) => console.log('AMQP connection closed', { message: error?.message, shard }))
 io.diagnose('error', (error, shard) => console.log('AMQP connection failed', { message: error.message, shard }))

package/source/channel.js

@@ -42,6 +42,8 @@ class Channel {
 
   #diagnostics = emitter.create()
 
+  #closed = false
+
   /**
    * @param {comq.amqp.Connection} connection
    * @param {comq.Topology} topology
@@ -126,6 +128,21 @@ class Channel {
     }
   }
 
+  /**
+   * Gives the channel back. A connection has a limited number of them, and one whose
+   * IO is done with it would otherwise be held until the whole connection goes.
+   */
+  async close () {
+    if (this.#closed) return
+
+    this.#closed = true
+
+    await this.seal()
+
+    // a channel that went down with its connection is the outcome this asks for
+    await this.#channel?.close().catch(noop)
+  }
+
   async seal () {
     this.#sealed = true
 
@@ -142,6 +159,10 @@ class Channel {
     this.#diagnostics.off(event, listener)
   }
 
+  get closed () {
+    return this.#closed
+  }
+
   async recover (connection) {
     this.#connection = connection
 

package/source/connection.js

@@ -64,17 +64,10 @@ class Connection {
   async open () {
     this.#closed = false
 
-    if (this.#opening !== null) return this.#opening
-
-    this.#opening = retry(this.#open).finally(() => { this.#opening = null })
-
-    await this.#opening
+    await this.#reopen()
 
     if (this.#closed) return
 
-    // close may have landed after this attempt succeeded but before `#opening` was cleared
-    if (this.#connection === undefined) return this.open()
-
     this.#running = true
   }
 
@@ -99,6 +92,7 @@ class Connection {
       const topology = presets[type]
       const channel = await channels.create(this.#connection, topology, index)
 
+      this.#channels = this.#channels.filter((channel) => !channel.closed)
       this.#channels.push(channel)
 
       return channel
@@ -140,6 +134,8 @@ class Connection {
     this.#diagnostics.emit('open')
 
     try {
+      this.#channels = this.#channels.filter((channel) => !channel.closed)
+
       for (const channel of this.#channels) await channel.recover(connection)
     } catch (exception) {
       this.#diagnostics.emit('error', exception)
@@ -169,10 +165,29 @@ class Connection {
     this.#connection = undefined
 
     if (error !== undefined && !this.#closed) {
-      this.open().catch((exception) => this.#diagnostics.emit('error', exception))
+      this.#reopen().catch((exception) => this.#diagnostics.emit('error', exception))
     }
   }
 
+  /**
+   * Establishes the connection, or joins the attempt under way. A lost connection is
+   * restored through here and not through `open()`: what a subclass makes of `open()` is
+   * its own — the singleton answers it with the first opening, made once and remembered,
+   * which is no way to make a second one.
+   *
+   * @return {Promise<void>}
+   */
+  async #reopen () {
+    this.#opening ??= retry(this.#open).finally(() => { this.#opening = null })
+
+    await this.#opening
+
+    if (this.#closed) return
+
+    // close may have landed after this attempt succeeded but before `#opening` was cleared
+    if (this.#connection === undefined) return this.#reopen()
+  }
+
   /**
    * An AMQP connection is only closed once the broker has replied with Close-Ok,
    * which never happens on a connection that has already been lost.
@@ -207,7 +222,20 @@ class Connection {
     connection.connection?.stream?.destroy(silence())
   }
 
-  #recover () {
+  /**
+   * @param {Error} [exception]
+   * @return {Promise<void> | false}
+   */
+  #recover (exception) {
+    // a connection that has no channel left to give is not something reconnecting
+    // fixes, and `#recovery` on a connection that is perfectly well is a promise
+    // nobody ever resolves — returning `false` lets the caller see the refusal
+    if (exception?.message === EXHAUSTED) {
+      this.#diagnostics.emit('exhausted', this.#connection?.connection?.channelMax)
+
+      return false
+    }
+
     return this.#recovery
   }
 
@@ -333,6 +361,9 @@ const SOCKET_OPTIONS = {
 
 const HEARTBEAT_SET = /[?&]heartbeat=/
 
+/** What amqplib says when the negotiated channel limit leaves no identifier free. */
+const EXHAUSTED = 'No channels left to allocate'
+
 const TRANSIENT_CODES = new Set([
   'ECONNREFUSED',
   'EAI_AGAIN',

package/source/events.js

@@ -1,7 +1,7 @@
 'use strict'
 
 /** @type {comq.diagnostics.Event[]} */
-exports.connection = ['open', 'close', 'error', 'reconnect']
+exports.connection = ['open', 'close', 'error', 'reconnect', 'exhausted']
 
 /** @type {comq.diagnostics.Event[]} */
 exports.channel = ['flow', 'drain', 'recover', 'discard', 'pause', 'resume', 'return', 'lost', 'remove']

package/source/io.js

@@ -163,6 +163,12 @@ class IO {
     await this.seal()
     await this.#destroyStreams(this.#replyPipes)
     await track(this)
+
+    // a connection is shared and outlives its IOs, so the channels are given back
+    // here — held to the end of the connection, they would run it out of them
+    await Promise.all([this.#requests, this.#replies, this.#events]
+      .map((channel) => channel?.close()))
+
     await this.#connection.close()
   })
 

package/source/shards/channel.js

@@ -77,6 +77,14 @@ class Channel {
     return await this.#one((channel) => channel.fire(queue, buffer, options))
   }
 
+  async close () {
+    await this.#all((channel) => channel.close())
+  }
+
+  get closed () {
+    return [...this.#channels].every((channel) => channel.closed)
+  }
+
   async seal () {
     await this.#all((channel) => channel.seal())
   }

package/test/amqplib.mock.js

@@ -15,6 +15,7 @@ class Channel extends EventEmitter {
   assertExchange = jest.fn(async () => undefined)
   bindQueue = jest.fn(async () => undefined)
   publish = jest.fn((_0, _1, _2, _3, resolve) => resolve?.(null))
+  close = jest.fn(async () => undefined)
 }
 
 class Connection extends EventEmitter {
@@ -37,7 +38,10 @@ class Connection extends EventEmitter {
     // a broker accepts the heartbeat the client asks for
     const heartbeat = Number(/[?&]heartbeat=(\d+)/.exec(url)?.[1] ?? 60)
 
-    this.connection = { stream, heartbeat }
+    // and negotiates the channel limit down to whatever the client asked for
+    const channelMax = Number(/[?&]channelMax=(\d+)/.exec(url)?.[1] ?? 2047)
+
+    this.connection = { stream, heartbeat, channelMax }
 
     // noinspection JSValidateTypes
     this.removeAllListeners = jest.spyOn(this, 'removeAllListeners')

package/test/channel.test.js

@@ -503,6 +503,49 @@ describe('seal', () => {
   })
 })
 
+describe('close', () => {
+  const queue = generate()
+  const consumer = jest.fn()
+
+  beforeEach(async () => {
+    channel = await create(connection, topology)
+    chan = await getCreatedChannel()
+  })
+
+  it('should give the channel back', async () => {
+    await channel.close()
+
+    expect(chan.close).toHaveBeenCalled()
+    expect(channel.closed).toStrictEqual(true)
+  })
+
+  it('should stop consuming before giving it back', async () => {
+    await channel.consume(queue, consumer)
+
+    const { consumerTag: tag } = await chan.consume.mock.results[0].value
+
+    await channel.close()
+
+    expect(chan.cancel).toHaveBeenCalledWith(tag)
+    expect(chan.cancel.mock.invocationCallOrder[0])
+      .toBeLessThan(chan.close.mock.invocationCallOrder[0])
+  })
+
+  it('should give it back once', async () => {
+    await channel.close()
+    await channel.close()
+
+    expect(chan.close).toHaveBeenCalledTimes(1)
+  })
+
+  // one that went down with its connection is the outcome this asks for
+  it('should ignore one that is already gone', async () => {
+    chan.close.mockImplementationOnce(async () => { throw new Error() })
+
+    await expect(channel.close()).resolves.not.toThrow()
+  })
+})
+
 describe('back pressure', () => {
   const exchange = generate()
   const queue = generate()

package/test/connection.mock.js

@@ -12,6 +12,8 @@ const channel = /** @type {jest.MockedFunction<(sharded?: boolean, index?: numbe
     publish: jest.fn(async () => undefined),
     diagnose: jest.fn(async () => undefined),
     seal: jest.fn(async () => undefined),
+    close: jest.fn(async () => undefined),
+    closed: false,
     recover: jest.fn(async () => undefined)
   }))
 

package/test/connection.test.js

@@ -184,7 +184,7 @@ describe('reconnection', () => {
 
     const boom = new Error('reconnect failed')
 
-    connection.open = jest.fn(async () => { throw boom })
+    amqplib.connect.mockRejectedValueOnce(boom)
 
     conn.emit('close', new Error('broker down'))
 
@@ -194,8 +194,38 @@ describe('reconnection', () => {
 
     expect(errors).toContain(boom)
     expect(unhandled).not.toHaveBeenCalled()
+
+    // the attempt is retried in the background, and would connect in a later case
+    await connection.close()
   })
 
+  // the singleton answers `open()` with the first opening, made once and remembered
+  it('should reconnect a connection whose open() is remembered', async () => {
+    class Remembered extends Connection {
+      #opened = null
+
+      async open () {
+        this.#opened ??= super.open()
+
+        await this.#opened
+      }
+    }
+
+    const remembered = new Remembered(generate())
+
+    await remembered.open()
+
+    const first = await amqplib.connect.mock.results.at(-1).value
+
+    first.emit('close', new Error('lost'))
+
+    await expect(remembered.createChannel('event')).resolves.toBeDefined()
+
+    expect(amqplib.connect).toHaveBeenCalledTimes(3)
+
+    await remembered.close()
+  }, 10000)
+
   it('should reconnect when connection is lost while channels recover', async () => {
     const channel = await connection.createChannel('request')
 
@@ -398,6 +428,106 @@ describe('create channel', () => {
   })
 })
 
+describe('closed channels', () => {
+  /** @type {jest.MockedObject<comq.amqp.Connection>} */
+  let conn
+
+  beforeEach(async () => {
+    // an earlier case leaves an implementation of its own behind
+    create.mockImplementation(async () => ({ recover: jest.fn(async () => undefined), closed: false }))
+
+    await connection.open()
+
+    conn = await amqplib.connect.mock.results[0].value
+  })
+
+  afterEach(async () => {
+    await connection.close()
+  })
+
+  // a channel given back is not one this connection has any more
+  it('should not recover one that was given back', async () => {
+    const channel = await connection.createChannel('event')
+
+    channel.closed = true
+
+    conn.emit('close', new Error())
+
+    await timeout(50)
+
+    expect(channel.recover).not.toHaveBeenCalled()
+  })
+
+  it('should recover one that is still in use', async () => {
+    const channel = await connection.createChannel('event')
+
+    conn.emit('close', new Error())
+
+    await timeout(50)
+
+    expect(channel.recover).toHaveBeenCalled()
+  })
+})
+
+describe('channel exhaustion', () => {
+  const EXHAUSTED = 'No channels left to allocate'
+
+  /** @type {jest.MockedObject<comq.amqp.Connection>} */
+  let conn
+
+  beforeEach(async () => {
+    await connection.open()
+
+    conn = await amqplib.connect.mock.results[0].value
+  })
+
+  it('should reject rather than wait for a recovery that is not coming', async () => {
+    create.mockImplementation(async () => { throw new Error(EXHAUSTED) })
+
+    // the regression is a hang, so the assertion is that it settles at all
+    const settled = await Promise.race([
+      connection.createChannel('event').then(() => 'resolved', (e) => e.message),
+      timeout(100).then(() => 'pending')
+    ])
+
+    expect(settled).toStrictEqual(EXHAUSTED)
+  })
+
+  it('should not retry', async () => {
+    create.mockImplementation(async () => { throw new Error(EXHAUSTED) })
+
+    await connection.createChannel('event').catch(() => undefined)
+
+    expect(create).toHaveBeenCalledTimes(1)
+  })
+
+  it('should report the limit the broker negotiated', async () => {
+    const listener = jest.fn()
+
+    await connection.diagnose('exhausted', listener)
+
+    create.mockImplementation(async () => { throw new Error(EXHAUSTED) })
+
+    await connection.createChannel('event').catch(() => undefined)
+
+    expect(listener).toHaveBeenCalledWith(conn.connection.channelMax)
+  })
+
+  it('should leave every other failure its recovery', async () => {
+    create.mockImplementation(async () => { throw new Error() })
+
+    setTimeout(() => {
+      create.mockImplementation(async () => generate())
+
+      conn.emit('close', new Error())
+    }, 1)
+
+    const channel = await connection.createChannel('request')
+
+    expect(channel).toStrictEqual(await create.mock.results[1].value)
+  })
+})
+
 describe('watchdog', () => {
   afterEach(() => {
     jest.useRealTimers()

package/test/io.close.test.js

@@ -93,6 +93,27 @@ describe('close', () => {
     await expect(io.close()).resolves.not.toThrow()
   })
 
+  // a connection is shared and outlives its IOs, and it has a limited number of channels
+  it('should give the channels back', async () => {
+    await reply()
+    await io.close()
+
+    expect(requests.close).toHaveBeenCalled()
+    expect(events.close).toHaveBeenCalled()
+  })
+
+  it('should give them back before letting go of the connection', async () => {
+    await reply()
+
+    connection.close.mockImplementation(async () => {
+      expect(requests.close).toHaveBeenCalled()
+    })
+
+    await io.close()
+
+    expect.assertions(1)
+  })
+
   it('should wait for event processing completion', async () => {
     const promise = /** @type {Promise<void>} */ new Promex()
     const queue = generate()

package/test/io.diagnostics.test.js

@@ -61,7 +61,7 @@ it('should not throw on error without listeners', async () => {
   expect(() => emit(new Error(generate()))).not.toThrow()
 })
 
-it.each(['open', 'close', 'error', 'reconnect'])('should re-emit %s from connection',
+it.each(['open', 'close', 'error', 'reconnect', 'exhausted'])('should re-emit %s from connection',
   /**
    * @param {comq.diagnostics.Event} event
    */

package/types/channel.d.ts

@@ -27,6 +27,10 @@ declare namespace comq {
 
     seal (): Promise<void>
 
+    close (): Promise<void>
+
+    readonly closed: boolean
+
     diagnose (event: _diagnostics.Event, listener: Function): void
 
     forget (event: _diagnostics.Event, listener: Function): void

package/types/diagnostic.d.ts

@@ -1,7 +1,7 @@
 declare namespace comq.diagnostics {
 
-  type Event = 'open' | 'close' | 'error' | 'reconnect' | 'flow' | 'drain' | 'remove' | 'lost' |
-    'recover' | 'discard' | 'pause' | 'resume' | 'return'
+  type Event = 'open' | 'close' | 'error' | 'reconnect' | 'exhausted' | 'flow' | 'drain' |
+    'remove' | 'lost' | 'recover' | 'discard' | 'pause' | 'resume' | 'return'
 
   interface Diagnosable {
     diagnose(event: Event, listener: Function): void

package/types/io.d.ts

@@ -65,6 +65,8 @@ declare namespace comq {
 
     diagnose(event: 'reconnect', listener: (index?: number) => void): void
 
+    diagnose(event: 'exhausted', listener: (limit?: number, index?: number) => void): void
+
     diagnose(event: 'flow', listener: (channel: _topology.type, index?: number) => void): void
 
     diagnose(event: 'drain', listener: (channel: _topology.type, index?: number) => void): void