comq 0.15.8 → 0.16.0

package/.releaserc

@@ -3,8 +3,24 @@
     "release"
   ],
   "plugins": [
-    "@semantic-release/commit-analyzer",
-    "@semantic-release/release-notes-generator",
+    [
+      "@semantic-release/commit-analyzer",
+      {
+        "preset": "conventionalcommits",
+        "releaseRules": [
+          {
+            "breaking": true,
+            "release": "minor"
+          }
+        ]
+      }
+    ],
+    [
+      "@semantic-release/release-notes-generator",
+      {
+        "preset": "conventionalcommits"
+      }
+    ],
     [
       "@semantic-release/npm"
     ],

package/features/channels.feature

@@ -0,0 +1,14 @@
+Feature: Channel exhaustion
+
+  A connection allocates channels from a range the broker negotiates, and once the range
+  is spent there is nothing a reconnection can do about it. The refusal is reported and
+  passed on rather than retried, so that whoever asked for a channel is told instead of
+  waiting for a recovery that is not coming.
+
+  Scenario: Reporting a connection with no channel left to allocate
+    Given the AMQP channel limit is set to 2
+    And an active connection to the broker
+    And a producer replying `echo` queue
+    When emitting an event to the `exhaustion` exchange is attempted
+    Then the exception is thrown: "No channels left to allocate"
+    And the `exhausted` event is emitted

package/features/steps/channels.js

@@ -0,0 +1,45 @@
+'use strict'
+
+const assert = require('node:assert')
+const { randomBytes } = require('node:crypto')
+
+const { Given, When, Then } = require('@cucumber/cucumber')
+
+Given('the AMQP channel limit is set to {number}',
+  /**
+   * @param {number} limit
+   */
+  function (limit) {
+    global.COMQ_TESTING_AMQP_CHANNEL_MAX = limit
+  })
+
+When('emitting an event to the {token} exchange is attempted',
+  /**
+   * @param {string} exchange
+   * @this {comq.features.Context}
+   */
+  async function (exchange) {
+    this.exception = undefined
+
+    await this.io.emit(exchange, randomBytes(8))
+      .catch((exception) => { this.exception = exception })
+  })
+
+Then('the exception is thrown: {string}',
+  /**
+   * @param {string} message
+   * @this {comq.features.Context}
+   */
+  function (message) {
+    assert.notEqual(this.exception, undefined, 'no exception was thrown')
+    assert.equal(this.exception.message, message)
+  })
+
+Then('the {token} event is emitted',
+  /**
+   * @param {comq.diagnostics.Event} event
+   * @this {comq.features.Context}
+   */
+  function (event) {
+    assert.equal(this.events[event], true, `the '${event}' event was not emitted`)
+  })

package/features/steps/context.js

@@ -100,17 +100,27 @@ class Context extends World {
   #url (i, user, password) {
     const address = this.networks[i]?.address ?? getAddress(i)
     const url = PROTOCOL + user + ':' + password + '@' + address
-    const heartbeat = global.COMQ_TESTING_AMQP_HEARTBEAT
+    const query = []
 
     // the watchdog measures silence, so it may only be shortened along with the
     // interval at which a healthy broker is expected to say something
-    return heartbeat === undefined ? url : url + '?heartbeat=' + heartbeat
+    if (global.COMQ_TESTING_AMQP_HEARTBEAT !== undefined) {
+      query.push('heartbeat=' + global.COMQ_TESTING_AMQP_HEARTBEAT)
+    }
+
+    // a broker negotiates the channel limit down to what the client asks for,
+    // which is how exhaustion is reached without opening two thousand channels
+    if (global.COMQ_TESTING_AMQP_CHANNEL_MAX !== undefined) {
+      query.push('channelMax=' + global.COMQ_TESTING_AMQP_CHANNEL_MAX)
+    }
+
+    return query.length === 0 ? url : url + '?' + query.join('&')
   }
 }
 
 const PROTOCOL = 'amqp://'
 
 /** @type {comq.diagnostics.Event[]} */
-const EVENTS = ['open', 'close', 'flow', 'discard', 'pause', 'resume']
+const EVENTS = ['open', 'close', 'flow', 'discard', 'pause', 'resume', 'exhausted']
 
 exports.Context = Context

package/features/steps/hooks.js

@@ -14,6 +14,7 @@ After(
   async function () {
     delete global.COMQ_TESTING_WATCHDOG_INTERVAL
     delete global.COMQ_TESTING_AMQP_HEARTBEAT
+    delete global.COMQ_TESTING_AMQP_CHANNEL_MAX
 
     await Promise.all(
       Array.from({ length: BROKERS_AMOUNT }, (_, n) => actions.up(n))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.15.8",
+  "version": "0.16.0",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",
@@ -33,15 +33,17 @@
     "@semantic-release/github": "^12.0.2",
     "@semantic-release/npm": "^13.1.2",
     "@semantic-release/release-notes-generator": "^14.1.0",
-    "@testcontainers/rabbitmq": "^12.0.4",
+    "@testcontainers/rabbitmq": "^12.1.0",
     "@types/amqplib": "^0.10.8",
     "@types/jest": "^30.0.0",
+    "conventional-changelog-conventionalcommits": "^8.0.0",
     "fast-glob": "^3.3.3",
-    "fuzzysort": "3.1.0",
+    "fuzzysort": "4.0.2",
     "husky": "^9.1.7",
     "jest": "^30.4.2",
-    "js-yaml": "^5.2.3",
+    "js-yaml": "^5.4.1",
     "randomstring": "^1.3.1",
+    "semantic-release": "^25.0.9",
     "snazzy": "^9.0.0",
     "standard": "^17.1.2",
     "testcontainers": "^12.0.4"

package/readme.md

@@ -498,6 +498,9 @@ Subscribe to one of the diagnostic events:
 - `error`: an attempt to restore the connection has failed. The exception is passed as an argument.
   Attempts continue until one succeeds.
 - `reconnect`: an attempt to restore the connection has started. Followed by `open` or `error`.
+- `exhausted`: the connection has no channel left to allocate. The negotiated channel limit is
+  passed as an argument. Reconnecting does not free a channel, so the operation that asked for
+  one is rejected rather than retried.
 - `flow`: back pressure is applied to a channel. [Channel type](./types/topology.d.ts) is passed as
   an argument.
 - `drain`: back pressure is removed from a channel. Channel type is passed.
@@ -533,6 +536,7 @@ established, there is no way to capture the initial `open` event.
 io.diagnose('flow', (type) => console.log(`Back pressure was applied to the ${type} channel`))
 
 io.diagnose('reconnect', (shard) => console.log('AMQP reconnecting', { shard }))
+io.diagnose('exhausted', (limit, shard) => console.log('AMQP channels exhausted', { limit, shard }))
 io.diagnose('open', (shard) => console.log('AMQP connection established', { shard }))
 io.diagnose('close', (error, shard) => console.log('AMQP connection closed', { message: error?.message, shard }))
 io.diagnose('error', (error, shard) => console.log('AMQP connection failed', { message: error.message, shard }))

package/source/channel.js

@@ -42,6 +42,8 @@ class Channel {
 
   #diagnostics = emitter.create()
 
+  #closed = false
+
   /**
    * @param {comq.amqp.Connection} connection
    * @param {comq.Topology} topology
@@ -126,6 +128,21 @@ class Channel {
     }
   }
 
+  /**
+   * Gives the channel back. A connection has a limited number of them, and one whose
+   * IO is done with it would otherwise be held until the whole connection goes.
+   */
+  async close () {
+    if (this.#closed) return
+
+    this.#closed = true
+
+    await this.seal()
+
+    // a channel that went down with its connection is the outcome this asks for
+    await this.#channel?.close().catch(noop)
+  }
+
   async seal () {
     this.#sealed = true
 
@@ -142,6 +159,10 @@ class Channel {
     this.#diagnostics.off(event, listener)
   }
 
+  get closed () {
+    return this.#closed
+  }
+
   async recover (connection) {
     this.#connection = connection
 

package/source/connection.js

@@ -99,6 +99,7 @@ class Connection {
       const topology = presets[type]
       const channel = await channels.create(this.#connection, topology, index)
 
+      this.#channels = this.#channels.filter((channel) => !channel.closed)
       this.#channels.push(channel)
 
       return channel
@@ -140,6 +141,8 @@ class Connection {
     this.#diagnostics.emit('open')
 
     try {
+      this.#channels = this.#channels.filter((channel) => !channel.closed)
+
       for (const channel of this.#channels) await channel.recover(connection)
     } catch (exception) {
       this.#diagnostics.emit('error', exception)
@@ -207,7 +210,20 @@ class Connection {
     connection.connection?.stream?.destroy(silence())
   }
 
-  #recover () {
+  /**
+   * @param {Error} [exception]
+   * @return {Promise<void> | false}
+   */
+  #recover (exception) {
+    // a connection that has no channel left to give is not something reconnecting
+    // fixes, and `#recovery` on a connection that is perfectly well is a promise
+    // nobody ever resolves — returning `false` lets the caller see the refusal
+    if (exception?.message === EXHAUSTED) {
+      this.#diagnostics.emit('exhausted', this.#connection?.connection?.channelMax)
+
+      return false
+    }
+
     return this.#recovery
   }
 
@@ -333,6 +349,9 @@ const SOCKET_OPTIONS = {
 
 const HEARTBEAT_SET = /[?&]heartbeat=/
 
+/** What amqplib says when the negotiated channel limit leaves no identifier free. */
+const EXHAUSTED = 'No channels left to allocate'
+
 const TRANSIENT_CODES = new Set([
   'ECONNREFUSED',
   'EAI_AGAIN',

package/source/events.js

@@ -1,7 +1,7 @@
 'use strict'
 
 /** @type {comq.diagnostics.Event[]} */
-exports.connection = ['open', 'close', 'error', 'reconnect']
+exports.connection = ['open', 'close', 'error', 'reconnect', 'exhausted']
 
 /** @type {comq.diagnostics.Event[]} */
 exports.channel = ['flow', 'drain', 'recover', 'discard', 'pause', 'resume', 'return', 'lost', 'remove']

package/source/io.js

@@ -163,6 +163,12 @@ class IO {
     await this.seal()
     await this.#destroyStreams(this.#replyPipes)
     await track(this)
+
+    // a connection is shared and outlives its IOs, so the channels are given back
+    // here — held to the end of the connection, they would run it out of them
+    await Promise.all([this.#requests, this.#replies, this.#events]
+      .map((channel) => channel?.close()))
+
     await this.#connection.close()
   })
 

package/source/shards/channel.js

@@ -77,6 +77,14 @@ class Channel {
     return await this.#one((channel) => channel.fire(queue, buffer, options))
   }
 
+  async close () {
+    await this.#all((channel) => channel.close())
+  }
+
+  get closed () {
+    return [...this.#channels].every((channel) => channel.closed)
+  }
+
   async seal () {
     await this.#all((channel) => channel.seal())
   }

package/test/amqplib.mock.js

@@ -15,6 +15,7 @@ class Channel extends EventEmitter {
   assertExchange = jest.fn(async () => undefined)
   bindQueue = jest.fn(async () => undefined)
   publish = jest.fn((_0, _1, _2, _3, resolve) => resolve?.(null))
+  close = jest.fn(async () => undefined)
 }
 
 class Connection extends EventEmitter {
@@ -37,7 +38,10 @@ class Connection extends EventEmitter {
     // a broker accepts the heartbeat the client asks for
     const heartbeat = Number(/[?&]heartbeat=(\d+)/.exec(url)?.[1] ?? 60)
 
-    this.connection = { stream, heartbeat }
+    // and negotiates the channel limit down to whatever the client asked for
+    const channelMax = Number(/[?&]channelMax=(\d+)/.exec(url)?.[1] ?? 2047)
+
+    this.connection = { stream, heartbeat, channelMax }
 
     // noinspection JSValidateTypes
     this.removeAllListeners = jest.spyOn(this, 'removeAllListeners')

package/test/channel.test.js

@@ -503,6 +503,49 @@ describe('seal', () => {
   })
 })
 
+describe('close', () => {
+  const queue = generate()
+  const consumer = jest.fn()
+
+  beforeEach(async () => {
+    channel = await create(connection, topology)
+    chan = await getCreatedChannel()
+  })
+
+  it('should give the channel back', async () => {
+    await channel.close()
+
+    expect(chan.close).toHaveBeenCalled()
+    expect(channel.closed).toStrictEqual(true)
+  })
+
+  it('should stop consuming before giving it back', async () => {
+    await channel.consume(queue, consumer)
+
+    const { consumerTag: tag } = await chan.consume.mock.results[0].value
+
+    await channel.close()
+
+    expect(chan.cancel).toHaveBeenCalledWith(tag)
+    expect(chan.cancel.mock.invocationCallOrder[0])
+      .toBeLessThan(chan.close.mock.invocationCallOrder[0])
+  })
+
+  it('should give it back once', async () => {
+    await channel.close()
+    await channel.close()
+
+    expect(chan.close).toHaveBeenCalledTimes(1)
+  })
+
+  // one that went down with its connection is the outcome this asks for
+  it('should ignore one that is already gone', async () => {
+    chan.close.mockImplementationOnce(async () => { throw new Error() })
+
+    await expect(channel.close()).resolves.not.toThrow()
+  })
+})
+
 describe('back pressure', () => {
   const exchange = generate()
   const queue = generate()

package/test/connection.mock.js

@@ -12,6 +12,8 @@ const channel = /** @type {jest.MockedFunction<(sharded?: boolean, index?: numbe
     publish: jest.fn(async () => undefined),
     diagnose: jest.fn(async () => undefined),
     seal: jest.fn(async () => undefined),
+    close: jest.fn(async () => undefined),
+    closed: false,
     recover: jest.fn(async () => undefined)
   }))
 

package/test/connection.test.js

@@ -398,6 +398,106 @@ describe('create channel', () => {
   })
 })
 
+describe('closed channels', () => {
+  /** @type {jest.MockedObject<comq.amqp.Connection>} */
+  let conn
+
+  beforeEach(async () => {
+    // an earlier case leaves an implementation of its own behind
+    create.mockImplementation(async () => ({ recover: jest.fn(async () => undefined), closed: false }))
+
+    await connection.open()
+
+    conn = await amqplib.connect.mock.results[0].value
+  })
+
+  afterEach(async () => {
+    await connection.close()
+  })
+
+  // a channel given back is not one this connection has any more
+  it('should not recover one that was given back', async () => {
+    const channel = await connection.createChannel('event')
+
+    channel.closed = true
+
+    conn.emit('close', new Error())
+
+    await timeout(50)
+
+    expect(channel.recover).not.toHaveBeenCalled()
+  })
+
+  it('should recover one that is still in use', async () => {
+    const channel = await connection.createChannel('event')
+
+    conn.emit('close', new Error())
+
+    await timeout(50)
+
+    expect(channel.recover).toHaveBeenCalled()
+  })
+})
+
+describe('channel exhaustion', () => {
+  const EXHAUSTED = 'No channels left to allocate'
+
+  /** @type {jest.MockedObject<comq.amqp.Connection>} */
+  let conn
+
+  beforeEach(async () => {
+    await connection.open()
+
+    conn = await amqplib.connect.mock.results[0].value
+  })
+
+  it('should reject rather than wait for a recovery that is not coming', async () => {
+    create.mockImplementation(async () => { throw new Error(EXHAUSTED) })
+
+    // the regression is a hang, so the assertion is that it settles at all
+    const settled = await Promise.race([
+      connection.createChannel('event').then(() => 'resolved', (e) => e.message),
+      timeout(100).then(() => 'pending')
+    ])
+
+    expect(settled).toStrictEqual(EXHAUSTED)
+  })
+
+  it('should not retry', async () => {
+    create.mockImplementation(async () => { throw new Error(EXHAUSTED) })
+
+    await connection.createChannel('event').catch(() => undefined)
+
+    expect(create).toHaveBeenCalledTimes(1)
+  })
+
+  it('should report the limit the broker negotiated', async () => {
+    const listener = jest.fn()
+
+    await connection.diagnose('exhausted', listener)
+
+    create.mockImplementation(async () => { throw new Error(EXHAUSTED) })
+
+    await connection.createChannel('event').catch(() => undefined)
+
+    expect(listener).toHaveBeenCalledWith(conn.connection.channelMax)
+  })
+
+  it('should leave every other failure its recovery', async () => {
+    create.mockImplementation(async () => { throw new Error() })
+
+    setTimeout(() => {
+      create.mockImplementation(async () => generate())
+
+      conn.emit('close', new Error())
+    }, 1)
+
+    const channel = await connection.createChannel('request')
+
+    expect(channel).toStrictEqual(await create.mock.results[1].value)
+  })
+})
+
 describe('watchdog', () => {
   afterEach(() => {
     jest.useRealTimers()

package/test/io.close.test.js

@@ -93,6 +93,27 @@ describe('close', () => {
     await expect(io.close()).resolves.not.toThrow()
   })
 
+  // a connection is shared and outlives its IOs, and it has a limited number of channels
+  it('should give the channels back', async () => {
+    await reply()
+    await io.close()
+
+    expect(requests.close).toHaveBeenCalled()
+    expect(events.close).toHaveBeenCalled()
+  })
+
+  it('should give them back before letting go of the connection', async () => {
+    await reply()
+
+    connection.close.mockImplementation(async () => {
+      expect(requests.close).toHaveBeenCalled()
+    })
+
+    await io.close()
+
+    expect.assertions(1)
+  })
+
   it('should wait for event processing completion', async () => {
     const promise = /** @type {Promise<void>} */ new Promex()
     const queue = generate()

package/test/io.diagnostics.test.js

@@ -61,7 +61,7 @@ it('should not throw on error without listeners', async () => {
   expect(() => emit(new Error(generate()))).not.toThrow()
 })
 
-it.each(['open', 'close', 'error', 'reconnect'])('should re-emit %s from connection',
+it.each(['open', 'close', 'error', 'reconnect', 'exhausted'])('should re-emit %s from connection',
   /**
    * @param {comq.diagnostics.Event} event
    */

package/types/channel.d.ts

@@ -27,6 +27,10 @@ declare namespace comq {
 
     seal (): Promise<void>
 
+    close (): Promise<void>
+
+    readonly closed: boolean
+
     diagnose (event: _diagnostics.Event, listener: Function): void
 
     forget (event: _diagnostics.Event, listener: Function): void

package/types/diagnostic.d.ts

@@ -1,7 +1,7 @@
 declare namespace comq.diagnostics {
 
-  type Event = 'open' | 'close' | 'error' | 'reconnect' | 'flow' | 'drain' | 'remove' | 'lost' |
-    'recover' | 'discard' | 'pause' | 'resume' | 'return'
+  type Event = 'open' | 'close' | 'error' | 'reconnect' | 'exhausted' | 'flow' | 'drain' |
+    'remove' | 'lost' | 'recover' | 'discard' | 'pause' | 'resume' | 'return'
 
   interface Diagnosable {
     diagnose(event: Event, listener: Function): void

package/types/io.d.ts

@@ -65,6 +65,8 @@ declare namespace comq {
 
     diagnose(event: 'reconnect', listener: (index?: number) => void): void
 
+    diagnose(event: 'exhausted', listener: (limit?: number, index?: number) => void): void
+
     diagnose(event: 'flow', listener: (channel: _topology.type, index?: number) => void): void
 
     diagnose(event: 'drain', listener: (channel: _topology.type, index?: number) => void): void