comq 0.15.7 → 0.16.0

package/.releaserc

@@ -3,12 +3,22 @@
     "release"
   ],
   "plugins": [
-    "@semantic-release/commit-analyzer",
-    "@semantic-release/release-notes-generator",
     [
-      "@semantic-release/changelog",
+      "@semantic-release/commit-analyzer",
       {
-        "changelogFile": "CHANGELOG.md"
+        "preset": "conventionalcommits",
+        "releaseRules": [
+          {
+            "breaking": true,
+            "release": "minor"
+          }
+        ]
+      }
+    ],
+    [
+      "@semantic-release/release-notes-generator",
+      {
+        "preset": "conventionalcommits"
       }
     ],
     [
@@ -19,8 +29,7 @@
       {
         "assets": [
           "package.json",
-          "package-lock.json",
-          "CHANGELOG.md"
+          "package-lock.json"
         ],
         "message": "chore(release): ${nextRelease.version}\n\n${nextRelease.notes}"
       }

package/features/channels.feature

@@ -0,0 +1,14 @@
+Feature: Channel exhaustion
+
+  A connection allocates channels from a range the broker negotiates, and once the range
+  is spent there is nothing a reconnection can do about it. The refusal is reported and
+  passed on rather than retried, so that whoever asked for a channel is told instead of
+  waiting for a recovery that is not coming.
+
+  Scenario: Reporting a connection with no channel left to allocate
+    Given the AMQP channel limit is set to 2
+    And an active connection to the broker
+    And a producer replying `echo` queue
+    When emitting an event to the `exhaustion` exchange is attempted
+    Then the exception is thrown: "No channels left to allocate"
+    And the `exhausted` event is emitted

package/features/recovery.feature

@@ -84,6 +84,36 @@ Feature: Request-Reply Topology Recovery
     5
     """
 
+  Scenario: Sending another request while the broker is down
+
+  The reply queue is already being consumed from, so the request is published as soon as the
+  request channel recovers, which is before the reply queue has been declared again
+
+    Given function replying `add_numbers` queue:
+    """
+    ({ a, b }) => { return a + b }
+    """
+    When the consumer sends the following request to the `add_numbers` queue:
+    """yaml
+    a: 1
+    b: 2
+    """
+    Then the consumer receives the reply:
+    """yaml
+    3
+    """
+    When the broker has crashed
+    And the consumer sends the following request to the `add_numbers` queue:
+    """yaml
+    a: 2
+    b: 3
+    """
+    Then the broker is up
+    And the consumer receives the reply:
+    """yaml
+    5
+    """
+
   Scenario: Defining emitter while the broker is down
     Given the broker is down
     And `logger` consuming events from the `numbers_added` exchange is expected

package/features/silence.feature

@@ -0,0 +1,46 @@
+Feature: Silent Connection Tolerance
+
+  A connection that has gone silent is what a machine that woke from sleep is
+  left with: the socket is open, neither the broker nor the operating system
+  reports anything, and nothing but the watchdog tells it from an idle connection.
+
+  Background:
+    Given watchdog interval is set to 2000ms with 60s AMQP heartbeat
+    And a network that can go silent
+
+  Scenario: Restoring a connection that has gone silent
+    Given an active connection to the broker
+    And a producer replying `echo` queue
+    When the network goes silent
+    Then the connection is lost within 10 seconds
+    When the consumer sends a request to the `echo` queue
+    Then the consumer receives the reply
+
+  Scenario: Restoring a sharded connection whose every shard has gone silent
+    Given an active sharded connection
+    And a producer replying `echo` queue
+    When the network goes silent
+    Then the connection is lost within 10 seconds
+    When the consumer sends a request to the `echo` queue
+    Then the consumer receives the reply
+
+  Scenario: Answering a request sent as the connection goes silent
+    Given an active connection to the broker
+    And a producer replying `echo` queue
+    When the consumer sends a request to the `echo` queue as the network goes silent
+    Then the consumer receives the reply
+    And the connection is lost within 10 seconds
+
+  Scenario: Answering a request sent as every shard goes silent
+    Given an active sharded connection
+    And a producer replying `echo` queue
+    When the consumer sends a request to the `echo` queue as the network goes silent
+    Then the consumer receives the reply
+    And the connection is lost within 10 seconds
+
+  Scenario: Reporting that there is nowhere left to publish
+    Given an active sharded connection
+    And a producer replying `echo` queue
+    When the network goes silent
+    Then publishing is paused within 10 seconds
+    And publishing is resumed within 10 seconds

package/features/steps/.types/context.d.ts

@@ -1,4 +1,5 @@
 import { Readable } from 'node:stream'
+import { Socket } from 'node:net'
 import * as _diagnostics from '../../../types/diagnostic'
 import * as _io from '../../../types/io'
 import * as _amqp from '../../../types/amqp'
@@ -34,10 +35,26 @@ declare namespace comq.features {
     streamsValues: Record<number, any[]>
     streamsEnded: Record<number, boolean>
     generatorDestroyed: boolean
+    networks: Network[]
 
     connect(user?: string, password?: string): Promise<void>
     assert(user?: string, password?: string): Promise<void>
     disconnect(): Promise<void>
+    unplug(): Promise<void>
+  }
+
+  interface Network {
+    readonly address: string
+
+    open(): Promise<void>
+    silence(): void
+    close(): Promise<void>
+  }
+
+  interface Tunnel {
+    client: Socket
+    upstream: Socket
+    silent: boolean
   }
 
 }

package/features/steps/backpressure.js

@@ -2,7 +2,7 @@
 
 const assert = require('node:assert')
 const { randomBytes } = require('node:crypto')
-const { quantity, timeout } = require('@toa.io/generic')
+const { quantity, timeout } = require('../../test/helpers')
 
 const { When } = require('@cucumber/cucumber')
 

package/features/steps/broker.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { random } = require('@toa.io/generic')
+const { random } = require('../../test/helpers')
 
 const { Given } = require('@cucumber/cucumber')
 const { BROKERS_AMOUNT, actions } = require('./brokers')

package/features/steps/brokers.js

@@ -2,7 +2,7 @@
 
 const { execFile } = require('node:child_process')
 const { promisify } = require('node:util')
-const { timeout } = require('@toa.io/generic')
+const { timeout } = require('../../test/helpers')
 const { RabbitMQContainer } = require('@testcontainers/rabbitmq')
 const { getContainerRuntimeClient } = require('testcontainers')
 

package/features/steps/channels.js

@@ -0,0 +1,45 @@
+'use strict'
+
+const assert = require('node:assert')
+const { randomBytes } = require('node:crypto')
+
+const { Given, When, Then } = require('@cucumber/cucumber')
+
+Given('the AMQP channel limit is set to {number}',
+  /**
+   * @param {number} limit
+   */
+  function (limit) {
+    global.COMQ_TESTING_AMQP_CHANNEL_MAX = limit
+  })
+
+When('emitting an event to the {token} exchange is attempted',
+  /**
+   * @param {string} exchange
+   * @this {comq.features.Context}
+   */
+  async function (exchange) {
+    this.exception = undefined
+
+    await this.io.emit(exchange, randomBytes(8))
+      .catch((exception) => { this.exception = exception })
+  })
+
+Then('the exception is thrown: {string}',
+  /**
+   * @param {string} message
+   * @this {comq.features.Context}
+   */
+  function (message) {
+    assert.notEqual(this.exception, undefined, 'no exception was thrown')
+    assert.equal(this.exception.message, message)
+  })
+
+Then('the {token} event is emitted',
+  /**
+   * @param {comq.diagnostics.Event} event
+   * @this {comq.features.Context}
+   */
+  function (event) {
+    assert.equal(this.events[event], true, `the '${event}' event was not emitted`)
+  })

package/features/steps/common.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const { When } = require('@cucumber/cucumber')
-const { timeout } = require('@toa.io/generic')
+const { timeout } = require('../../test/helpers')
 
 When('after {number}ms',
   /**

package/features/steps/connection.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const assert = require('node:assert')
-const { timeout } = require('@toa.io/generic')
+const { timeout } = require('../../test/helpers')
 
 const { Given, When, Then } = require('@cucumber/cucumber')
 
@@ -167,6 +167,27 @@ Then('the connection is {connection-event}',
     assert.equal(this.events[event], true, 'connection was not ' + key)
   })
 
+Then('the connection is lost within {number} second(s)',
+  /**
+   * @param {number} seconds
+   * @this {comq.features.Context}
+   */
+  async function (seconds) {
+    await emitted.call(this, 'close', seconds, 'connection was not lost')
+  })
+
+Then('publishing is {paused-event} within {number} second(s)',
+  /**
+   * @param {'paused' | 'resumed'} key
+   * @param {number} seconds
+   * @this {comq.features.Context}
+   */
+  async function (key, seconds) {
+    const event = PUBLISHING_EVENTS[key]
+
+    await emitted.call(this, event, seconds, 'publishing was not ' + key)
+  })
+
 Given('the connection has started sealing',
   /**
    * @this {comq.features.Context}
@@ -197,6 +218,25 @@ const connect = async (context, user, password) => {
   }
 }
 
+/**
+ * @param {comq.diagnostics.Event} event
+ * @param {number} seconds
+ * @param {string} message
+ * @this {comq.features.Context}
+ */
+async function emitted (event, seconds, message) {
+  const deadline = Date.now() + seconds * 1000
+
+  while (this.events[event] !== true && Date.now() < deadline) await timeout(50)
+
+  assert.equal(this.events[event], true, message)
+}
+
+const PUBLISHING_EVENTS = {
+  paused: 'pause',
+  resumed: 'resume'
+}
+
 const CONNECTION_EVENTS = {
   lost: 'close',
   restored: 'open'

package/features/steps/context.js

@@ -34,6 +34,9 @@ class Context extends World {
   streamsEnded = {}
   generatorDestroyed = false
 
+  /** @type {comq.features.Network[]} */
+  networks = []
+
   async connect (user, password) {
     const urls = this.#urls(user, password)
 
@@ -46,6 +49,12 @@ class Context extends World {
     await this.#connect(urls, assert)
   }
 
+  async unplug () {
+    await Promise.all(this.networks.map((network) => network.close()))
+
+    this.networks = []
+  }
+
   async disconnect () {
     if (this.io === undefined) return
 
@@ -89,18 +98,29 @@ class Context extends World {
   }
 
   #url (i, user, password) {
-    const url = PROTOCOL + user + ':' + password + '@' + getAddress(i)
-    const heartbeat = global.COMQ_TESTING_AMQP_HEARTBEAT
+    const address = this.networks[i]?.address ?? getAddress(i)
+    const url = PROTOCOL + user + ':' + password + '@' + address
+    const query = []
 
     // the watchdog measures silence, so it may only be shortened along with the
     // interval at which a healthy broker is expected to say something
-    return heartbeat === undefined ? url : url + '?heartbeat=' + heartbeat
+    if (global.COMQ_TESTING_AMQP_HEARTBEAT !== undefined) {
+      query.push('heartbeat=' + global.COMQ_TESTING_AMQP_HEARTBEAT)
+    }
+
+    // a broker negotiates the channel limit down to what the client asks for,
+    // which is how exhaustion is reached without opening two thousand channels
+    if (global.COMQ_TESTING_AMQP_CHANNEL_MAX !== undefined) {
+      query.push('channelMax=' + global.COMQ_TESTING_AMQP_CHANNEL_MAX)
+    }
+
+    return query.length === 0 ? url : url + '?' + query.join('&')
   }
 }
 
 const PROTOCOL = 'amqp://'
 
 /** @type {comq.diagnostics.Event[]} */
-const EVENTS = ['open', 'close', 'flow', 'discard']
+const EVENTS = ['open', 'close', 'flow', 'discard', 'pause', 'resume', 'exhausted']
 
 exports.Context = Context

package/features/steps/events.js

@@ -3,7 +3,7 @@
 const stream = require('node:stream')
 const assert = require('node:assert')
 const { randomBytes } = require('node:crypto')
-const { timeout, quantity, match } = require('@toa.io/generic')
+const { timeout, quantity, match } = require('../../test/helpers')
 const { parse } = require('./yaml')
 const { Given, When, Then } = require('@cucumber/cucumber')
 

package/features/steps/hooks.js

@@ -14,12 +14,14 @@ After(
   async function () {
     delete global.COMQ_TESTING_WATCHDOG_INTERVAL
     delete global.COMQ_TESTING_AMQP_HEARTBEAT
+    delete global.COMQ_TESTING_AMQP_CHANNEL_MAX
 
     await Promise.all(
       Array.from({ length: BROKERS_AMOUNT }, (_, n) => actions.up(n))
     )
 
     await this.disconnect()
+    await this.unplug()
   })
 
 AfterAll(async function () {

package/features/steps/network.js

@@ -0,0 +1,50 @@
+'use strict'
+
+const { Given, When } = require('@cucumber/cucumber')
+const { BROKERS_AMOUNT } = require('./brokers')
+const { Network } = require('./networks')
+
+Given('a network that can go silent',
+  /**
+   * @this {comq.features.Context}
+   */
+  async function () {
+    for (let n = 0; n < BROKERS_AMOUNT; n++) {
+      const network = new Network(n)
+
+      await network.open()
+
+      this.networks[n] = network
+    }
+  })
+
+When('the network goes silent',
+  /**
+   * @this {comq.features.Context}
+   */
+  function () {
+    silence.call(this)
+  })
+
+// a request that is answered over a connection that is already silent proves
+// nothing, hence the window between the two is left as short as a step boundary
+When('the consumer sends a request to the {token} queue as the network goes silent',
+  /**
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  function (queue) {
+    this.reply = this.io.request(queue, 'hello')
+
+    silence.call(this)
+  })
+
+/**
+ * @this {comq.features.Context}
+ */
+function silence () {
+  // only a connection lost from now on counts as lost
+  delete this.events.close
+
+  for (const network of this.networks) network.silence()
+}

package/features/steps/networks.js

@@ -0,0 +1,90 @@
+'use strict'
+
+const net = require('node:net')
+const { getAddress } = require('./brokers')
+
+/**
+ * A network that can go silent, which is what a machine that woke from sleep is
+ * left with: the socket stays open, nothing is delivered over it, and neither
+ * the broker nor the operating system reports anything. Stopping, killing or
+ * pausing a broker does not reproduce it, since all three end up closing the
+ * socket the moment the connection is used.
+ */
+class Network {
+  /** @type {net.Server} */
+  #server
+
+  /** @type {Set<comq.features.Tunnel>} */
+  #tunnels = new Set()
+
+  /** @type {string} */
+  #host
+
+  /** @type {number} */
+  #port
+
+  /**
+   * @param {number} [n] broker index
+   */
+  constructor (n = 0) {
+    const [host, port] = getAddress(n).split(':')
+
+    this.#host = host
+    this.#port = Number(port)
+  }
+
+  get address () {
+    return 'localhost:' + this.#server.address().port
+  }
+
+  async open () {
+    this.#server = net.createServer(this.#tunnel)
+
+    await new Promise((resolve) => this.#server.listen(0, '127.0.0.1', resolve))
+  }
+
+  /**
+   * Only the tunnels that are already open go silent: a connection established
+   * afterwards is forwarded as usual, just as it is once a machine is awake.
+   */
+  silence () {
+    for (const tunnel of this.#tunnels) tunnel.silent = true
+  }
+
+  async close () {
+    for (const tunnel of this.#tunnels) this.#collapse(tunnel)
+
+    await new Promise((resolve) => this.#server.close(resolve))
+  }
+
+  /**
+   * @param {net.Socket} client
+   */
+  #tunnel = (client) => {
+    const upstream = net.connect(this.#port, this.#host)
+
+    /** @type {comq.features.Tunnel} */
+    const tunnel = { client, upstream, silent: false }
+
+    this.#tunnels.add(tunnel)
+
+    client.on('data', (chunk) => { if (!tunnel.silent) upstream.write(chunk) })
+    upstream.on('data', (chunk) => { if (!tunnel.silent) client.write(chunk) })
+
+    for (const socket of [client, upstream]) {
+      socket.on('error', () => this.#collapse(tunnel))
+      socket.on('close', () => this.#collapse(tunnel))
+    }
+  }
+
+  /**
+   * @param {comq.features.Tunnel} tunnel
+   */
+  #collapse (tunnel) {
+    this.#tunnels.delete(tunnel)
+    tunnel.client.destroy()
+    tunnel.upstream.destroy()
+  }
+}
+
+exports.Network = Network

package/features/steps/parameters.js

@@ -38,6 +38,12 @@ defineParameterType({
   transformer: (value) => value
 })
 
+defineParameterType({
+  name: 'paused-event',
+  regexp: /(paused|resumed)/,
+  transformer: (value) => value
+})
+
 defineParameterType({
   name: 'message',
   regexp: /(request|event)/,

package/features/steps/poison.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const assert = require('node:assert')
-const { timeout } = require('@toa.io/generic')
+const { timeout } = require('../../test/helpers')
 const { Then } = require('@cucumber/cucumber')
 
 Then('the message is discarded',

package/features/steps/rpc.js

@@ -4,7 +4,7 @@ const stream = require('node:stream')
 const assert = require('node:assert')
 const { randomBytes } = require('node:crypto')
 const { parse } = require('./yaml')
-const { match, timeout, quantity, random } = require('@toa.io/generic')
+const { match, timeout, quantity, random } = require('../../test/helpers')
 const { Given, When, Then } = require('@cucumber/cucumber')
 const { Readable } = require('node:stream')
 const { actions, BROKERS_AMOUNT } = require('./brokers')

package/features/steps/tasks.js

@@ -2,7 +2,7 @@
 
 const assert = require('node:assert')
 const { Given, Then, When } = require('@cucumber/cucumber')
-const { quantity, timeout } = require('@toa.io/generic')
+const { quantity, timeout } = require('../../test/helpers')
 const { randomBytes } = require('node:crypto')
 const stream = require('node:stream')
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.15.7",
+  "version": "0.16.0",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",
@@ -27,30 +27,29 @@
     "feat": "scripts/feat.js"
   },
   "devDependencies": {
-    "@cucumber/cucumber": "^13.2.0",
-    "@semantic-release/changelog": "^7.0.0",
+    "@cucumber/cucumber": "^13.2.1",
     "@semantic-release/commit-analyzer": "^13.0.1",
-    "@semantic-release/git": "^10.0.1",
+    "@semantic-release/git": "^11.0.1",
     "@semantic-release/github": "^12.0.2",
     "@semantic-release/npm": "^13.1.2",
     "@semantic-release/release-notes-generator": "^14.1.0",
-    "@testcontainers/rabbitmq": "^12.0.4",
-    "@toa.io/generic": "^1.0.0-alpha.208",
+    "@testcontainers/rabbitmq": "^12.1.0",
     "@types/amqplib": "^0.10.8",
     "@types/jest": "^30.0.0",
+    "conventional-changelog-conventionalcommits": "^8.0.0",
     "fast-glob": "^3.3.3",
-    "fuzzysort": "3.1.0",
+    "fuzzysort": "4.0.2",
     "husky": "^9.1.7",
     "jest": "^30.4.2",
-    "js-yaml": "^5.2.3",
+    "js-yaml": "^5.4.1",
     "randomstring": "^1.3.1",
+    "semantic-release": "^25.0.9",
     "snazzy": "^9.0.0",
     "standard": "^17.1.2",
     "testcontainers": "^12.0.4"
   },
   "dependencies": {
     "amqplib": "^2.0.1",
-    "msgpackr": "^2.0.5",
     "promex": "^0.1.1",
     "reretry": "^0.1.0"
   },