comq 0.12.0 → 0.14.0

package/.github/workflows/codeql.yml

@@ -42,11 +42,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@v4
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -60,7 +60,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v3
+      uses: github/codeql-action/autobuild@v4
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -73,6 +73,6 @@ jobs:
     #     ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@v4
       with:
         category: "/language:${{matrix.language}}"

package/.github/workflows/release.yaml

@@ -1,73 +1,43 @@
 name: Release
 
 on:
-  pull_request:
-    types: [closed]
+  push:
     branches:
-      - release
-      - alpha
-    paths:
-      - 'source/**'
-      - 'types/**'
-      - 'package-lock.json'
+      - dev
 
-env:
-  CI: 1
+permissions:
+  contents: write
+  issues: write
+  pull-requests: write
+  id-token: write
 
 jobs:
-  publish:
-    if: github.event.pull_request.merged == true
+  release:
     runs-on: ubuntu-latest
+    env:
+      HUSKY: 0
 
     steps:
-      - name: Setup Node
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-          registry-url: 'https://registry.npmjs.org'
-
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
+          persist-credentials: false
 
-      - name: Run tests
-        run: npm i && npm test
-
-      - name: Import GPG key
-        id: gpg
-        uses: crazy-max/ghaction-import-gpg@v6
+      - name: Setup Node
+        uses: actions/setup-node@v6
         with:
-          gpg_private_key: ${{ secrets.TOABOT_GPG_PRIVATE_KEY }}
-          passphrase: ${{ secrets.TOABOT_GPG_PASSPHRASE }}
-          git_user_signingkey: true
-          git_commit_gpgsign: true
+          node-version: "24"
+          registry-url: https://registry.npmjs.org
+          package-manager-cache: false
 
-      - name: Bump version
-        run: |
-          if [ ${{ github.base_ref }} = release ]; then
-            npx lerna@7.3.0 version --conventional-graduate --no-commit-hooks --no-changelog --yes --message "ci: %s" --sign-git-commit --sign-git-tag
-          else
-            npx lerna@7.3.0 version --conventional-prerelease --preid alpha --dist-tag alpha --no-commit-hooks --no-changelog --yes --message "ci: %s" --git-tag-command "echo %s" --sign-git-commit
-          fi
-          
-          git fetch
-          git checkout dev
-          git merge ${{ github.base_ref }}
-          git push --all origin --no-verify
-        env:
-          GITHUB_TOKEN: ${{ secrets.TOABOT_GITHUB_TOKEN }}
-          GIT_AUTHOR_NAME: ${{ steps.gpg.outputs.name }}
-          GIT_AUTHOR_EMAIL: ${{ steps.gpg.outputs.email }}
-          GIT_COMMITTER_NAME: ${{ steps.gpg.outputs.name }}
-          GIT_COMMITTER_EMAIL: ${{ steps.gpg.outputs.email }}
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run tests
+        run: npm test
 
-      - name: Publish packages
+      - name: Semantic release
         env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
-        run: |
-          if [ ${{ github.base_ref }} = release ]; then
-            npm publish
-          else
-            npm publish --tag alpha
-          fi
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: npx semantic-release

package/.releaserc

@@ -0,0 +1,35 @@
+{
+  "branches": [
+    "dev"
+  ],
+  "plugins": [
+    "@semantic-release/commit-analyzer",
+    "@semantic-release/release-notes-generator",
+    [
+      "@semantic-release/changelog",
+      {
+        "changelogFile": "CHANGELOG.md"
+      }
+    ],
+    [
+      "@semantic-release/npm"
+    ],
+    [
+      "@semantic-release/git",
+      {
+        "assets": [
+          "package.json",
+          "package-lock.json",
+          "CHANGELOG.md"
+        ],
+        "message": "chore(release): ${nextRelease.version}\n\n${nextRelease.notes}"
+      }
+    ],
+    [
+      "@semantic-release/github",
+      {
+        "owner": "toa-io"
+      }
+    ]
+  ]
+}

package/AGENTS.md

@@ -0,0 +1,35 @@
+# AGENTS.md
+
+## Cursor Cloud specific instructions
+
+### Project Overview
+
+ComQ is a Node.js communication library over AMQP (RabbitMQ). It provides RPC, pub/sub, task queues, pipelines, and sharded connections.
+
+### Services
+
+Feature tests start two RabbitMQ instances automatically via Testcontainers (`features/steps/brokers.js`). Credentials: `developer` / `secret`. AMQP ports are assigned dynamically on the host.
+
+### Docker
+
+Feature tests require a running Docker daemon. In nested container environments, start Docker with:
+
+```bash
+sudo dockerd --exec-opt native.cgroupdriver=cgroupfs &
+sudo chmod 666 /var/run/docker.sock
+```
+
+Do not use `docker compose up` for feature tests — Testcontainers manages broker lifecycle. `docker compose` may fail here because `deploy.resources.limits.memory` requires cgroupv2 memory limits that are unavailable in nested setups.
+
+### Commands
+
+- `npm test` — runs `standard` (lint) + `jest` (300 unit tests, no RabbitMQ required)
+- `npm run features` — runs all Cucumber feature tests (starts RabbitMQ via Testcontainers)
+- `npm run lint` — runs `standard --fix --verbose | snazzy`
+
+### Testing Notes
+
+- Unit tests (`jest`) use mocks and do NOT require running RabbitMQ.
+- Feature tests (`cucumber-js`) start brokers in `BeforeAll` and stop them in `AfterAll`.
+- Recovery/connection tests (in `features/connection.feature`, `features/recovery.feature`, `features/shards.feature`) stop/kill broker containers and take 5-15 minutes to complete.
+- To run only fast feature tests (no broker stop/start): `npx cucumber-js --fail-fast -t 'not @manual' features/events.feature features/rpc.feature features/tasks.feature features/singleton.feature features/properties.feature features/tasks.shards.feature`

package/CHANGELOG.md

@@ -0,0 +1,11 @@
+# [0.14.0](https://github.com/toa-io/comq/compare/v0.13.0...v0.14.0) (2026-05-24)
+
+
+### Features
+
+* **features:** manage RabbitMQ brokers with Testcontainers ([1d9ca59](https://github.com/toa-io/comq/commit/1d9ca591ff458c75caf87ab63224b15942ba7d07))
+* **features:** use Testcontainers for RabbitMQ in Gherkin tests ([#219](https://github.com/toa-io/comq/issues/219)) ([a33e66b](https://github.com/toa-io/comq/commit/a33e66bfc357f0f089c728bc60c08f6f24b2c8d1))
+
+# Changelog
+
+All notable changes to this project are documented in this file by [semantic-release](https://github.com/semantic-release/semantic-release).

package/examples/streams/consumer.js

@@ -1,7 +1,7 @@
 import { connect } from 'comq'
 import { url } from '../const.js'
 
-const limit = Number(process.argv[2]) || Math.round(Math.random() * 8 + 2)
+const limit = Number(process.argv[2]) ?? Math.round(Math.random() * 8 + 2)
 
 let io
 let stream
@@ -19,9 +19,9 @@ async function run () {
 }
 
 async function fetch () {
-  console.log(`Fetching stream with ${limit} numbers`)
+  console.log(`Fetching stream with ${limit === 0 ? 'infinite' : limit} numbers`)
 
-  stream = await io.fetch('get_numbers', { limit })
+  stream = await io.request('get_numbers', { limit })
 
   for await (const number of stream) console.log('Received:', number)
 

package/examples/streams/producer.js

@@ -18,6 +18,8 @@ async function run () {
 async function * produce ({ limit }) {
   console.log('Request received with limit:', limit)
 
+  if (limit === 0) limit = Infinity
+
   for (let i = 0; i < limit; i++) {
     yield i
 

package/features/backpressure.feature

@@ -8,5 +8,4 @@ Feature: Back pressure handling
     """
     () => { return Buffer.from('ok') }
     """
-    When I'm sending 100kB requests to the `flood` queue at 5kHz for 0.5 seconds
-    Then back pressure was applied
+    When I'm flooding the `flood` queue until back pressure is applied

package/features/readme.md

@@ -1,4 +1,11 @@
 # Features
 
-To run features, start the RabbitMQ server by running `docker compose up -d` in the package root,
-then execute `npm run features`.
+Feature tests start two RabbitMQ instances automatically via [Testcontainers](https://node.testcontainers.org/modules/rabbitmq/).
+
+Run them with:
+
+```bash
+npm run features
+```
+
+Docker must be available on the host. No manual `docker compose` setup is required.

package/features/shards.feature

@@ -41,7 +41,7 @@ Feature: Sharded Connection
   Scenario: Shard crashes while publishing events
     Given an active sharded connection
     And events are exclusively consumed from the `flood` exchange
-    And I'm publishing 1kB events to the `flood` exchange at 100Hz
+    And I'm publishing 1kB events to the `flood` exchange at 10Hz
     When one of the brokers has crashed
     Then no exceptions are thrown
     And all events have been received

package/features/steps/backpressure.js

@@ -4,43 +4,27 @@ const assert = require('node:assert')
 const { randomBytes } = require('node:crypto')
 const { quantity, timeout } = require('@toa.io/generic')
 
-const { When, Then } = require('@cucumber/cucumber')
+const { When } = require('@cucumber/cucumber')
 
-When('I\'m sending {quantity}B requests to the {token} queue at {quantity}Hz for {number} second(s)',
+const PAYLOAD = quantity('500k')
+const BATCH = 20
+const DEADLINE = 25_000
+
+When('I\'m flooding the {token} queue until back pressure is applied',
   /**
-   * @param {string} bytesQ
    * @param {string} queue
-   * @param {string} frequencyQ
-   * @param {number} seconds
    * @this {comq.features.Context}
    */
-  async function (bytesQ, queue, frequencyQ, seconds) {
-    const bytes = quantity(bytesQ)
-    const frequency = quantity(frequencyQ)
-    const buffer = randomBytes(bytes)
-    const times = seconds * frequency
-
-    // intervals less than 1ms are not accurate
-    const interval = Math.max((1000 / frequency), 1)
-    const each = Math.max(1 / (1000 / frequency), 1)
-
-    const promises = []
+  async function (queue) {
+    const buffer = randomBytes(PAYLOAD)
+    const deadline = Date.now() + DEADLINE
+    const pending = []
 
-    for (let i = 0; i < times; i++) {
-      const promise = this.io.request(queue, buffer)
+    while (!this.events.flow && Date.now() < deadline) {
+      for (let i = 0; i < BATCH; i++) { pending.push(this.io.request(queue, buffer).catch(() => {})) }
 
-      promises.push(promise)
-
-      if ((i + 1) % each === 0) await timeout(interval)
+      await timeout(0)
     }
 
-    await Promise.all(promises)
-  })
-
-Then('back pressure was applied',
-  /**
-   * @this {comq.features.Context}
-   */
-  function () {
     assert.equal(this.events.flow, true, 'Back pressure hasn\'t been applied')
   })

package/features/steps/broker.js

@@ -1,13 +1,13 @@
 'use strict'
 
-const { timeout, random } = require('@toa.io/generic')
-const { execute } = require('@toa.io/command')
+const { random } = require('@toa.io/generic')
 
 const { Given } = require('@cucumber/cucumber')
+const { BROKERS_AMOUNT, actions } = require('./brokers')
 
 Given('the broker is/has {status}',
   /**
-   * @param {'up' | 'down'} status
+   * @param {'up' | 'down' | 'crashed'} status
    * @this {comq.features.Context}
    */
   async function (status) {
@@ -18,7 +18,7 @@ Given('the broker is/has {status}',
 
 Given('one of the brokers is/has {status}',
   /**
-   * @param {'up' | 'down'} status
+   * @param {'up' | 'down' | 'crashed'} status
    * @this {comq.features.Context}
    */
   async function (status) {
@@ -28,29 +28,3 @@ Given('one of the brokers is/has {status}',
 
     await actions[status](shard)
   })
-
-const actions = {
-  up: async (n = 0) => {
-    await execute('docker start comq-rmq-' + n)
-    await healthy(n)
-  },
-  down: async (n = 0) => {
-    await execute('docker stop comq-rmq-' + n)
-  },
-  crashed: async (n = 0) => {
-    await execute('docker kill comq-rmq-' + n)
-  }
-}
-
-async function healthy (n) {
-  let process
-
-  do {
-    await timeout(HEALTHCHECK_INTERVAL)
-
-    process = await execute('docker inspect -f {{.State.Health.Status}} comq-rmq-' + n)
-  } while (process.output !== 'healthy')
-}
-
-const BROKERS_AMOUNT = 2
-const HEALTHCHECK_INTERVAL = global.COMQ_TESTING_HEALTHCHECK_INTERVAL ?? 1000

package/features/steps/brokers.js

@@ -0,0 +1,125 @@
+'use strict'
+
+const { execFile } = require('node:child_process')
+const { promisify } = require('node:util')
+const { timeout } = require('@toa.io/generic')
+const { RabbitMQContainer } = require('@testcontainers/rabbitmq')
+const { getContainerRuntimeClient } = require('testcontainers')
+
+const docker = promisify(execFile)
+
+const IMAGE = 'rabbitmq:3.10.0-management'
+const USER = 'developer'
+const PASSWORD = 'secret'
+const AMQP_PORT = 5672
+const BROKERS_AMOUNT = 2
+const HOST_PORTS = [5673, 5674]
+const HEALTHCHECK_INTERVAL = global.COMQ_TESTING_HEALTHCHECK_INTERVAL ?? 1000
+
+/** @type {import('@testcontainers/rabbitmq').StartedRabbitMQContainer[]} */
+const brokers = []
+
+async function startBrokers () {
+  if (brokers.length > 0) return
+
+  for (let n = 0; n < BROKERS_AMOUNT; n++) await removeStale(`comq-rmq-${n}`)
+
+  const started = await Promise.all(
+    HOST_PORTS.map((hostPort, n) =>
+      new RabbitMQContainer(IMAGE)
+        .withEnvironment({
+          RABBITMQ_DEFAULT_USER: USER,
+          RABBITMQ_DEFAULT_PASS: PASSWORD
+        })
+        .withAutoRemove(false)
+        .withName(`comq-rmq-${n}`)
+        .withExposedPorts({ container: AMQP_PORT, host: hostPort })
+        .withHealthCheck({
+          test: ['CMD', 'rabbitmq-diagnostics', '-q', 'ping'],
+          interval: 5000,
+          timeout: 1000,
+          retries: 5
+        })
+        .start()
+    )
+  )
+
+  brokers.push(...started)
+
+  await Promise.all(
+    Array.from({ length: BROKERS_AMOUNT }, (_, n) => healthy(n))
+  )
+}
+
+async function stopBrokers () {
+  await Promise.all(brokers.map((broker) => broker.stop({ remove: true })))
+  brokers.length = 0
+}
+
+/**
+ * @param {string} name
+ */
+async function removeStale (name) {
+  const client = await getContainerRuntimeClient()
+  const listed = await client.container.list()
+
+  for (const info of listed) {
+    if (!info.Names.some((entry) => entry === `/${name}`)) continue
+
+    const container = client.container.getById(info.Id)
+
+    try {
+      await client.container.stop(container, { timeout: 0 })
+    } catch {
+      // already stopped
+    }
+
+    await container.remove({ force: true })
+  }
+}
+
+/**
+ * @param {number} [n]
+ * @returns {string}
+ */
+function getAddress (n = 0) {
+  if (brokers[n] === undefined) throw new Error(`Broker ${n} is not started`)
+
+  return `localhost:${HOST_PORTS[n]}`
+}
+
+/**
+ * @param {number} [n]
+ */
+async function healthy (n = 0) {
+  do {
+    await timeout(HEALTHCHECK_INTERVAL)
+
+    const { stdout } = await docker('docker', ['inspect', '-f', '{{.State.Health.Status}}', `comq-rmq-${n}`])
+
+    if (stdout.trim() === 'healthy') return
+  } while (true)
+}
+
+const actions = {
+  up: async (n = 0) => {
+    await docker('docker', ['start', `comq-rmq-${n}`])
+    await healthy(n)
+  },
+  down: async (n = 0) => {
+    await docker('docker', ['stop', `comq-rmq-${n}`])
+  },
+  crashed: async (n = 0) => {
+    await docker('docker', ['kill', `comq-rmq-${n}`])
+  }
+}
+
+module.exports = {
+  BROKERS_AMOUNT,
+  USER,
+  PASSWORD,
+  startBrokers,
+  stopBrokers,
+  getAddress,
+  actions
+}

package/features/steps/context.js

@@ -2,6 +2,7 @@
 
 const { World } = require('@cucumber/cucumber')
 const { connect, assert } = require('../../')
+const { getAddress, USER, PASSWORD } = require('./brokers')
 
 /**
  * @implements {comq.features.Context}
@@ -88,14 +89,11 @@ class Context extends World {
   }
 
   #url (i, user, password) {
-    return PROTOCOL + user + ':' + password + '@' + SHARDS[i]
+    return PROTOCOL + user + ':' + password + '@' + getAddress(i)
   }
 }
 
 const PROTOCOL = 'amqp://'
-const SHARDS = ['localhost:5673', 'localhost:5674']
-const USER = 'developer'
-const PASSWORD = 'secret'
 
 /** @type {comq.diagnostics.Event[]} */
 const EVENTS = ['open', 'close', 'flow', 'discard']

package/features/steps/hooks.js

@@ -1,11 +1,24 @@
 'use strict'
 
-const { After } = require('@cucumber/cucumber')
+const { BeforeAll, After, AfterAll } = require('@cucumber/cucumber')
+const { startBrokers, stopBrokers, actions, BROKERS_AMOUNT } = require('./brokers')
+
+BeforeAll({ timeout: 120_000 }, async function () {
+  await startBrokers()
+})
 
 After(
   /**
    * @this {comq.features.Context}
    */
   async function () {
+    await Promise.all(
+      Array.from({ length: BROKERS_AMOUNT }, (_, n) => actions.up(n))
+    )
+
     await this.disconnect()
   })
+
+AfterAll(async function () {
+  await stopBrokers()
+})

package/fixplan.md

@@ -0,0 +1,507 @@
+# comq — reconnection hang fix plan
+
+This document describes the defects identified in the single-connection
+reconnection path (`source/connection.js`, `source/channel.js`,
+`source/attributes/failsafe.js`) and concrete, minimally-invasive fixes for
+each one. Problems are ordered by their likelihood of producing the observed
+symptom: *"after RabbitMQ connection is lost, the application is sometimes
+never recovered."*
+
+---
+
+## Problem 1 — Deadlock in `Channel.recover()` via `recall()` + `failsafe`
+
+### Symptom
+
+During reconnect, `Channel.recover()` rebuilds the AMQP channel and then
+replays previously recorded `consume` / `subscribe` calls via `recall(this)`.
+`#recovery` is resolved only *after* `recall(this)` completes:
+
+```142:160:source/channel.js
+async recover (connection) {
+    this.#connection = connection
+
+    await this.create()
+
+    lazy.reset(this)
+    await recall(this)
+
+    this.#unpause(INTERRUPTION)
+
+    for (const confirmation of this.#confirmations) confirmation.reject(INTERRUPTION)
+
+    // handle interruptions
+    await new Promise(resolve => setTimeout(resolve, 0))
+
+    this.#recovery.resolve()
+    this.#recovery = new Promex()
+    this.#diagnostics.emit('recover')
+  }
+```
+
+The replayed methods are wrapped in `failsafe(this, this.#recover, ...)`. If
+any replayed operation throws a *non-permanent* exception (`Channel closed`,
+`Channel ended, no reply will be forthcoming`, or the internal
+`INTERRUPTION`), `failsafe` calls:
+
+```333:336:source/channel.js
+async #recover (exception) {
+    if (permanent(exception)) return false
+    else await this.#recovery
+  }
+```
+
+`failsafe` now awaits `this.#recovery`, which is the same Promex the enclosing
+`recover()` is supposed to resolve further down. `recall(this)` cannot
+finish, so `#recovery` is never resolved — the channel is stuck, and every
+`send`/`publish`/`consume` parked on `failsafe → #recovery` hangs **forever**.
+
+This reliably happens when the freshly created channel dies mid-setup, which
+is common after a broker restart while heartbeats are still flapping, or
+when `#assertQueue`/`#assertExchange` hits a transient amqplib "Channel
+closed" race.
+
+### Fix
+
+Add a reentrancy guard so that `failsafe.#recover` refuses to wait on
+`#recovery` while `recover()` is itself running — it must propagate instead,
+letting `recall()` surface the failure to `Connection.#open`, which (after
+Problem 2 is fixed) will ask `retry()` for another reopen attempt.
+
+```js
+// source/channel.js
+#recovering = false
+
+async recover (connection) {
+  this.#recovering = true
+  try {
+    this.#connection = connection
+    await this.create()
+    lazy.reset(this)
+    await recall(this)
+    this.#unpause(INTERRUPTION)
+    for (const confirmation of this.#confirmations) confirmation.reject(INTERRUPTION)
+    await new Promise(resolve => setTimeout(resolve, 0))
+    this.#recovery.resolve()
+    this.#recovery = new Promex()
+    this.#diagnostics.emit('recover')
+  } finally {
+    this.#recovering = false
+  }
+}
+
+async #recover (exception) {
+  if (permanent(exception)) return false
+  if (this.#recovering) return false // never wait on #recovery from inside recover()
+  await this.#recovery
+}
+```
+
+Alternative: wrap `recall(this)` in try/catch inside `recover()`, and on
+failure, reject the *old* `#recovery` (with a sentinel), create a fresh one,
+and rethrow. The reentrancy guard is simpler and strictly local.
+
+### Test
+
+Add a unit test in `test/channel.test.js` that:
+
+1. Creates a channel, records a `consume` call.
+2. Triggers `recover()` but makes the mocked `channel.consume` throw a
+   `new Error('Channel closed')` on the replay.
+3. Asserts `recover()` rejects (rather than hanging) within a short timeout.
+
+---
+
+## Problem 2 — `channel.recover()` errors escape the `retry()` loop in `Connection.#open`
+
+### Symptom
+
+Only `amqp.connect` is guarded by try/catch in `#open`; channel recovery
+runs *after* the catch block:
+
+```73:92:source/connection.js
+#open = async (retry) => {
+    try {
+      this.#connection = await amqp.connect(this.#url)
+    } catch (exception) {
+      if (this.#transient(exception)) return retry
+      else throw exception
+    }
+
+    // This prevents the process from crashing; 'close' will be emitted next.
+    // https://amqp-node.github.io/amqplib/channel_api.html#model_events
+    this.#connection.on('error', noop)
+
+    this.#connection.on('close', this.#close)
+    this.#diagnostics.emit('open')
+
+    for (const channel of this.#channels) await channel.recover(this.#connection)
+
+    this.#recovery.resolve()
+    this.#recovery = new Promex()
+  }
+```
+
+If any `channel.recover()` throws (permanent topology error such as
+`PRECONDITION_FAILED`, or the freshly opened connection dies before channel
+creation completes, or the deadlock from Problem 1 is broken by rejection),
+`#open` rejects → `retry()` rethrows → `open()` rejects. And `open()` was
+called from:
+
+```97:103:source/connection.js
+#close = async (error) => {
+    this.#diagnostics.emit('close', error)
+    this.#connection.removeAllListeners()
+    this.#connection = undefined
+
+    if (error !== undefined) await this.open()
+  }
+```
+
+…which has no error handler — rejection becomes an `unhandledRejection`, and
+the Connection stops trying to reconnect. Whether the broken amqplib
+`Connection` eventually emits another `close` to re-arm the flow is not
+guaranteed (TCP half-open, missing heartbeat). The process ends up with a
+live event loop but no broker connectivity.
+
+### Fix
+
+Keep channel recovery inside the `retry()` contract: on any failure, close
+the half-baked connection and ask `retry` to try again.
+
+```js
+#open = async (retry) => {
+  try {
+    this.#connection = await amqp.connect(this.#url)
+  } catch (exception) {
+    if (this.#transient(exception)) return retry
+    else throw exception
+  }
+
+  this.#connection.on('error', noop)
+  this.#connection.on('close', this.#close)
+  this.#diagnostics.emit('open')
+
+  try {
+    for (const channel of this.#channels) {
+      await channel.recover(this.#connection)
+    }
+  } catch (exception) {
+    this.#diagnostics.emit('error', exception)
+    // prevent close re-entry from this broken connection
+    this.#connection.removeAllListeners()
+    try { await this.#connection.close() } catch {}
+    this.#connection = undefined
+    return retry
+  }
+
+  this.#recovery.resolve()
+  this.#recovery = new Promex()
+}
+```
+
+This keeps `retry()` in charge, honours its backoff, and never surfaces an
+unhandled rejection to `#close`.
+
+### Test
+
+In `test/connection.test.js`, mock `channel.recover` to throw once, then
+succeed; assert `amqplib.connect` is called at least twice and `createChannel`
+eventually resolves.
+
+---
+
+## Problem 3 — Unhandled rejection from `#close`
+
+### Symptom
+
+`#close` is an event handler attached to the amqplib `Connection`'s `close`
+event. It `await this.open()` with no surrounding `try/catch`. If `open()`
+rejects (see Problem 2), the rejection becomes an `unhandledRejection`. On
+processes that exit on unhandled rejections, this kills the app; on others,
+it silently leaves the library without a working connection.
+
+### Fix
+
+Detach lifecycle from the event emitter so that the rejection is either
+swallowed on purpose (with diagnostics) or restarted:
+
+```js
+#close = (error) => {
+  this.#diagnostics.emit('close', error)
+  this.#connection.removeAllListeners()
+  this.#connection = undefined
+
+  if (error !== undefined) {
+    this.open().catch((exception) => this.#diagnostics.emit('error', exception))
+  }
+}
+```
+
+Users that observe diagnostics get a signal (the existing `close` diagnostic
+plus a new `error` diagnostic) without the process being destabilised.
+
+### Test
+
+Subscribe to the `error` diagnostic, force `open()` to reject, emit `close`
+with an error on the mocked amqp connection, assert the diagnostic fires and
+the process does not register an unhandled rejection.
+
+---
+
+## Problem 4 — `#transient` is too narrow on the initial connect
+
+### Symptom
+
+```109:114:source/connection.js
+#transient (exception) {
+    const abruptly = exception.message === 'Socket closed abruptly during opening handshake'
+    const tls = exception.message === 'Client network socket disconnected before secure TLS connection was established'
+
+    return this.#running || abruptly || tls
+  }
+```
+
+Before `#running` flips to `true` (i.e. on the *first* `open()`), only two
+exact error messages are considered transient. Everyday transient failures
+(`ECONNREFUSED`, `EAI_AGAIN`, `ENOTFOUND`, `ETIMEDOUT`, `ECONNRESET`) cause
+the boot to throw instead of retrying, forcing process supervisors to do the
+retry. After first connect, `#running` is `true` forever, so *everything*
+becomes transient — the asymmetry between "before first open" and "after
+first open" is surprising.
+
+### Fix
+
+Introduce an explicit whitelist of transient failures that applies in both
+phases:
+
+```js
+#transient (exception) {
+  const transientCodes = new Set([
+    'ECONNREFUSED', 'EAI_AGAIN', 'ENOTFOUND', 'ETIMEDOUT', 'ECONNRESET',
+    'EHOSTUNREACH', 'ENETUNREACH'
+  ])
+  const transientMessages = new Set([
+    'Socket closed abruptly during opening handshake',
+    'Client network socket disconnected before secure TLS connection was established'
+  ])
+
+  if (this.#running) return true
+  if (transientCodes.has(exception.code)) return true
+  if (transientMessages.has(exception.message)) return true
+  return false
+}
+```
+
+Optionally gate the `#running` shortcut behind a maximum attempt count or a
+"permanent" classifier for things like `ACCESS_REFUSED`, so a misconfigured
+credential doesn't retry forever — but that is a separate behaviour change
+and should be opt-in.
+
+### Test
+
+Parametrised test in `test/connection.test.js` for each listed error code,
+asserting `open()` resolves after one failing `amqplib.connect`.
+
+---
+
+## Problem 5 — Reentrant `#close` can run two overlapping `#open` loops
+
+### Symptom
+
+If the new connection dies *while* `#open` is still iterating
+`channel.recover()`, amqplib fires `close` on that connection. The handler
+executes:
+
+```js
+this.#connection.removeAllListeners()
+this.#connection = undefined
+if (error !== undefined) await this.open()
+```
+
+While the outer `#open` is still running and will, on its next loop
+iteration, pass the now-`undefined` `this.#connection` to `channel.recover`
+(crash → see Problem 2). You now have two concurrent `open()` retry loops
+racing; whichever one wins may leave the other with a live amqp connection
+that's unreferenced.
+
+### Fix
+
+Serialise `open()` with a single in-flight promise, and stop reading
+`this.#connection` as a shared variable mid-loop:
+
+```js
+/** @type {Promise<void> | null} */
+#opening = null
+
+async open () {
+  if (this.#opening !== null) return this.#opening
+  this.#opening = retry(this.#open).finally(() => { this.#opening = null })
+  await this.#opening
+  this.#running = true
+}
+
+#open = async (retry) => {
+  let connection
+  try {
+    connection = await amqp.connect(this.#url)
+  } catch (exception) {
+    if (this.#transient(exception)) return retry
+    else throw exception
+  }
+
+  connection.on('error', noop)
+  connection.on('close', this.#close)
+  this.#connection = connection
+  this.#diagnostics.emit('open')
+
+  try {
+    for (const channel of this.#channels) await channel.recover(connection)
+  } catch (exception) {
+    this.#diagnostics.emit('error', exception)
+    connection.removeAllListeners()
+    try { await connection.close() } catch {}
+    if (this.#connection === connection) this.#connection = undefined
+    return retry
+  }
+
+  this.#recovery.resolve()
+  this.#recovery = new Promex()
+}
+```
+
+Two wins:
+
+1. Channel recovery uses a local `connection` reference, so a concurrent
+   `#close` can't swap it out mid-loop.
+2. Concurrent `open()` calls converge on a single retry loop.
+
+### Test
+
+Simulate two `close` events in quick succession (before the first recovery
+finishes) and assert only one `amqp.connect` is *in progress* at any time and
+the final state has exactly one live connection.
+
+---
+
+## Problem 6 — Partial `Channel.recover()` leaves the channel wedged
+
+### Symptom
+
+If `recover()` throws between `await this.create()` and the final
+`#recovery.resolve()`, the Channel has already:
+
+- replaced `this.#channel` with a new one,
+- reset lazy locks,
+- possibly pushed new consumer tags,
+
+but has **not**:
+
+- rejected pending confirmations,
+- unpaused `#paused`,
+- resolved `#recovery`.
+
+Every user operation currently parked on `#recovery` stays parked forever
+unless a subsequent recover succeeds. Combined with Problem 1, this is the
+mechanism by which single failures persist.
+
+### Fix
+
+Make `recover()` structurally transactional — always release waiters on
+failure by rejecting `#recovery` (with a sentinel that `failsafe.#recover`
+classifies as permanent, so callers see a real error instead of hanging):
+
+```js
+async recover (connection) {
+  this.#recovering = true
+  try {
+    this.#connection = connection
+    await this.create()
+    lazy.reset(this)
+    await recall(this)
+    this.#unpause(INTERRUPTION)
+    for (const confirmation of this.#confirmations) confirmation.reject(INTERRUPTION)
+    await new Promise(resolve => setTimeout(resolve, 0))
+    this.#recovery.resolve()
+    this.#recovery = new Promex()
+    this.#diagnostics.emit('recover')
+  } catch (exception) {
+    // release everyone parked on #recovery; they will propagate the error
+    this.#unpause(exception)
+    for (const confirmation of this.#confirmations) confirmation.reject(exception)
+    this.#recovery.reject(exception)
+    this.#recovery = new Promex()
+    throw exception
+  } finally {
+    this.#recovering = false
+  }
+}
+```
+
+Combined with Problem 2's fix, the Connection retry loop will catch this,
+close the broken connection, and retry — instead of wedging callers.
+
+### Test
+
+Force `create()` to throw, call `recover()`, assert that a previously
+`await`-ing `send()` rejects (rather than hanging) with the propagated
+exception.
+
+---
+
+## Problem 7 — No liveness/heartbeat watchdog
+
+### Symptom (optional but recommended)
+
+amqplib's `Connection` relies on AMQP heartbeats and the underlying TCP
+socket. In the half-open TCP case (NAT drop, cloud NLB idle timeout,
+broker OOM without RST), neither `close` nor `error` may be emitted for a
+long time — enough to look "stuck" even though the retry machinery is
+healthy.
+
+### Fix
+
+Add a watchdog that forcibly destroys the underlying amqp `Connection` if no
+progress has been observed within a configurable window. The simplest
+implementation:
+
+```js
+// source/connection.js
+#heartbeatTimer = null
+
+#armWatchdog (connection, timeoutMs = 60_000) {
+  const reset = () => {
+    clearTimeout(this.#heartbeatTimer)
+    this.#heartbeatTimer = setTimeout(() => {
+      try { connection.connection?.stream?.destroy() } catch {}
+    }, timeoutMs)
+  }
+  reset()
+  connection.on('heartbeat', reset) // amqplib emits on each heartbeat
+  connection.on('close', () => clearTimeout(this.#heartbeatTimer))
+}
+```
+
+Call `#armWatchdog(this.#connection)` from `#open`. Any truly silent
+connection will be destroyed, emitting `close`, and the normal retry loop
+will take over.
+
+### Test
+
+Integration test (can be skipped by default) that blocks RabbitMQ traffic
+with `iptables` / network namespaces in CI, asserting the library recovers
+within watchdog + retry window.
+
+---
+
+## Rollout order
+
+1. Problem 1 and Problem 2 together — they remove the deadlock and keep
+   recovery inside the retry loop. These are the minimum fix for the hang.
+2. Problem 3 — guard `#close`'s rejection.
+3. Problem 6 — transactional `Channel.recover()`.
+4. Problem 5 — serialise `open()` / local connection reference.
+5. Problem 4 — broaden transient classifier.
+6. Problem 7 — optional watchdog.
+
+Each step is independently deployable and testable.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.12.0",
+  "version": "0.14.0",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",
@@ -18,35 +18,39 @@
     "AMQP",
     "RabbitMQ"
   ],
-  "engines": {
-    "node": ">= 12.0.0"
-  },
   "publishConfig": {
     "access": "public"
   },
   "main": "source/index.js",
   "types": "types/index.d.ts",
   "devDependencies": {
-    "@cucumber/cucumber": "^10.0.1",
-    "@toa.io/command": "^0.4.0",
-    "@toa.io/generic": "^1.0.0-alpha.0",
-    "@types/amqplib": "^0.10.3",
-    "@types/jest": "^29.5.8",
-    "husky": "^9.0.11",
-    "jest": "^29.5.0",
-    "js-yaml": "^4.1.0",
-    "randomstring": "^1.3.0",
+    "@semantic-release/changelog": "^6.0.3",
+    "@semantic-release/commit-analyzer": "^13.0.1",
+    "@semantic-release/git": "^10.0.1",
+    "@semantic-release/github": "^12.0.2",
+    "@semantic-release/npm": "^13.1.2",
+    "@semantic-release/release-notes-generator": "^14.1.0",
+    "@cucumber/cucumber": "^12.9.0",
+    "@testcontainers/rabbitmq": "^12.0.0",
+    "@toa.io/generic": "^1.0.0-alpha.208",
+    "@types/amqplib": "^0.10.8",
+    "@types/jest": "^30.0.0",
+    "husky": "^9.1.7",
+    "jest": "^30.4.2",
+    "js-yaml": "^4.1.1",
+    "randomstring": "^1.3.1",
     "snazzy": "^9.0.0",
-    "standard": "^17.1.0"
+    "standard": "^17.1.2",
+    "testcontainers": "^12.0.0"
   },
   "dependencies": {
-    "amqplib": "^0.10.4",
-    "msgpackr": "^1.10.2",
+    "amqplib": "^2.0.1",
+    "msgpackr": "^2.0.1",
     "promex": "^0.1.1",
     "reretry": "^0.1.0"
   },
   "scripts": {
-    "prepare": "echo run: husky install",
+    "prepare": "husky",
     "test": "standard && jest",
     "features": "cucumber-js --fail-fast -t 'not @manual'",
     "lint": "standard --fix --verbose | snazzy"
@@ -55,8 +59,5 @@
     "env": [
       "jest"
     ]
-  },
-  "workspaces": [
-    "."
-  ]
+  }
 }

package/types/io.d.ts

@@ -15,7 +15,7 @@ declare namespace comq {
   }
 
   interface Destroyable extends EventEmitter {
-    destroy (): void
+    destroy(): void
   }
 
   interface Request {
@@ -25,51 +25,55 @@ declare namespace comq {
   }
 
   interface IO extends _diagnostics.Diagnosable {
-    reply (queue: string, produce: Producer): Promise<void>
+    reply(queue: string, produce: Producer): Promise<void>
 
-    request<Reply = any, Request = any> (queue: string, payload: Request, encoding?: _encoding.Encoding): Promise<Reply> | Promise<Readable>
+    request<Reply = any, Request = any>(queue: string, payload: Request, encoding?: _encoding.Encoding): Promise<Reply> | Promise<Readable>
 
-    request (queue: string, stream: Readable, encoding?: _encoding.Encoding): Promise<Readable>
+    request(queue: string, stream: Readable, encoding?: _encoding.Encoding): Promise<Readable>
 
-    consume<T = any> (exchange: string, group: string, consumer: Consumer<T>): Promise<void>
+    consume<T = any>(exchange: string, group: string, consumer: Consumer<T>): Promise<void>
 
-    consume<T = any> (exchange: string, consumer: Consumer<T>): Promise<void>
+    consume<T = any>(exchange: string, consumer: Consumer<T>): Promise<void>
 
-    emit (exchange: string, payload: any, encoding?: _encoding.Encoding): Promise<void>
+    emit(exchange: string, payload: any, encoding?: _encoding.Encoding): Promise<void>
 
-    emit (exchange: string, payload: any, properties?: _amqp.Properties): Promise<void>
+    emit(exchange: string, payload: any, properties?: _amqp.Properties): Promise<void>
 
-    emit (exchange: string, stream: Readable, encoding?: _encoding.Encoding): Promise<void>
+    emit(exchange: string, stream: Readable, encoding?: _encoding.Encoding): Promise<void>
 
-    emit (exchange: string, stream: Readable, properties?: _amqp.Properties): Promise<void>
+    emit(exchange: string, stream: Readable, properties?: _amqp.Properties): Promise<void>
 
-    enqueue (exchange: string, payload: any, encoding?: _encoding.Encoding): Promise<void>
+    enqueue(exchange: string, payload: any, encoding?: _encoding.Encoding): Promise<void>
 
-    enqueue (exchange: string, payload: any, properties?: _amqp.Properties): Promise<void>
+    enqueue(exchange: string, payload: any, properties?: _amqp.Properties): Promise<void>
 
-    enqueue (exchange: string, stream: Readable, encoding?: _encoding.Encoding): Promise<void>
+    enqueue(exchange: string, stream: Readable, encoding?: _encoding.Encoding): Promise<void>
 
-    enqueue (exchange: string, stream: Readable, properties?: _amqp.Properties): Promise<void>
+    enqueue(exchange: string, stream: Readable, properties?: _amqp.Properties): Promise<void>
 
-    process<T = any> (queue: string, processor: Consumer<T>): Promise<void>
+    process<T = any>(queue: string, processor: Consumer<T>): Promise<void>
 
-    seal (): Promise<void>
+    seal(): Promise<void>
 
-    close (): Promise<void>
+    close(): Promise<void>
 
-    diagnose (event: 'open', listener: (index?: number) => void): void
+    diagnose(event: 'open', listener: (index?: number) => void): void
 
-    diagnose (event: 'close', listener: (index?: number) => void): void
+    diagnose(event: 'close', listener: (index?: number) => void): void
 
-    diagnose (event: 'flow', listener: (channel: _topology.type, index?: number) => void): void
+    diagnose(event: 'flow', listener: (channel: _topology.type, index?: number) => void): void
 
-    diagnose (event: 'drain', listener: (channel: _topology.type, index?: number) => void): void
+    diagnose(event: 'drain', listener: (channel: _topology.type, index?: number) => void): void
 
-    diagnose (event: 'remove', listener: (index?: number) => void): void
+    diagnose(event: 'remove', listener: (index?: number) => void): void
 
-    diagnose (event: 'recover', listener: (channel: _topology.type, index?: number) => void): void
+    diagnose(event: 'recover', listener: (channel: _topology.type, index?: number) => void): void
 
-    diagnose (event: 'discard', listener: (channel: _topology.type, message: any, index?: number) => void): void
+    diagnose(event: 'discard', listener: (channel: _topology.type, message: any, index?: number) => void): void
+
+    diagnose(event: 'pause', listener: (channel: _topology.type) => void): void
+
+    diagnose(event: 'resume', listener: (channel: _topology.type) => void): void
   }
 }
 

package/lerna.json

@@ -1,4 +0,0 @@
-{
-  "version": "0.12.0",
-  "conventionalCommits": true
-}