comq 0.13.0 → 0.15.0

package/features/steps/rpc.js

@@ -3,7 +3,7 @@
 const stream = require('node:stream')
 const assert = require('node:assert')
 const { randomBytes } = require('node:crypto')
-const { load: parse } = require('js-yaml')
+const { parse } = require('./yaml')
 const { match, timeout, quantity } = require('@toa.io/generic')
 const { Given, When, Then } = require('@cucumber/cucumber')
 const { Readable } = require('node:stream')

package/features/steps/yaml.js

@@ -0,0 +1,15 @@
+'use strict'
+
+const { load } = require('js-yaml')
+
+/**
+ * @param {string} [yaml]
+ * @returns {any}
+ */
+function parse (yaml) {
+  if (yaml === undefined || yaml.trim() === '') return undefined
+
+  return load(yaml)
+}
+
+exports.parse = parse

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.13.0",
+  "version": "0.15.0",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",
@@ -23,37 +23,47 @@
   },
   "main": "source/index.js",
   "types": "types/index.d.ts",
+  "bin": {
+    "feat": "scripts/feat.js"
+  },
   "devDependencies": {
-    "@cucumber/cucumber": "^10.0.1",
-    "@toa.io/command": "^0.4.0",
-    "@toa.io/generic": "^1.0.0-alpha.0",
-    "@types/amqplib": "^0.10.3",
-    "@types/jest": "^29.5.8",
-    "husky": "^9.0.11",
-    "jest": "^29.5.0",
-    "js-yaml": "^4.1.1",
-    "randomstring": "^1.3.0",
+    "@cucumber/cucumber": "^13.2.0",
+    "@semantic-release/changelog": "^7.0.0",
+    "@semantic-release/commit-analyzer": "^13.0.1",
+    "@semantic-release/git": "^10.0.1",
+    "@semantic-release/github": "^12.0.2",
+    "@semantic-release/npm": "^13.1.2",
+    "@semantic-release/release-notes-generator": "^14.1.0",
+    "@testcontainers/rabbitmq": "^12.0.4",
+    "@toa.io/generic": "^1.0.0-alpha.208",
+    "@types/amqplib": "^0.10.8",
+    "@types/jest": "^30.0.0",
+    "fast-glob": "^3.3.3",
+    "fuzzysort": "3.1.0",
+    "husky": "^9.1.7",
+    "jest": "^30.4.2",
+    "js-yaml": "^5.2.3",
+    "randomstring": "^1.3.1",
     "snazzy": "^9.0.0",
-    "standard": "^17.1.0"
+    "standard": "^17.1.2",
+    "testcontainers": "^12.0.4"
   },
   "dependencies": {
     "amqplib": "^2.0.1",
-    "msgpackr": "^2.0.1",
+    "msgpackr": "^2.0.5",
     "promex": "^0.1.1",
     "reretry": "^0.1.0"
   },
   "scripts": {
-    "prepare": "echo run: husky install",
+    "prepare": "husky",
     "test": "standard && jest",
     "features": "cucumber-js --fail-fast -t 'not @manual'",
+    "features:fast": "cucumber-js --fail-fast -t 'not @manual and not @heavy'",
     "lint": "standard --fix --verbose | snazzy"
   },
   "standard": {
     "env": [
       "jest"
     ]
-  },
-  "workspaces": [
-    "."
-  ]
+  }
 }

package/readme.md

@@ -103,10 +103,11 @@ await io.reply('add_numbers', ({ a, b }) => (a + b))
 
 ## Request
 
-`async IO.request(queue: string, payload: any, encoding?: string): any`
+`async IO.request(queue: string, payload: any, encoding?: string, timeout?: number): any`
 
 Send encoded Request message with `replyTo` and `correlationId` properties set and
-return decoded Reply content.
+return decoded Reply content. Optional `timeout` (ms) rejects with `ETIMEDOUT` if no
+Reply arrives in time; encoding may be omitted when only `timeout` is passed.
 
 On the initial call, queues for Requests and Replies are asserted.
 
@@ -114,6 +115,7 @@ On the initial call, queues for Requests and Replies are asserted.
 
 ```javascript
 const sum = await io.request('add_numbers', { a: 1, b: 2 })
+const sumOrTimeout = await io.request('add_numbers', { a: 1, b: 2 }, 5000)
 ```
 
 ## Consumption
@@ -329,14 +331,15 @@ combined into a cluster.
 Outgoing messages are sent to a single connection chosen at random from the shard pool. Shards that lose their
 underlying connection or experience channel [back pressure](#flow-control) on a corresponding channel are removed from
 the pool until the issue is resolved. Pending messages meeting these conditions are immediately routed among the
-remaining shards in the pool. If no shards are available, messages will wait until a shard's connection is
-re-established.
+remaining shards in the pool. If no shards are available, `send` / `publish` / `request` wait until a shard's
+connection is re-established (unbounded). For RPC, pass a [`request` timeout](#request) so callers fail with
+`ETIMEDOUT` instead of hanging while the pool is empty or a Reply never arrives.
 
 Incoming messages are consumed from all shards.
 
 `async connect(...shards: string[]): IO`
 
-Returns an instance of `IO` once a successful connection to one of the shards is established.
+Returns an instance of `IO` once connections to the shards are established.
 
 ### Example
 

package/scripts/feat.js

@@ -0,0 +1,67 @@
+#!/usr/bin/env node
+'use strict'
+
+const { spawnSync } = require('node:child_process')
+const path = require('node:path')
+const fg = require('fast-glob')
+const fuzzysort = require('fuzzysort')
+
+const ROOT = path.resolve(__dirname, '..')
+const USAGE = 'Usage: feat <query> [name] [cucumber-args...]'
+
+function toKey (file) {
+  return path.relative(ROOT, file)
+    .replace(/^features\//, '')
+    .replace(/\.feature$/, '')
+    .replace(/\./g, '/')
+}
+
+function normalizeQuery (query) {
+  return query.replace(/\./g, '/').replace(/\\/g, '/')
+}
+
+async function main () {
+  const args = process.argv.slice(2)
+  const dashIndex = args.findIndex((arg) => arg.startsWith('-'))
+  const positional = dashIndex === -1 ? args : args.slice(0, dashIndex)
+  const query = positional[0]
+  const name = positional[1]
+
+  const cucumberArgs = [
+    ...(name ? ['--name', name] : []),
+    ...positional.slice(2),
+    ...(dashIndex === -1 ? [] : args.slice(dashIndex))
+  ]
+
+  if (!query) {
+    console.error(USAGE)
+    process.exit(1)
+  }
+
+  const files = await fg('features/**/*.feature', { cwd: ROOT, absolute: true })
+  const items = files.map((file) => ({ file, key: toKey(file) }))
+  const result = fuzzysort.go(normalizeQuery(query), items, { key: 'key', limit: 1 })
+
+  if (result.length === 0) {
+    console.error(`No feature files match "${query}"`)
+    process.exit(1)
+  }
+
+  const selected = result.map(({ obj }) => obj.file)
+
+  for (const file of selected) { console.error(path.relative(ROOT, file)) }
+
+  const cucumberBin = path.join(ROOT, 'node_modules', '.bin', 'cucumber-js')
+
+  const run = spawnSync(cucumberBin, [...selected, ...cucumberArgs], {
+    cwd: ROOT,
+    stdio: 'inherit'
+  })
+
+  process.exit(run.status ?? 1)
+}
+
+main().catch((error) => {
+  console.error(error.message)
+  process.exit(1)
+})

package/source/channel.js

@@ -34,6 +34,9 @@ class Channel {
   /** @type {Promex} */
   #recovery = new Promex()
 
+  /** @type {boolean} */
+  #recovering = false
+
   /** @type {Set<Promex>} */
   #confirmations = new Set()
 
@@ -145,7 +148,14 @@ class Channel {
     await this.create()
 
     lazy.reset(this)
-    await recall(this)
+
+    this.#recovering = true
+
+    try {
+      await recall(this)
+    } finally {
+      this.#recovering = false
+    }
 
     this.#unpause(INTERRUPTION)
 
@@ -332,6 +342,7 @@ class Channel {
 
   async #recover (exception) {
     if (permanent(exception)) return false
+    if (this.#recovering) return false
     else await this.#recovery
   }
 }

package/source/connection.js

@@ -25,9 +25,15 @@ class Connection {
   /** @type {Promex} */
   #recovery = new Promex()
 
+  /** @type {Promise<void> | null} */
+  #opening = null
+
   /** @type {boolean} */
   #running = false
 
+  /** @type {NodeJS.Timeout | null} */
+  #heartbeatTimer = null
+
   #diagnostics = emitter.create()
 
   /**
@@ -35,10 +41,17 @@ class Connection {
    */
   constructor (url) {
     this.#url = url
+
+    // EventEmitter throws on 'error' with no listeners
+    this.#diagnostics.on('error', noop)
   }
 
   async open () {
-    await retry(this.#open)
+    if (this.#opening !== null) return this.#opening
+
+    this.#opening = retry(this.#open).finally(() => { this.#opening = null })
+
+    await this.#opening
 
     this.#running = true
   }
@@ -71,8 +84,11 @@ class Connection {
   }
 
   #open = async (retry) => {
+    /** @type {comq.amqp.Connection} */
+    let connection
+
     try {
-      this.#connection = await amqp.connect(this.#url)
+      connection = await amqp.connect(this.#url)
     } catch (exception) {
       if (this.#transient(exception)) return retry
       else throw exception
@@ -80,40 +96,104 @@ class Connection {
 
     // This prevents the process from crashing; 'close' will be emitted next.
     // https://amqp-node.github.io/amqplib/channel_api.html#model_events
-    this.#connection.on('error', noop)
+    connection.on('error', noop)
 
-    this.#connection.on('close', this.#close)
+    connection.on('close', (error) => this.#close(connection, error))
+    this.#connection = connection
+    this.#armWatchdog(connection)
     this.#diagnostics.emit('open')
 
-    for (const channel of this.#channels) await channel.recover(this.#connection)
+    try {
+      for (const channel of this.#channels) await channel.recover(connection)
+    } catch (exception) {
+      this.#disarmWatchdog()
+      this.#diagnostics.emit('error', exception)
+      connection.removeAllListeners()
+
+      await connection.close().catch(noop)
+
+      if (this.#connection === connection) this.#connection = undefined
+
+      return retry
+    }
 
     this.#recovery.resolve()
     this.#recovery = new Promex()
   }
 
   /**
-   * @param {Error} error
+   * @param {comq.amqp.Connection} connection
+   * @param {Error} [error]
    */
-  #close = async (error) => {
+  #close = (connection, error) => {
+    this.#disarmWatchdog()
+
+    if (this.#connection !== connection) return
+
     this.#diagnostics.emit('close', error)
-    this.#connection.removeAllListeners()
+    connection.removeAllListeners()
     this.#connection = undefined
 
-    if (error !== undefined) await this.open()
+    if (error !== undefined) {
+      this.open().catch((exception) => this.#diagnostics.emit('error', exception))
+    }
   }
 
   #recover () {
     return this.#recovery
   }
 
+  /**
+   * @param {comq.amqp.Connection} connection
+   * @param {number} [timeoutMs]
+   */
+  #armWatchdog (connection, timeoutMs = WATCHDOG_MS) {
+    const socket = connection.connection?.stream
+
+    if (socket === undefined) return
+
+    const reset = () => {
+      clearTimeout(this.#heartbeatTimer)
+      this.#heartbeatTimer = setTimeout(() => socket.destroy(), timeoutMs)
+      this.#heartbeatTimer.unref()
+    }
+
+    reset()
+    socket.on('data', reset)
+  }
+
+  #disarmWatchdog () {
+    clearTimeout(this.#heartbeatTimer)
+    this.#heartbeatTimer = null
+  }
+
   #transient (exception) {
-    const abruptly = exception.message === 'Socket closed abruptly during opening handshake'
-    const tls = exception.message === 'Client network socket disconnected before secure TLS connection was established'
+    if (this.#running) return true
+    if (TRANSIENT_CODES.has(exception.code)) return true
+    if (TRANSIENT_MESSAGES.has(exception.message)) return true
 
-    return this.#running || abruptly || tls
+    return false
   }
 }
 
+/** @type {number} */
+const WATCHDOG_MS = 60_000
+
+const TRANSIENT_CODES = new Set([
+  'ECONNREFUSED',
+  'EAI_AGAIN',
+  'ENOTFOUND',
+  'ETIMEDOUT',
+  'ECONNRESET',
+  'EHOSTUNREACH',
+  'ENETUNREACH'
+])
+
+const TRANSIENT_MESSAGES = new Set([
+  'Socket closed abruptly during opening handshake',
+  'Client network socket disconnected before secure TLS connection was established'
+])
+
 function noop () {}
 
 exports.Connection = Connection

package/source/events.js

@@ -1,7 +1,7 @@
 'use strict'
 
 /** @type {comq.diagnostics.Event[]} */
-exports.connection = ['open', 'close']
+exports.connection = ['open', 'close', 'error']
 
 /** @type {comq.diagnostics.Event[]} */
 exports.channel = ['flow', 'drain', 'recover', 'discard', 'pause', 'resume']

package/source/io.js

@@ -3,7 +3,7 @@
 const stream = require('node:stream')
 const { EventEmitter } = require('node:events')
 const { randomBytes } = require('node:crypto')
-const { setTimeout } = require('node:timers/promises')
+const { setTimeout: delay } = require('node:timers/promises')
 const { Promex } = require('promex')
 const { memo, failsafe, lazy, track } = require('./attributes')
 
@@ -75,24 +75,45 @@ class IO {
       /**
        * @param {string} queue
        * @param {any | Readable} payload
-       * @param {comq.Encoding} [encoding]
+       * @param {comq.Encoding | number} [encoding]
+       * @param {number} [timeout]
        * @returns {Promise<any | Readable>}
        */
-      async (queue, payload, encoding) => {
+      async (queue, payload, encoding, timeout) => {
+        if (typeof encoding === 'number') {
+          timeout = encoding
+          encoding = undefined
+        }
+
         if (payload instanceof stream.Readable) {
           return pipeline(
             payload,
-            (payload) => this.request(queue, payload, encoding),
+            (payload) => this.request(queue, payload, encoding, timeout),
             this.#requests
           )
         }
 
-        const request = this.#createRequest(queue, payload, encoding)
+        const request = this.#createRequest(queue, payload, /** @type {comq.Encoding} */ encoding)
         const reply = this.#createReply(request)
 
-        await this.#requests.send(queue, request.buffer, request.properties)
+        const done = this.#requests.send(queue, request.buffer, request.properties)
+          .then(() => reply)
+
+        if (typeof timeout !== 'number') return done
 
-        return reply
+        try {
+          return await Promise.race([
+            done,
+            delay(timeout).then(() => Promise.reject(timeoutError(timeout)))
+          ])
+        } catch (exception) {
+          if (exception.code === 'ETIMEDOUT') {
+            request.emitter.removeAllListeners(request.properties.correlationId)
+            reply.reject(exception)
+          }
+
+          throw exception
+        }
       }))
 
   consume = lazy(this, this.#createEventChannel,
@@ -396,7 +417,7 @@ class IO {
     Even if these messages are lost, the reply stream will be closed anyway,
     either due to missing heartbeat or the deletion of the stream queue.
     */
-    await setTimeout(50)
+    await delay(50)
   }
 
   /**
@@ -456,6 +477,19 @@ const DEFAULT = 'application/json'
 
 const RETRANSMISSION = /** @type {Error} */ Symbol('retransmission')
 
+/**
+ * @param {number} timeout
+ * @returns {Error}
+ */
+function timeoutError (timeout) {
+  const error = /** @type {Error & { code: string }} */ new Error(
+    `Request timed out after ${timeout}ms`)
+
+  error.code = 'ETIMEDOUT'
+
+  return error
+}
+
 function noop () {}
 
 exports.IO = IO

package/test/amqplib.mock.js

@@ -21,6 +21,12 @@ class Connection extends EventEmitter {
   constructor () {
     super()
 
+    const stream = new EventEmitter()
+
+    stream.destroy = jest.fn()
+
+    this.connection = { stream }
+
     // noinspection JSValidateTypes
     this.removeAllListeners = jest.spyOn(this, 'removeAllListeners')
 

package/test/channel.test.js

@@ -552,6 +552,29 @@ describe('recovery', () => {
     await expect(channel.consume(queue, consumer)).rejects.toStrictEqual(exception)
   })
 
+  it('should not deadlock when recall hits a transient error during recover', async () => {
+    await channel.consume(queue, consumer)
+
+    const replacement = await amqplib.connect()
+    const method = `create${topology.confirms ? 'Confirm' : ''}Channel`
+
+    replacement[method].mockImplementationOnce(async () => {
+      const transient = await amqplib.connect()
+      const bad = await transient[method]()
+
+      bad.consume.mockImplementation(async () => { throw new Error('Channel closed') })
+
+      return bad
+    })
+
+    const deadlock = timeout(200).then(() => {
+      throw new Error('recover hung (deadlock)')
+    })
+
+    await expect(Promise.race([channel.recover(replacement), deadlock]))
+      .rejects.toThrow('Channel closed')
+  })
+
   describe.each(/** @type {[string, boolean][]} */ [
     ['', false],
     [' not', true]

package/test/connection.test.js

@@ -40,7 +40,15 @@ describe('initial connection', () => {
   })
 
   it.each(/** @type {[string, Partial<Error>][]} */[
-    ['Socket closed', { message: 'Socket closed abruptly during opening handshake' }]
+    ['Socket closed', { message: 'Socket closed abruptly during opening handshake' }],
+    ['TLS disconnect', { message: 'Client network socket disconnected before secure TLS connection was established' }],
+    ['ECONNREFUSED', { code: 'ECONNREFUSED' }],
+    ['EAI_AGAIN', { code: 'EAI_AGAIN' }],
+    ['ENOTFOUND', { code: 'ENOTFOUND' }],
+    ['ETIMEDOUT', { code: 'ETIMEDOUT' }],
+    ['ECONNRESET', { code: 'ECONNRESET' }],
+    ['EHOSTUNREACH', { code: 'EHOSTUNREACH' }],
+    ['ENETUNREACH', { code: 'ENETUNREACH' }]
   ])('should reconnect on %s',
     async (_, error) => {
       amqplib.connect.mockImplementationOnce(async () => { throw error })
@@ -50,10 +58,9 @@ describe('initial connection', () => {
       expect(amqplib.connect).toHaveBeenCalledTimes(2)
     })
 
-  it.each(/** @type {[string, Partial<Error>][]} */[
-    ['ECONNREFUSED', { code: 'ECONNREFUSED' }],
-    ['any exception', new Error(generate())]
-  ])('should throw if error is %s', async (_, exception) => {
+  it('should throw if error is permanent', async () => {
+    const exception = new Error(generate())
+
     amqplib.connect.mockImplementationOnce(async () => { throw exception })
 
     await expect(connection.open()).rejects.toStrictEqual(exception)
@@ -109,6 +116,68 @@ describe('reconnection', () => {
 
     expect(channel.recover).toHaveBeenCalledWith(replacement)
   })
+
+  it('should retry when channel recover fails', async () => {
+    const channel = await connection.createChannel('request')
+
+    channel.recover
+      .mockRejectedValueOnce(new Error('Channel closed'))
+      .mockResolvedValue(undefined)
+
+    conn.emit('close', new Error())
+
+    const start = Date.now()
+
+    while (channel.recover.mock.calls.length < 2 && Date.now() - start < 10000) {
+      await timeout(50)
+    }
+
+    expect(channel.recover).toHaveBeenCalledTimes(2)
+    expect(amqplib.connect.mock.calls.length).toBeGreaterThanOrEqual(3)
+  }, 15000)
+
+  it('should emit error when reconnect open fails', async () => {
+    const errors = []
+    const unhandled = jest.fn()
+
+    connection.diagnose('error', (exception) => errors.push(exception))
+    process.on('unhandledRejection', unhandled)
+
+    const boom = new Error('reconnect failed')
+
+    connection.open = jest.fn(async () => { throw boom })
+
+    conn.emit('close', new Error('broker down'))
+
+    await timeout(10)
+
+    process.off('unhandledRejection', unhandled)
+
+    expect(errors).toContain(boom)
+    expect(unhandled).not.toHaveBeenCalled()
+  })
+
+  it('should ignore close from a stale connection', async () => {
+    const stale = conn
+    const closeHandler = stale.on.mock.calls.find(([event]) => event === 'close')[1]
+
+    stale.emit('close', new Error('gone'))
+
+    await timeout(50)
+
+    const live = await amqplib.connect.mock.results[1].value
+    const connects = amqplib.connect.mock.calls.length
+
+    expect(live).not.toBe(stale)
+
+    live.removeAllListeners.mockClear()
+    closeHandler(new Error('late'))
+
+    await timeout(10)
+
+    expect(amqplib.connect).toHaveBeenCalledTimes(connects)
+    expect(live.removeAllListeners).not.toHaveBeenCalled()
+  })
 })
 
 describe('create channel', () => {
@@ -203,6 +272,44 @@ describe('create channel', () => {
   })
 })
 
+describe('watchdog', () => {
+  afterEach(() => {
+    jest.useRealTimers()
+  })
+
+  it('should destroy silent connection when watchdog expires', async () => {
+    jest.useFakeTimers()
+
+    await connection.open()
+
+    const conn = await amqplib.connect.mock.results[0].value
+
+    await jest.advanceTimersByTimeAsync(60_000)
+
+    expect(conn.connection.stream.destroy).toHaveBeenCalled()
+  })
+
+  it('should reset watchdog on socket data', async () => {
+    jest.useFakeTimers()
+
+    await connection.open()
+
+    const conn = await amqplib.connect.mock.results[0].value
+
+    await jest.advanceTimersByTimeAsync(50_000)
+
+    conn.connection.stream.emit('data', Buffer.alloc(0))
+
+    await jest.advanceTimersByTimeAsync(50_000)
+
+    expect(conn.connection.stream.destroy).not.toHaveBeenCalled()
+
+    await jest.advanceTimersByTimeAsync(10_000)
+
+    expect(conn.connection.stream.destroy).toHaveBeenCalled()
+  })
+})
+
 describe('close', () => {
   it('should close connection', async () => {
     await connection.open()