comq 0.14.0 → 0.15.1

package/source/connection.js

@@ -25,9 +25,15 @@ class Connection {
   /** @type {Promex} */
   #recovery = new Promex()
 
+  /** @type {Promise<void> | null} */
+  #opening = null
+
   /** @type {boolean} */
   #running = false
 
+  /** @type {NodeJS.Timeout | null} */
+  #heartbeatTimer = null
+
   #diagnostics = emitter.create()
 
   /**
@@ -35,10 +41,17 @@ class Connection {
    */
   constructor (url) {
     this.#url = url
+
+    // EventEmitter throws on 'error' with no listeners
+    this.#diagnostics.on('error', noop)
   }
 
   async open () {
-    await retry(this.#open)
+    if (this.#opening !== null) return this.#opening
+
+    this.#opening = retry(this.#open).finally(() => { this.#opening = null })
+
+    await this.#opening
 
     this.#running = true
   }
@@ -71,8 +84,11 @@ class Connection {
   }
 
   #open = async (retry) => {
+    /** @type {comq.amqp.Connection} */
+    let connection
+
     try {
-      this.#connection = await amqp.connect(this.#url)
+      connection = await amqp.connect(this.#url)
     } catch (exception) {
       if (this.#transient(exception)) return retry
       else throw exception
@@ -80,40 +96,104 @@ class Connection {
 
     // This prevents the process from crashing; 'close' will be emitted next.
     // https://amqp-node.github.io/amqplib/channel_api.html#model_events
-    this.#connection.on('error', noop)
+    connection.on('error', noop)
 
-    this.#connection.on('close', this.#close)
+    connection.on('close', (error) => this.#close(connection, error))
+    this.#connection = connection
+    this.#armWatchdog(connection)
     this.#diagnostics.emit('open')
 
-    for (const channel of this.#channels) await channel.recover(this.#connection)
+    try {
+      for (const channel of this.#channels) await channel.recover(connection)
+    } catch (exception) {
+      this.#disarmWatchdog()
+      this.#diagnostics.emit('error', exception)
+      connection.removeAllListeners()
+
+      await connection.close().catch(noop)
+
+      if (this.#connection === connection) this.#connection = undefined
+
+      return retry
+    }
 
     this.#recovery.resolve()
     this.#recovery = new Promex()
   }
 
   /**
-   * @param {Error} error
+   * @param {comq.amqp.Connection} connection
+   * @param {Error} [error]
    */
-  #close = async (error) => {
+  #close = (connection, error) => {
+    this.#disarmWatchdog()
+
+    if (this.#connection !== connection) return
+
     this.#diagnostics.emit('close', error)
-    this.#connection.removeAllListeners()
+    connection.removeAllListeners()
     this.#connection = undefined
 
-    if (error !== undefined) await this.open()
+    if (error !== undefined) {
+      this.open().catch((exception) => this.#diagnostics.emit('error', exception))
+    }
   }
 
   #recover () {
     return this.#recovery
   }
 
+  /**
+   * @param {comq.amqp.Connection} connection
+   * @param {number} [timeoutMs]
+   */
+  #armWatchdog (connection, timeoutMs = WATCHDOG_MS) {
+    const socket = connection.connection?.stream
+
+    if (socket === undefined) return
+
+    const reset = () => {
+      clearTimeout(this.#heartbeatTimer)
+      this.#heartbeatTimer = setTimeout(() => socket.destroy(), timeoutMs)
+      this.#heartbeatTimer.unref()
+    }
+
+    reset()
+    socket.on('data', reset)
+  }
+
+  #disarmWatchdog () {
+    clearTimeout(this.#heartbeatTimer)
+    this.#heartbeatTimer = null
+  }
+
   #transient (exception) {
-    const abruptly = exception.message === 'Socket closed abruptly during opening handshake'
-    const tls = exception.message === 'Client network socket disconnected before secure TLS connection was established'
+    if (this.#running) return true
+    if (TRANSIENT_CODES.has(exception.code)) return true
+    if (TRANSIENT_MESSAGES.has(exception.message)) return true
 
-    return this.#running || abruptly || tls
+    return false
   }
 }
 
+/** @type {number} */
+const WATCHDOG_MS = 60_000
+
+const TRANSIENT_CODES = new Set([
+  'ECONNREFUSED',
+  'EAI_AGAIN',
+  'ENOTFOUND',
+  'ETIMEDOUT',
+  'ECONNRESET',
+  'EHOSTUNREACH',
+  'ENETUNREACH'
+])
+
+const TRANSIENT_MESSAGES = new Set([
+  'Socket closed abruptly during opening handshake',
+  'Client network socket disconnected before secure TLS connection was established'
+])
+
 function noop () {}
 
 exports.Connection = Connection

package/source/events.js

@@ -1,7 +1,7 @@
 'use strict'
 
 /** @type {comq.diagnostics.Event[]} */
-exports.connection = ['open', 'close']
+exports.connection = ['open', 'close', 'error']
 
 /** @type {comq.diagnostics.Event[]} */
 exports.channel = ['flow', 'drain', 'recover', 'discard', 'pause', 'resume']

package/source/io.js

@@ -3,7 +3,7 @@
 const stream = require('node:stream')
 const { EventEmitter } = require('node:events')
 const { randomBytes } = require('node:crypto')
-const { setTimeout } = require('node:timers/promises')
+const { setTimeout: delay } = require('node:timers/promises')
 const { Promex } = require('promex')
 const { memo, failsafe, lazy, track } = require('./attributes')
 
@@ -75,24 +75,45 @@ class IO {
       /**
        * @param {string} queue
        * @param {any | Readable} payload
-       * @param {comq.Encoding} [encoding]
+       * @param {comq.Encoding | number} [encoding]
+       * @param {number} [timeout]
        * @returns {Promise<any | Readable>}
        */
-      async (queue, payload, encoding) => {
+      async (queue, payload, encoding, timeout) => {
+        if (typeof encoding === 'number') {
+          timeout = encoding
+          encoding = undefined
+        }
+
         if (payload instanceof stream.Readable) {
           return pipeline(
             payload,
-            (payload) => this.request(queue, payload, encoding),
+            (payload) => this.request(queue, payload, encoding, timeout),
             this.#requests
           )
         }
 
-        const request = this.#createRequest(queue, payload, encoding)
+        const request = this.#createRequest(queue, payload, /** @type {comq.Encoding} */ encoding)
         const reply = this.#createReply(request)
 
-        await this.#requests.send(queue, request.buffer, request.properties)
+        const done = this.#requests.send(queue, request.buffer, request.properties)
+          .then(() => reply)
+
+        if (typeof timeout !== 'number') return done
 
-        return reply
+        try {
+          return await Promise.race([
+            done,
+            delay(timeout).then(() => Promise.reject(timeoutError(timeout)))
+          ])
+        } catch (exception) {
+          if (exception.code === 'ETIMEDOUT') {
+            request.emitter.removeAllListeners(request.properties.correlationId)
+            reply.reject(exception)
+          }
+
+          throw exception
+        }
       }))
 
   consume = lazy(this, this.#createEventChannel,
@@ -396,7 +417,7 @@ class IO {
     Even if these messages are lost, the reply stream will be closed anyway,
     either due to missing heartbeat or the deletion of the stream queue.
     */
-    await setTimeout(50)
+    await delay(50)
   }
 
   /**
@@ -456,6 +477,19 @@ const DEFAULT = 'application/json'
 
 const RETRANSMISSION = /** @type {Error} */ Symbol('retransmission')
 
+/**
+ * @param {number} timeout
+ * @returns {Error}
+ */
+function timeoutError (timeout) {
+  const error = /** @type {Error & { code: string }} */ new Error(
+    `Request timed out after ${timeout}ms`)
+
+  error.code = 'ETIMEDOUT'
+
+  return error
+}
+
 function noop () {}
 
 exports.IO = IO

package/test/ReplyStream.test.js

@@ -0,0 +1,45 @@
+'use strict'
+
+const { EventEmitter } = require('node:events')
+const { once } = require('node:events')
+const { ReplyStream } = require('../source/.io/ReplyStream')
+
+/** @type {jest.Mock} */
+let reply
+
+/** @type {ReplyStream} */
+let stream
+
+beforeEach(() => {
+  global.COMQ_TESTING_MAX_BUFFER_SIZE = 3
+  global.COMQ_TESTING_IDLE_INTERVAL = 60_000
+
+  reply = jest.fn().mockResolvedValue(true)
+
+  const request = {
+    emitter: new EventEmitter(),
+    properties: { correlationId: 'test-correlation' }
+  }
+
+  stream = new ReplyStream(request, reply)
+})
+
+afterEach(() => {
+  delete global.COMQ_TESTING_MAX_BUFFER_SIZE
+  delete global.COMQ_TESTING_IDLE_INTERVAL
+
+  stream.destroy()
+})
+
+it('should destroy without calling reply when buffer overflows before control.ok', async () => {
+  const closed = once(stream, 'close')
+
+  for (let index = 1; index <= 5; index++) {
+    stream.arrange(index, { headers: { index } })
+  }
+
+  await closed
+
+  expect(stream.destroyed).toBe(true)
+  expect(reply).not.toHaveBeenCalled()
+})

package/test/amqplib.mock.js

@@ -21,6 +21,12 @@ class Connection extends EventEmitter {
   constructor () {
     super()
 
+    const stream = new EventEmitter()
+
+    stream.destroy = jest.fn()
+
+    this.connection = { stream }
+
     // noinspection JSValidateTypes
     this.removeAllListeners = jest.spyOn(this, 'removeAllListeners')
 

package/test/channel.test.js

@@ -552,6 +552,29 @@ describe('recovery', () => {
     await expect(channel.consume(queue, consumer)).rejects.toStrictEqual(exception)
   })
 
+  it('should not deadlock when recall hits a transient error during recover', async () => {
+    await channel.consume(queue, consumer)
+
+    const replacement = await amqplib.connect()
+    const method = `create${topology.confirms ? 'Confirm' : ''}Channel`
+
+    replacement[method].mockImplementationOnce(async () => {
+      const transient = await amqplib.connect()
+      const bad = await transient[method]()
+
+      bad.consume.mockImplementation(async () => { throw new Error('Channel closed') })
+
+      return bad
+    })
+
+    const deadlock = timeout(200).then(() => {
+      throw new Error('recover hung (deadlock)')
+    })
+
+    await expect(Promise.race([channel.recover(replacement), deadlock]))
+      .rejects.toThrow('Channel closed')
+  })
+
   describe.each(/** @type {[string, boolean][]} */ [
     ['', false],
     [' not', true]

package/test/connection.test.js

@@ -40,7 +40,15 @@ describe('initial connection', () => {
   })
 
   it.each(/** @type {[string, Partial<Error>][]} */[
-    ['Socket closed', { message: 'Socket closed abruptly during opening handshake' }]
+    ['Socket closed', { message: 'Socket closed abruptly during opening handshake' }],
+    ['TLS disconnect', { message: 'Client network socket disconnected before secure TLS connection was established' }],
+    ['ECONNREFUSED', { code: 'ECONNREFUSED' }],
+    ['EAI_AGAIN', { code: 'EAI_AGAIN' }],
+    ['ENOTFOUND', { code: 'ENOTFOUND' }],
+    ['ETIMEDOUT', { code: 'ETIMEDOUT' }],
+    ['ECONNRESET', { code: 'ECONNRESET' }],
+    ['EHOSTUNREACH', { code: 'EHOSTUNREACH' }],
+    ['ENETUNREACH', { code: 'ENETUNREACH' }]
   ])('should reconnect on %s',
     async (_, error) => {
       amqplib.connect.mockImplementationOnce(async () => { throw error })
@@ -50,10 +58,9 @@ describe('initial connection', () => {
       expect(amqplib.connect).toHaveBeenCalledTimes(2)
     })
 
-  it.each(/** @type {[string, Partial<Error>][]} */[
-    ['ECONNREFUSED', { code: 'ECONNREFUSED' }],
-    ['any exception', new Error(generate())]
-  ])('should throw if error is %s', async (_, exception) => {
+  it('should throw if error is permanent', async () => {
+    const exception = new Error(generate())
+
     amqplib.connect.mockImplementationOnce(async () => { throw exception })
 
     await expect(connection.open()).rejects.toStrictEqual(exception)
@@ -109,6 +116,68 @@ describe('reconnection', () => {
 
     expect(channel.recover).toHaveBeenCalledWith(replacement)
   })
+
+  it('should retry when channel recover fails', async () => {
+    const channel = await connection.createChannel('request')
+
+    channel.recover
+      .mockRejectedValueOnce(new Error('Channel closed'))
+      .mockResolvedValue(undefined)
+
+    conn.emit('close', new Error())
+
+    const start = Date.now()
+
+    while (channel.recover.mock.calls.length < 2 && Date.now() - start < 10000) {
+      await timeout(50)
+    }
+
+    expect(channel.recover).toHaveBeenCalledTimes(2)
+    expect(amqplib.connect.mock.calls.length).toBeGreaterThanOrEqual(3)
+  }, 15000)
+
+  it('should emit error when reconnect open fails', async () => {
+    const errors = []
+    const unhandled = jest.fn()
+
+    connection.diagnose('error', (exception) => errors.push(exception))
+    process.on('unhandledRejection', unhandled)
+
+    const boom = new Error('reconnect failed')
+
+    connection.open = jest.fn(async () => { throw boom })
+
+    conn.emit('close', new Error('broker down'))
+
+    await timeout(10)
+
+    process.off('unhandledRejection', unhandled)
+
+    expect(errors).toContain(boom)
+    expect(unhandled).not.toHaveBeenCalled()
+  })
+
+  it('should ignore close from a stale connection', async () => {
+    const stale = conn
+    const closeHandler = stale.on.mock.calls.find(([event]) => event === 'close')[1]
+
+    stale.emit('close', new Error('gone'))
+
+    await timeout(50)
+
+    const live = await amqplib.connect.mock.results[1].value
+    const connects = amqplib.connect.mock.calls.length
+
+    expect(live).not.toBe(stale)
+
+    live.removeAllListeners.mockClear()
+    closeHandler(new Error('late'))
+
+    await timeout(10)
+
+    expect(amqplib.connect).toHaveBeenCalledTimes(connects)
+    expect(live.removeAllListeners).not.toHaveBeenCalled()
+  })
 })
 
 describe('create channel', () => {
@@ -203,6 +272,44 @@ describe('create channel', () => {
   })
 })
 
+describe('watchdog', () => {
+  afterEach(() => {
+    jest.useRealTimers()
+  })
+
+  it('should destroy silent connection when watchdog expires', async () => {
+    jest.useFakeTimers()
+
+    await connection.open()
+
+    const conn = await amqplib.connect.mock.results[0].value
+
+    await jest.advanceTimersByTimeAsync(60_000)
+
+    expect(conn.connection.stream.destroy).toHaveBeenCalled()
+  })
+
+  it('should reset watchdog on socket data', async () => {
+    jest.useFakeTimers()
+
+    await connection.open()
+
+    const conn = await amqplib.connect.mock.results[0].value
+
+    await jest.advanceTimersByTimeAsync(50_000)
+
+    conn.connection.stream.emit('data', Buffer.alloc(0))
+
+    await jest.advanceTimersByTimeAsync(50_000)
+
+    expect(conn.connection.stream.destroy).not.toHaveBeenCalled()
+
+    await jest.advanceTimersByTimeAsync(10_000)
+
+    expect(conn.connection.stream.destroy).toHaveBeenCalled()
+  })
+})
+
 describe('close', () => {
   it('should close connection', async () => {
     await connection.open()

package/test/io.request.timeout.test.js

@@ -0,0 +1,61 @@
+'use strict'
+
+const { generate } = require('randomstring')
+
+const mock = require('./connection.mock')
+const { IO } = require('../source/io')
+
+/** @type {comq.IO} */
+let io
+
+/** @type {jest.MockedObject<comq.Connection>} */
+let connection
+
+const queue = generate()
+const payload = { [generate()]: generate() }
+
+beforeEach(() => {
+  jest.clearAllMocks()
+
+  connection = mock.connection()
+  io = new IO(connection)
+})
+
+it('should reject when request times out', async () => {
+  await expect(io.request(queue, payload, 50)).rejects.toMatchObject({
+    message: expect.stringContaining('timed out'),
+    code: 'ETIMEDOUT'
+  })
+})
+
+it('should reject when request times out with encoding', async () => {
+  await expect(io.request(queue, payload, 'application/json', 50)).rejects.toMatchObject({
+    code: 'ETIMEDOUT'
+  })
+})
+
+it('should time out while send is still pending', async () => {
+  connection.createChannel.mockImplementation(async () => hangingChannel())
+
+  await expect(io.request(queue, payload, 50)).rejects.toMatchObject({
+    code: 'ETIMEDOUT'
+  })
+})
+
+/**
+ * @returns {jest.MockedObject<comq.Channel>}
+ */
+function hangingChannel () {
+  return /** @type {jest.MockedObject<comq.Channel>} */ ({
+    sharded: false,
+    consume: jest.fn(async () => undefined),
+    send: jest.fn(() => new Promise(() => {})),
+    deliver: jest.fn(async () => undefined),
+    fire: jest.fn(async () => undefined),
+    subscribe: jest.fn(async () => undefined),
+    publish: jest.fn(async () => undefined),
+    diagnose: jest.fn(),
+    seal: jest.fn(async () => undefined),
+    recover: jest.fn(async () => undefined)
+  })
+}

package/types/diagnostic.d.ts

@@ -1,6 +1,6 @@
 declare namespace comq.diagnostics {
 
-  type Event = 'open' | 'close' | 'flow' | 'drain' | 'remove' | 'recover' | 'discard' |
+  type Event = 'open' | 'close' | 'error' | 'flow' | 'drain' | 'remove' | 'recover' | 'discard' |
     'pause' | 'resume' | 'return'
 
   interface Diagnosable {

package/types/io.d.ts

@@ -27,9 +27,11 @@ declare namespace comq {
   interface IO extends _diagnostics.Diagnosable {
     reply(queue: string, produce: Producer): Promise<void>
 
-    request<Reply = any, Request = any>(queue: string, payload: Request, encoding?: _encoding.Encoding): Promise<Reply> | Promise<Readable>
+    request<Reply = any, Request = any>(queue: string, payload: Request, encoding?: _encoding.Encoding, timeout?: number): Promise<Reply> | Promise<Readable>
 
-    request(queue: string, stream: Readable, encoding?: _encoding.Encoding): Promise<Readable>
+    request<Reply = any, Request = any>(queue: string, payload: Request, timeout: number): Promise<Reply> | Promise<Readable>
+
+    request(queue: string, stream: Readable, encoding?: _encoding.Encoding, timeout?: number): Promise<Readable>
 
     consume<T = any>(exchange: string, group: string, consumer: Consumer<T>): Promise<void>
 
@@ -61,6 +63,8 @@ declare namespace comq {
 
     diagnose(event: 'close', listener: (index?: number) => void): void
 
+    diagnose(event: 'error', listener: (error: Error, index?: number) => void): void
+
     diagnose(event: 'flow', listener: (channel: _topology.type, index?: number) => void): void
 
     diagnose(event: 'drain', listener: (channel: _topology.type, index?: number) => void): void

package/.github/workflows/codeql.yml

@@ -1,78 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ "main" ]
-    paths: [ "source/**" ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ "main" ]
-    paths: [ "source/**" ]
-  schedule:
-    - cron: '15 8 * * 3'
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'javascript' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Use only 'java' to analyze code written in Java, Kotlin or both
-        # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
-        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v6
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v4
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-
-        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        # queries: security-extended,security-and-quality
-
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v4
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-
-    #   If the Autobuild fails above, remove it and uncomment the following three lines.
-    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
-
-    # - run: |
-    #     echo "Run, Build Application using script"
-    #     ./location_of_script_within_repo/buildscript.sh
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v4
-      with:
-        category: "/language:${{matrix.language}}"