comq 0.15.7 → 0.16.0

package/test/connection.test.js

@@ -4,7 +4,8 @@
 
 const { generate } = require('randomstring')
 
-const { timeout, promex, random } = require('@toa.io/generic')
+const { Promex } = require('promex')
+const { timeout, random } = require('./helpers')
 const { amqplib, connect } = require('./amqplib.mock')
 const { channel: create } = require('./connection.mock')
 const mock = { amqplib, channel: { create } }
@@ -37,7 +38,8 @@ describe('initial connection', () => {
   it('should connect', async () => {
     await connection.open()
 
-    expect(amqplib.connect).toHaveBeenCalledWith(url, { timeout: expect.any(Number) })
+    expect(amqplib.connect).toHaveBeenCalledWith(expect.stringContaining(url),
+      expect.objectContaining({ timeout: expect.any(Number) }))
   })
 
   it.each(/** @type {[string, Partial<Error>][]} */[
@@ -138,6 +140,41 @@ describe('reconnection', () => {
     expect(amqplib.connect.mock.calls.length).toBeGreaterThanOrEqual(3)
   }, 15000)
 
+  it('should keep an error sink on a dropped connection', async () => {
+    const channel = await connection.createChannel('request')
+
+    channel.recover
+      .mockRejectedValueOnce(new Error('Channel closed'))
+      .mockResolvedValue(undefined)
+
+    conn.emit('close', new Error('lost'))
+
+    await timeout(10)
+
+    const dropped = await amqplib.connect.mock.results[1].value
+
+    // amqplib goes on emitting on a connection it does not know is gone, and an
+    // 'error' with no listener left takes the process down
+    expect(() => dropped.emit('error', new Error('Heartbeat timeout'))).not.toThrow()
+
+    // let the attempt that follows the failed recovery settle
+    const start = Date.now()
+
+    while (channel.recover.mock.calls.length < 2 && Date.now() - start < 10000) await timeout(50)
+  }, 15000)
+
+  it('should reconnect despite a listener that throws', async () => {
+    // a diagnostic listener has no business breaking the reconnection it reports
+    connection.diagnose('reconnect', () => { throw new Error('listener') })
+    connection.diagnose('open', () => { throw new Error('listener') })
+
+    conn.emit('close', new Error('lost'))
+
+    await expect(connection.createChannel('event')).resolves.toBeDefined()
+
+    expect(amqplib.connect).toHaveBeenCalledTimes(2)
+  }, 10000)
+
   it('should emit error when reconnect open fails', async () => {
     const errors = []
     const unhandled = jest.fn()
@@ -339,7 +376,7 @@ describe('create channel', () => {
 
     jest.clearAllMocks()
 
-    const promise = promex()
+    const promise = new Promex()
 
     amqplib.connect.mockImplementationOnce(() => promise)
 
@@ -361,6 +398,106 @@ describe('create channel', () => {
   })
 })
 
+describe('closed channels', () => {
+  /** @type {jest.MockedObject<comq.amqp.Connection>} */
+  let conn
+
+  beforeEach(async () => {
+    // an earlier case leaves an implementation of its own behind
+    create.mockImplementation(async () => ({ recover: jest.fn(async () => undefined), closed: false }))
+
+    await connection.open()
+
+    conn = await amqplib.connect.mock.results[0].value
+  })
+
+  afterEach(async () => {
+    await connection.close()
+  })
+
+  // a channel given back is not one this connection has any more
+  it('should not recover one that was given back', async () => {
+    const channel = await connection.createChannel('event')
+
+    channel.closed = true
+
+    conn.emit('close', new Error())
+
+    await timeout(50)
+
+    expect(channel.recover).not.toHaveBeenCalled()
+  })
+
+  it('should recover one that is still in use', async () => {
+    const channel = await connection.createChannel('event')
+
+    conn.emit('close', new Error())
+
+    await timeout(50)
+
+    expect(channel.recover).toHaveBeenCalled()
+  })
+})
+
+describe('channel exhaustion', () => {
+  const EXHAUSTED = 'No channels left to allocate'
+
+  /** @type {jest.MockedObject<comq.amqp.Connection>} */
+  let conn
+
+  beforeEach(async () => {
+    await connection.open()
+
+    conn = await amqplib.connect.mock.results[0].value
+  })
+
+  it('should reject rather than wait for a recovery that is not coming', async () => {
+    create.mockImplementation(async () => { throw new Error(EXHAUSTED) })
+
+    // the regression is a hang, so the assertion is that it settles at all
+    const settled = await Promise.race([
+      connection.createChannel('event').then(() => 'resolved', (e) => e.message),
+      timeout(100).then(() => 'pending')
+    ])
+
+    expect(settled).toStrictEqual(EXHAUSTED)
+  })
+
+  it('should not retry', async () => {
+    create.mockImplementation(async () => { throw new Error(EXHAUSTED) })
+
+    await connection.createChannel('event').catch(() => undefined)
+
+    expect(create).toHaveBeenCalledTimes(1)
+  })
+
+  it('should report the limit the broker negotiated', async () => {
+    const listener = jest.fn()
+
+    await connection.diagnose('exhausted', listener)
+
+    create.mockImplementation(async () => { throw new Error(EXHAUSTED) })
+
+    await connection.createChannel('event').catch(() => undefined)
+
+    expect(listener).toHaveBeenCalledWith(conn.connection.channelMax)
+  })
+
+  it('should leave every other failure its recovery', async () => {
+    create.mockImplementation(async () => { throw new Error() })
+
+    setTimeout(() => {
+      create.mockImplementation(async () => generate())
+
+      conn.emit('close', new Error())
+    }, 1)
+
+    const channel = await connection.createChannel('request')
+
+    expect(channel).toStrictEqual(await create.mock.results[1].value)
+  })
+})
+
 describe('watchdog', () => {
   afterEach(() => {
     jest.useRealTimers()
@@ -373,11 +510,23 @@ describe('watchdog', () => {
 
     const conn = await amqplib.connect.mock.results[0].value
 
-    await jest.advanceTimersByTimeAsync(60_000)
+    await jest.advanceTimersByTimeAsync(46_000)
 
     expect(conn.connection.stream.destroy).toHaveBeenCalled()
   })
 
+  it('should reconnect after the watchdog destroys a silent connection', async () => {
+    jest.useFakeTimers()
+
+    await connection.open()
+
+    await jest.advanceTimersByTimeAsync(46_000)
+
+    // a socket destroyed without an error is a socket amqplib never reports,
+    // which leaves the connection silently dead instead of recovering
+    expect(amqplib.connect).toHaveBeenCalledTimes(2)
+  })
+
   it('should not let a replaced socket disarm the watchdog', async () => {
     jest.useFakeTimers()
 
@@ -407,11 +556,11 @@ describe('watchdog', () => {
 
     const conn = await amqplib.connect.mock.results[0].value
 
-    await jest.advanceTimersByTimeAsync(50_000)
+    await jest.advanceTimersByTimeAsync(40_000)
 
     conn.connection.stream.emit('data', Buffer.alloc(0))
 
-    await jest.advanceTimersByTimeAsync(50_000)
+    await jest.advanceTimersByTimeAsync(40_000)
 
     expect(conn.connection.stream.destroy).not.toHaveBeenCalled()
 
@@ -419,6 +568,77 @@ describe('watchdog', () => {
 
     expect(conn.connection.stream.destroy).toHaveBeenCalled()
   })
+
+  it('should derive the watchdog from the negotiated heartbeat', async () => {
+    jest.useFakeTimers()
+
+    const connection = new Connection('amqp://localhost?heartbeat=30')
+
+    await connection.open()
+
+    const conn = await amqplib.connect.mock.results[0].value
+
+    // the broker is only expected to say something every 15 seconds
+    await jest.advanceTimersByTimeAsync(60_000)
+
+    expect(conn.connection.stream.destroy).not.toHaveBeenCalled()
+
+    await jest.advanceTimersByTimeAsync(31_000)
+
+    expect(conn.connection.stream.destroy).toHaveBeenCalled()
+  })
+
+  it('should not arm the watchdog when heartbeats are disabled', async () => {
+    jest.useFakeTimers()
+
+    const connection = new Connection('amqp://localhost?heartbeat=0')
+
+    await connection.open()
+
+    const conn = await amqplib.connect.mock.results[0].value
+
+    // an idle connection that has agreed to no heartbeats is silent by design
+    await jest.advanceTimersByTimeAsync(600_000)
+
+    expect(conn.connection.stream.destroy).not.toHaveBeenCalled()
+  })
+})
+
+describe('heartbeat', () => {
+  it('should ask for a heartbeat', async () => {
+    const connection = new Connection('amqp://developer@localhost')
+
+    await connection.open()
+
+    expect(amqplib.connect).toHaveBeenCalledWith('amqp://developer@localhost?heartbeat=15',
+      expect.anything())
+  })
+
+  it('should append the heartbeat to an existing query', async () => {
+    const connection = new Connection('amqp://localhost?frameMax=8192')
+
+    await connection.open()
+
+    expect(amqplib.connect).toHaveBeenCalledWith('amqp://localhost?frameMax=8192&heartbeat=15',
+      expect.anything())
+  })
+
+  it.each(['amqp://localhost?heartbeat=5', 'amqp://localhost?heartbeat=0'])(
+    'should keep the heartbeat requested by the caller (%s)',
+    async (url) => {
+      const connection = new Connection(url)
+
+      await connection.open()
+
+      expect(amqplib.connect).toHaveBeenCalledWith(url, expect.anything())
+    })
+
+  it('should keep the socket alive', async () => {
+    await connection.open()
+
+    expect(amqplib.connect)
+      .toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ keepAlive: true }))
+  })
 })
 
 describe('close', () => {

package/test/decode.test.js

@@ -2,7 +2,6 @@
 
 const { randomBytes } = require('node:crypto')
 const { generate } = require('randomstring')
-const { pack } = require('msgpackr')
 
 const { decode } = require('../source/decode')
 
@@ -23,18 +22,6 @@ it('should decode application/json', async () => {
   expect(decoded).toStrictEqual(object)
 })
 
-it('should decode application/msgpack', async () => {
-  const object = { [generate()]: generate() }
-  const content = pack(object)
-  const contentType = 'application/msgpack'
-  const properties = { contentType }
-  const message = /** @type {comq.amqp.Message} */ { content, properties }
-
-  const decoded = decode(message)
-
-  expect(decoded).toStrictEqual(object)
-})
-
 it('should return buffer if content type is not defined', async () => {
   const object = { [generate()]: generate() }
   const json = JSON.stringify(object)

package/test/emitter.test.js

@@ -0,0 +1,71 @@
+'use strict'
+
+const { generate } = require('randomstring')
+const emitter = require('../source/emitter')
+
+/** @type {import('node:events').EventEmitter} */
+let instance
+
+beforeEach(() => {
+  instance = emitter.create()
+})
+
+it('should be', async () => {
+  expect(emitter.create).toBeInstanceOf(Function)
+})
+
+it('should emit', async () => {
+  const listener = jest.fn()
+  const event = generate()
+  const args = [generate(), generate()]
+
+  instance.on(event, listener)
+
+  expect(instance.emit(event, ...args)).toStrictEqual(true)
+  expect(listener).toHaveBeenCalledWith(...args)
+})
+
+it('should report an event no one listens to', async () => {
+  expect(instance.emit(generate())).toStrictEqual(false)
+})
+
+it('should not throw on error without listeners', async () => {
+  expect(() => instance.emit('error', new Error(generate()))).not.toThrow()
+})
+
+it('should not let a listener break the emission', async () => {
+  const event = generate()
+  const listener = jest.fn()
+
+  instance.on(event, () => { throw new Error(generate()) })
+  instance.on(event, listener)
+
+  expect(() => instance.emit(event)).not.toThrow()
+
+  // the listeners that follow a broken one are still called
+  expect(listener).toHaveBeenCalled()
+})
+
+it('should call a `once` listener once', async () => {
+  const event = generate()
+  const listener = jest.fn()
+
+  instance.once(event, listener)
+
+  instance.emit(event)
+  instance.emit(event)
+
+  expect(listener).toHaveBeenCalledTimes(1)
+})
+
+it('should not call a listener that has been removed', async () => {
+  const event = generate()
+  const listener = jest.fn()
+
+  instance.on(event, listener)
+  instance.off(event, listener)
+
+  instance.emit(event)
+
+  expect(listener).not.toHaveBeenCalled()
+})

package/test/encode.test.js

@@ -1,7 +1,6 @@
 'use strict'
 
 const { generate } = require('randomstring')
-const { pack } = require('msgpackr')
 
 const { encode } = require('../source/encode')
 
@@ -26,14 +25,6 @@ it('should encode as json', async () => {
   expect(buffer).toStrictEqual(expected)
 })
 
-it('should encode as msgpack', async () => {
-  const value = { [generate()]: generate() }
-  const packed = pack(value)
-  const buffer = encode(value, 'application/msgpack')
-
-  expect(buffer).toStrictEqual(packed)
-})
-
 /** @type {[comq.Encoding, string, Buffer][]} */
 const samples = [
   ['text/plain', 'some string', Buffer.from('some string')]

package/test/encodings.js

@@ -1,4 +1,4 @@
 'use strict'
 
 /** @type {comq.Encoding[]} */
-exports.encodings = ['application/msgpack', 'application/json']
+exports.encodings = ['application/json']

package/test/helpers.js

@@ -0,0 +1,77 @@
+'use strict'
+
+const timeout = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+
+const immediate = () => new Promise((resolve) => setImmediate(resolve))
+
+const random = (max = 100) => Math.floor(Math.random() * max)
+
+const flip = () => Math.random() < 0.5
+
+const sample = (array) => array[Math.floor(Math.random() * array.length)]
+
+const quantity = (input) => {
+  const matched = String(input).match(/^(\d+(?:\.\d+)?)([^\d\W]*)$/)
+
+  if (matched === null) {
+    throw new Error(`'${input}' doesn't look like a quantity of something`)
+  }
+
+  const number = +matched[1]
+  const suffix = matched[2]
+
+  if (suffix.length === 0) {
+    return number
+  }
+
+  const pair = MULTIPLIERS.find(([unit]) => suffix.startsWith(unit))
+
+  if (pair === undefined) {
+    throw new Error(`'${suffix}' doesn't look like a quantity unit`)
+  }
+
+  return number * pair[1]
+}
+
+const match = (reference, candidate) => {
+  if (typeof candidate !== typeof reference) {
+    return false
+  }
+
+  if (Array.isArray(candidate)) {
+    return Array.isArray(reference) &&
+      candidate.every((value) => reference.some((item) => match(item, value)))
+  }
+
+  if (typeof candidate === 'object') {
+    if (candidate === null) {
+      return reference === null
+    }
+
+    if (reference === null) {
+      return false
+    }
+
+    return Object.entries(candidate).every(([key, value]) => match(reference[key], value))
+  }
+
+  return reference === candidate
+}
+
+const MULTIPLIERS = Object.entries({
+  k: 1000,
+  Ki: 1024,
+  K: 1024,
+  Mi: 1024 ** 2,
+  M: 1000 ** 2,
+  Gi: 1024 ** 3,
+  G: 1000 ** 3
+})
+
+exports.timeout = timeout
+exports.immediate = immediate
+exports.random = random
+exports.flip = flip
+exports.sample = sample
+exports.quantity = quantity
+exports.match = match

package/test/io.close.test.js

@@ -2,7 +2,8 @@
 
 const { randomBytes } = require('node:crypto')
 const { generate } = require('randomstring')
-const { promex, immediate } = require('@toa.io/generic')
+const { Promex } = require('promex')
+const { immediate } = require('./helpers')
 
 const mock = require('./connection.mock')
 const { IO } = require('../source/io')
@@ -92,8 +93,29 @@ describe('close', () => {
     await expect(io.close()).resolves.not.toThrow()
   })
 
+  // a connection is shared and outlives its IOs, and it has a limited number of channels
+  it('should give the channels back', async () => {
+    await reply()
+    await io.close()
+
+    expect(requests.close).toHaveBeenCalled()
+    expect(events.close).toHaveBeenCalled()
+  })
+
+  it('should give them back before letting go of the connection', async () => {
+    await reply()
+
+    connection.close.mockImplementation(async () => {
+      expect(requests.close).toHaveBeenCalled()
+    })
+
+    await io.close()
+
+    expect.assertions(1)
+  })
+
   it('should wait for event processing completion', async () => {
-    const promise = /** @type {Promise<void>} */ promex()
+    const promise = /** @type {Promise<void>} */ new Promex()
     const queue = generate()
     const group = generate()
     const consumer = jest.fn(async () => promise)
@@ -128,7 +150,7 @@ describe('close', () => {
   })
 
   it('should wait for request processing completion', async () => {
-    const promise = /** @type {Promise<void>} */ promex()
+    const promise = /** @type {Promise<void>} */ new Promex()
     const producer = jest.fn(async () => promise)
 
     await reply(/** @type {Function} */ producer)

package/test/io.diagnostics.test.js

@@ -28,7 +28,7 @@ describe.each(['request', 'reply', 'event'])('%s channel events',
    * @param {comq.topology.type} type
    */
   (type) => {
-    it.each(['flow', 'drain', 'recover', 'discard', 'pause', 'resume'])('should re-emit %s',
+    it.each(['flow', 'drain', 'recover', 'discard', 'pause', 'resume', 'lost', 'remove'])('should re-emit %s',
       /**
        * @param {comq.diagnostics.Event} event
        */
@@ -61,7 +61,7 @@ it('should not throw on error without listeners', async () => {
   expect(() => emit(new Error(generate()))).not.toThrow()
 })
 
-it.each(['open', 'close', 'error', 'reconnect'])('should re-emit %s from connection',
+it.each(['open', 'close', 'error', 'reconnect', 'exhausted'])('should re-emit %s from connection',
   /**
    * @param {comq.diagnostics.Event} event
    */

package/test/io.reply.test.js

@@ -194,12 +194,12 @@ describe('encoding', () => {
 
   it('should set octet-stream for Buffer reply', async () => {
     const request = { [generate()]: generate() }
-    const content = encode(request, 'application/msgpack')
+    const content = encode(request, 'application/json')
 
     const properties = {
       replyTo: generate(),
       correlationId: generate(),
-      contentType: 'application/msgpack'
+      contentType: 'application/json'
     }
 
     const message = /** @type {comq.amqp.Message} */ { content, properties }

package/test/io.request.test.js

@@ -5,7 +5,7 @@
 const stream = require('node:stream')
 const { randomBytes } = require('node:crypto')
 const { generate } = require('randomstring')
-const { immediate } = require('@toa.io/generic')
+const { immediate } = require('./helpers')
 const { encode } = require('../source/encode')
 
 const mock = require('./connection.mock')
@@ -182,6 +182,20 @@ describe('send', () => {
     expect(requests.send).toHaveBeenCalledTimes(2)
   })
 
+  // a reply that could not be routed has been dropped by the broker
+  it('should resend unanswered Requests when the reply channel recovers', async () => {
+    expect(replies.diagnose).toHaveBeenCalledWith('recover', expect.any(Function))
+
+    const calls = replies.diagnose.mock.calls.filter((call) => call[0] === 'recover')
+    const listeners = calls.map((call) => call[1])
+
+    for (const listener of listeners) listener()
+
+    await immediate()
+
+    expect(requests.send).toHaveBeenCalledTimes(2)
+  })
+
   it('should resend unanswered Requests on sharded connection', async () => {
     jest.clearAllMocks()
 
@@ -338,7 +352,7 @@ describe('reply', () => {
       expect(output).toStrictEqual(content)
     })
 
-  const encodings = ['application/msgpack', 'application/json']
+  const encodings = ['application/json']
 
   it.each(encodings)('should decode %s',
     /**

package/test/pipeline.test.js

@@ -3,7 +3,7 @@
 const { EventEmitter } = require('node:events')
 const stream = require('node:stream')
 const { pipeline, transform } = require('../source/pipeline')
-const { timeout } = require('@toa.io/generic')
+const { timeout } = require('./helpers')
 
 /** @type {Channel} */
 let channel