comq 0.16.1 → 0.17.0

package/test/channel.test.js

@@ -971,3 +971,61 @@ const getCreatedChannel = (conn) => {
 
   return (conn ?? connection)[method].mock.results[0].value
 }
+
+describe('consumer tags', () => {
+  const queue = generate()
+  const consumer = jest.fn()
+
+  beforeEach(async () => {
+    channel = await create(connection, topology)
+    chan = await getCreatedChannel()
+  })
+
+  // the consumers of a lost channel went down with it, and their tags would pile
+  // up with every reconnection to be cancelled on a channel that never had them
+  it('should cancel the consumers of the current channel only', async () => {
+    await channel.consume(queue, consumer)
+
+    const replacement = await amqplib.connect()
+
+    await channel.recover(replacement)
+
+    const repl = await getCreatedChannel(replacement)
+
+    expect(repl.consume).toHaveBeenCalledTimes(1)
+
+    const { consumerTag } = await repl.consume.mock.results[0].value
+
+    await channel.seal()
+
+    expect(repl.cancel).toHaveBeenCalledTimes(1)
+    expect(repl.cancel).toHaveBeenCalledWith(consumerTag)
+  })
+
+  it('should let go of the recorded subscriptions once sealed', async () => {
+    await channel.consume(queue, consumer)
+    await channel.seal()
+
+    const replacement = await amqplib.connect()
+
+    await channel.recover(replacement)
+
+    const repl = await getCreatedChannel(replacement)
+
+    expect(repl.consume).not.toHaveBeenCalled()
+  })
+})
+
+describe('release', () => {
+  it('should be called once the channel is closed', async () => {
+    const release = jest.fn()
+
+    channel = await create(connection, topology, undefined, release)
+
+    expect(release).not.toHaveBeenCalled()
+
+    await channel.close()
+
+    expect(release).toHaveBeenCalledWith(channel)
+  })
+})

package/test/connection.mock.js

@@ -9,8 +9,11 @@ const channel = /** @type {jest.MockedFunction<(sharded?: boolean, index?: numbe
     send: jest.fn(async () => undefined),
     fire: jest.fn(async () => undefined),
     subscribe: jest.fn(async () => undefined),
+    bound: jest.fn(async () => undefined),
     publish: jest.fn(async () => undefined),
+    route: jest.fn(async () => undefined),
     diagnose: jest.fn(async () => undefined),
+    forget: jest.fn(() => undefined),
     seal: jest.fn(async () => undefined),
     close: jest.fn(async () => undefined),
     closed: false,
@@ -26,7 +29,8 @@ const connection = (sharded = false) => (/** @type {jest.MockedObject<comq.Conne
   createChannel: jest.fn(async (type, index) => channel(sharded, index)),
   open: jest.fn(async () => undefined),
   close: jest.fn(async () => undefined),
-  diagnose: jest.fn(() => undefined)
+  diagnose: jest.fn(() => undefined),
+  forget: jest.fn(() => undefined)
 })
 
 exports.connection = connection

package/test/connection.test.js

@@ -356,7 +356,7 @@ describe('create channel', () => {
       const preset = presets[type]
       const channel = await connection.createChannel(type)
 
-      expect(create).toHaveBeenCalledWith(conn, preset, undefined)
+      expect(create).toHaveBeenCalledWith(conn, preset, undefined, expect.any(Function))
       expect(channel).toStrictEqual(await create.mock.results[0].value)
     })
 
@@ -366,7 +366,7 @@ describe('create channel', () => {
 
     await connection.createChannel(type, index)
 
-    expect(create).toHaveBeenCalledWith(expect.anything(), expect.anything(), index)
+    expect(create).toHaveBeenCalledWith(expect.anything(), expect.anything(), index, expect.any(Function))
   })
 
   it('should create channel after exception', async () => {
@@ -467,6 +467,21 @@ describe('closed channels', () => {
 
     expect(channel.recover).toHaveBeenCalled()
   })
+
+  // a closed channel held by a shared connection would hold its IO until the
+  // connection happens to make another channel
+  it('should let go of one that has been given back', async () => {
+    const channel = await connection.createChannel('event')
+    const release = create.mock.calls[0][3]
+
+    release(channel)
+
+    conn.emit('close', new Error())
+
+    await timeout(50)
+
+    expect(channel.recover).not.toHaveBeenCalled()
+  })
 })
 
 describe('channel exhaustion', () => {
@@ -795,3 +810,20 @@ describe('diagnostics', () => {
     }
   })
 })
+
+describe('forget', () => {
+  afterEach(async () => {
+    await connection.close()
+  })
+
+  it('should stop reporting to the listener', async () => {
+    const listener = /** @type {Function} */ jest.fn()
+
+    await connection.diagnose('open', listener)
+    connection.forget('open', listener)
+
+    await connection.open()
+
+    expect(listener).not.toHaveBeenCalled()
+  })
+})

package/test/io.diagnostics.test.js

@@ -93,3 +93,14 @@ const findChannel = (type) => {
 
   return connection.createChannel.mock.results[index].value
 }
+
+// a connection is shared and outlives its IOs, so a listener left on it would hold the IO
+it('should forget the connection listeners on close', async () => {
+  await io.close()
+
+  for (const [event, listener] of connection.diagnose.mock.calls) {
+    expect(connection.forget).toHaveBeenCalledWith(event, listener)
+  }
+
+  expect(connection.forget).toHaveBeenCalledTimes(connection.diagnose.mock.calls.length)
+})

package/test/io.route.test.js

@@ -0,0 +1,94 @@
+'use strict'
+
+const { generate } = require('randomstring')
+
+const { decode } = require('../source/decode')
+const { encode } = require('../source/encode')
+
+const mock = require('./connection.mock')
+const { IO } = require('../source/io')
+
+/** @type {comq.IO} */
+let io
+
+/** @type {jest.MockedObject<comq.Connection>} */
+let connection
+
+/** @type {jest.MockedObject<comq.Channel>} */
+let events
+
+const exchange = generate()
+const queue = generate()
+const key = generate()
+const payload = generate()
+
+beforeEach(async () => {
+  jest.clearAllMocks()
+
+  connection = mock.connection()
+  io = new IO(connection)
+})
+
+describe('route', () => {
+  beforeEach(async () => {
+    await io.route(exchange, key, payload)
+
+    events = await connection.createChannel.mock.results[0].value
+  })
+
+  it('should create an event channel', async () => {
+    expect(connection.createChannel).toHaveBeenCalledWith('event')
+  })
+
+  it('should route to the exchange under the key', async () => {
+    expect(events.route).toHaveBeenCalledTimes(1)
+
+    const [name, routingKey, buffer, properties] = events.route.mock.calls[0]
+
+    expect(name).toStrictEqual(exchange)
+    expect(routingKey).toStrictEqual(key)
+    expect(decode({ content: buffer, properties })).toStrictEqual(payload)
+  })
+
+  it('should pass properties through', async () => {
+    jest.clearAllMocks()
+
+    const properties = { headers: { 'x-region': 'eu' }, persistent: true }
+
+    await io.route(exchange, key, payload, properties)
+
+    expect(events.route.mock.calls[0][3]).toMatchObject(properties)
+  })
+})
+
+describe('subscribe', () => {
+  const consumer = jest.fn()
+
+  it('should bind the queue under the key and consume it', async () => {
+    await io.subscribe(exchange, queue, key, consumer)
+
+    events = await connection.createChannel.mock.results[0].value
+
+    expect(events.bound).toHaveBeenCalledTimes(1)
+
+    const [name, bound, routingKey] = events.bound.mock.calls[0]
+
+    expect(name).toStrictEqual(exchange)
+    expect(bound).toStrictEqual(queue)
+    expect(routingKey).toStrictEqual(key)
+  })
+
+  it('should hand the payload to the consumer', async () => {
+    await io.subscribe(exchange, queue, key, consumer)
+
+    events = await connection.createChannel.mock.results[0].value
+
+    const deliver = events.bound.mock.calls[0][3]
+    const properties = { contentType: 'application/json' }
+    const message = { content: encode(payload, properties.contentType), properties }
+
+    await deliver(message)
+
+    expect(consumer).toHaveBeenCalledWith(payload, properties)
+  })
+})

package/test/memory.test.js

@@ -0,0 +1,273 @@
+'use strict'
+
+// Watches what a closed IO leaves behind. Every scenario keeps weak references to
+// what it has opened and closed, and expects the garbage collector to have taken
+// all of it; the heap growth per iteration is reported alongside.
+
+const v8 = require('node:v8')
+const vm = require('node:vm')
+const stream = require('node:stream')
+const { generate } = require('randomstring')
+const { timeout } = require('./helpers')
+const mock = require('./amqplib.mock')
+const { amqplib, connect } = mock
+
+jest.mock('amqplib', () => mock.amqplib)
+
+const { connect: open, assert } = require('../source')
+const { SingletonConnection } = require('../source/singleton')
+
+v8.setFlagsFromString('--expose-gc')
+
+const gc = vm.runInNewContext('gc')
+
+/** @type {{ scenario: string, iterations: number, watched: number, retained: number, growth: string }[]} */
+const report = []
+
+beforeEach(() => {
+  jest.clearAllMocks()
+  amqplib.connect.mockImplementation(async (url) => wired(await connect(url)))
+
+  global.COMQ_TESTING_SHUTDOWN_TIMEOUT = 1
+  global.COMQ_TESTING_WATCHDOG_INTERVAL = 60_000
+})
+
+afterEach(() => {
+  SingletonConnection.__lets_pretend_this_method_doesnt_exist()
+
+  delete global.COMQ_TESTING_SHUTDOWN_TIMEOUT
+  delete global.COMQ_TESTING_WATCHDOG_INTERVAL
+})
+
+afterAll(() => {
+  console.info('Memory report (heap growth is per iteration, after garbage collection)\n' +
+    formatted(report))
+})
+
+it('should let go of an IO and its channels once closed', async () => {
+  const retained = await measure('connect, reply, consume, emit, close', 100, async () => {
+    const io = await open(generate())
+
+    await io.reply(generate(), () => generate())
+    await io.consume(generate(), generate(), () => undefined)
+    await io.emit(generate(), { value: generate() })
+    await io.enqueue(generate(), { value: generate() })
+
+    const channels = await amqpChannels()
+
+    await io.close()
+
+    return [io, ...channels]
+  })
+
+  expect(retained).toStrictEqual(0)
+})
+
+// a singleton connection outlives every IO opened on it
+it('should let go of the IOs of a singleton connection', async () => {
+  const url = generate()
+
+  const retained = await measure('assert, reply, close (singleton)', 100, async () => {
+    const io = await assert(url)
+
+    await io.reply(generate(), () => generate())
+
+    const channels = await amqpChannels()
+
+    await io.close()
+
+    return [io, ...channels]
+  })
+
+  expect(retained).toStrictEqual(0)
+})
+
+it('should let go of the streams passed to request and emit', async () => {
+  const io = await open(generate())
+  const queue = generate()
+  const exchange = generate()
+
+  await io.reply(queue, (request) => request)
+
+  const retained = await measure('request and emit with a stream', 100, async () => {
+    const requests = stream.Readable.from([{ a: generate() }, { b: generate() }])
+    const events = stream.Readable.from([{ a: generate() }, { b: generate() }])
+
+    const replies = await io.request(queue, requests)
+
+    // eslint-disable-next-line no-void, no-unused-vars
+    for await (const _ of replies) void 0
+
+    await io.emit(exchange, events)
+
+    return [requests, events]
+  })
+
+  await io.close()
+
+  expect(retained).toStrictEqual(0)
+})
+
+it('should let go of the channels lost with a connection', async () => {
+  const io = await open(generate())
+
+  await io.reply(generate(), () => generate())
+  await io.consume(generate(), generate(), () => undefined)
+
+  const retained = await measure('reconnection', 50, async () => {
+    const before = await amqpChannels()
+    const conn = await amqplib.connect.mock.results.at(-1).value
+
+    let recovered = 0
+
+    const recovery = new Promise((resolve) => io.diagnose('recover', () => { if (++recovered === 3) resolve() }))
+
+    conn.emit('close', new Error(generate()))
+
+    await recovery
+
+    return before
+  })
+
+  await io.close()
+
+  expect(retained).toStrictEqual(0)
+})
+
+/**
+ * @param {string} scenario
+ * @param {number} iterations
+ * @param {(index: number) => Promise<object[]>} iteration returning what must be collected afterwards
+ * @return {Promise<number>} how many of the watched objects are still alive
+ */
+async function measure (scenario, iterations, iteration) {
+  /** @type {WeakRef<object>[]} */
+  const refs = []
+
+  await collect()
+
+  const before = process.memoryUsage().heapUsed
+
+  // in a frame of its own, so that nothing of the last iteration is left on the stack
+  await watch(refs, iterations, iteration)
+
+  // mocks remember what they returned, and spies are registered until restored,
+  // either of which would keep what they were made on alive
+  jest.clearAllMocks()
+  jest.restoreAllMocks()
+
+  await collect()
+
+  const after = process.memoryUsage().heapUsed
+  const retained = refs.filter((ref) => ref.deref() !== undefined).length
+  const growth = ((after - before) / iterations / 1024).toFixed(1) + ' KB'
+
+  report.push({ scenario, iterations, watched: refs.length, retained, growth })
+
+  return retained
+}
+
+/**
+ * @param {WeakRef<object>[]} refs
+ * @param {number} iterations
+ * @param {(index: number) => Promise<object[]>} iteration
+ */
+async function watch (refs, iterations, iteration) {
+  for (let i = 0; i < iterations; i++) {
+    const objects = await iteration(i)
+
+    for (const object of objects) refs.push(new WeakRef(object))
+  }
+}
+
+/**
+ * The channels amqplib has been asked for since the last call.
+ *
+ * @return {Promise<object[]>}
+ */
+async function amqpChannels () {
+  const channels = []
+
+  for (const { value } of amqplib.connect.mock.results) {
+    const conn = await value
+
+    for (const method of ['createChannel', 'createConfirmChannel']) {
+      for (const result of conn[method].mock.results) channels.push(await result.value)
+
+      conn[method].mockClear()
+    }
+  }
+
+  return channels
+}
+
+/**
+ * A broker of sorts: what is published to a queue is delivered to its consumer,
+ * so that requests are answered and streams of them come to an end.
+ *
+ * @param {jest.MockedObject<comq.amqp.Connection>} conn
+ * @return {jest.MockedObject<comq.amqp.Connection>}
+ */
+function wired (conn) {
+  const consumers = new Map()
+  const tags = new Map()
+
+  // a broker forgets the consumers of a connection that is gone
+  conn.on('close', () => consumers.clear())
+
+  for (const method of ['createChannel', 'createConfirmChannel']) {
+    const create = conn[method].getMockImplementation()
+
+    conn[method].mockImplementation(async () => {
+      const chan = await create()
+
+      chan.consume.mockImplementation(async (queue, callback) => {
+        const consumerTag = generate()
+
+        consumers.set(queue, callback)
+        tags.set(consumerTag, queue)
+
+        return { consumerTag }
+      })
+
+      chan.cancel.mockImplementation(async (consumerTag) => {
+        consumers.delete(tags.get(consumerTag))
+        tags.delete(consumerTag)
+      })
+
+      chan.publish.mockImplementation((exchange, routingKey, content, properties, resolve) => {
+        resolve?.(null)
+
+        const callback = consumers.get(routingKey)
+
+        if (callback !== undefined) setImmediate(callback, { content, properties, fields: { exchange, routingKey } })
+
+        return true
+      })
+
+      return chan
+    })
+  }
+
+  return conn
+}
+
+async function collect () {
+  // an object a WeakRef was made for in the current task is kept until the task ends
+  await timeout(0)
+
+  gc()
+  gc()
+
+  await timeout(0)
+
+  gc()
+}
+
+function formatted (rows) {
+  const columns = ['scenario', 'iterations', 'watched', 'retained', 'growth']
+  const width = (column) => Math.max(column.length, ...rows.map((row) => String(row[column]).length))
+  const line = (cells) => cells.map((cell, i) => String(cell).padEnd(width(columns[i]))).join('  ')
+
+  return [line(columns), ...rows.map((row) => line(columns.map((column) => row[column])))].join('\n')
+}

package/test/pipeline.test.js

@@ -78,4 +78,25 @@ class Channel extends EventEmitter {
   diagnose (event, callback) {
     this.on(event, callback)
   }
+
+  forget (event, callback) {
+    this.off(event, callback)
+  }
 }
+
+// the source must not stay a listener of the channel once the pipeline is done
+it('should let go of the source once done', async () => {
+  const input = stream.Readable.from([1, 2, 3])
+  const pipe = pipeline(input, (x) => x, channel)
+
+  expect(channel.listenerCount('pause')).toBe(1)
+  expect(channel.listenerCount('resume')).toBe(1)
+
+  // eslint-disable-next-line no-void, no-unused-vars
+  for await (const _ of pipe) void 0
+
+  await timeout(0)
+
+  expect(channel.listenerCount('pause')).toBe(0)
+  expect(channel.listenerCount('resume')).toBe(0)
+})

package/test/shards/connection.test.js

@@ -132,3 +132,18 @@ describe.each(/** @type {comq.diagnostics.Event[]} */ ['open', 'close'])('diagno
       expect(listener).toHaveBeenCalledWith(event, ...args, index)
     })
   })
+
+describe('forget', () => {
+  it('should stop re-emitting to the listener', async () => {
+    const listener = /** @type {Function} */ jest.fn()
+
+    connection.diagnose('open', listener)
+    connection.forget('open', listener)
+
+    const call = connections[0].diagnose.mock.calls.find((call) => call[0] === 'open')
+
+    call[1]()
+
+    expect(listener).not.toHaveBeenCalled()
+  })
+})

package/types/channel.d.ts

@@ -19,10 +19,16 @@ declare namespace comq {
 
     subscribe (exchange: string, queue: string, consumer: channels.Consumer): Promise<void>
 
+    /** consumes `queue`, bound to a routed `exchange` under `key` */
+    bound (exchange: string, queue: string, key: string, consumer: channels.Consumer): Promise<void>
+
     send (queue: string, buffer: Buffer, options?: Options.Publish): Promise<void>
 
     publish (exchange: string, buffer: Buffer, options?: Options.Publish): Promise<void>
 
+    /** publishes to a routed exchange under `key`, where `publish` fans out */
+    route (exchange: string, key: string, buffer: Buffer, options?: Options.Publish): Promise<void>
+
     fire (queue: string, buffer: Buffer, options?: Options.Publish): Promise<boolean>
 
     seal (): Promise<void>

package/types/connection.d.ts

@@ -18,6 +18,8 @@ declare namespace comq {
     createChannel(type: _topology.type, index: number): Promise<_channel.Channel>
 
     diagnose(event: _diagnostics.Event, listener: Function): void
+
+    forget(event: _diagnostics.Event, listener: Function): void
   }
 
   type Connect = (...urls: string[]) => Promise<_io.IO>

package/types/io.d.ts

@@ -10,8 +10,21 @@ declare namespace comq {
   type Producer<Input = any, Output = any> = (message: Input) => Output | Promise<Output>
   type Consumer<T = any> = (message: T, headers?: _amqp.Properties) => void | Promise<void>
 
-  interface ReplyEmitter extends EventEmitter {
+  type ReplyHandler = (payload: any, properties: _amqp.Properties, size?: number) => void
+
+  interface ReplyEmitter {
     readonly queue: string
+
+    /** A correlation identifier unique across every consumer of a producer. */
+    next(): string
+
+    on(correlationId: string, handler: ReplyHandler): void
+
+    off(correlationId: string, handler?: ReplyHandler): void
+
+    emit(correlationId: string, payload: any, properties: _amqp.Properties, size?: number): boolean
+
+    readonly pending: number
   }
 
   interface Destroyable extends EventEmitter {
@@ -19,7 +32,6 @@ declare namespace comq {
   }
 
   interface Request {
-    buffer: Buffer
     emitter: ReplyEmitter
     properties: _amqp.Properties
   }
@@ -51,6 +63,20 @@ declare namespace comq {
 
     enqueue(exchange: string, stream: Readable, properties?: _amqp.Properties): Promise<void>
 
+    /**
+     * Publishes to a routed (`direct`) exchange under a key, where `emit` fans out: the
+     * message reaches the queues bound under that key and no others.
+     */
+    route(exchange: string, key: string, payload: any, encoding?: _encoding.Encoding): Promise<void>
+
+    route(exchange: string, key: string, payload: any, properties?: _amqp.Properties): Promise<void>
+
+    /**
+     * Consumes a named durable queue bound to a routed exchange under a key, where `consume`
+     * takes everything published to a fanout.
+     */
+    subscribe<T = any>(exchange: string, queue: string, key: string, consumer: Consumer<T>): Promise<void>
+
     process<T = any>(queue: string, processor: Consumer<T>): Promise<void>
 
     seal(): Promise<void>