comq 0.16.1 → 0.17.0

package/source/io.js

@@ -1,7 +1,6 @@
 'use strict'
 
 const stream = require('node:stream')
-const { randomBytes } = require('node:crypto')
 const { setTimeout } = require('node:timers/promises')
 const { Promex } = require('promex')
 const { memo, failsafe, lazy, track } = require('./attributes')
@@ -38,6 +37,9 @@ class IO {
   /** @type {Map<Promex, comq.Request>} */
   #pendingReplies = new Map()
 
+  /** @type {[comq.diagnostics.Event, Function][]} */
+  #forwarders = []
+
   /** @type {Set<comq.Destroyable>} */
   #replyStreams = new Set()
 
@@ -53,7 +55,10 @@ class IO {
     this.#connection = connection
 
     for (const event of events.connection) {
-      this.#connection.diagnose(event, (...args) => this.#diagnostics.emit(event, ...args))
+      const forwarder = (...args) => this.#diagnostics.emit(event, ...args)
+
+      this.#connection.diagnose(event, forwarder)
+      this.#forwarders.push([event, forwarder])
     }
   }
 
@@ -87,10 +92,11 @@ class IO {
           )
         }
 
-        const request = this.#createRequest(queue, payload, encoding)
+        const [buffer, contentType] = this.#encode(payload, encoding)
+        const request = this.#createRequest(queue, contentType)
         const reply = this.#createReply(request)
 
-        await this.#requests.send(queue, request.buffer, request.properties)
+        await this.#requests.send(queue, buffer, request.properties)
 
         return reply
       }))
@@ -126,20 +132,46 @@ class IO {
         )
       }
 
-      /** @type {comq.amqp.options.Publish} */
-      const properties = {}
+      const [buffer, properties] = this.#properties(payload, encoding)
 
-      if (typeof encoding === 'object') { // properties passed
-        Object.assign(properties, encoding)
+      await this.#events[method](exchange, buffer, properties)
+    })
 
-        encoding = /** @type {comq.Encoding} */ properties.contentType
-      }
+  /**
+   * Publishes to a routed exchange under a key, where `emit` fans out. The exchange is
+   * `direct`, so a message reaches the queues bound under that key and no others.
+   */
+  route = lazy(this, this.#createEventChannel,
+    /**
+     * @param {string} exchange
+     * @param {string} key
+     * @param {any} payload
+     * @param {comq.Encoding | comq.amqp.options.Publish} [encoding]
+     * @returns {Promise<void>}
+     */
+    async (exchange, key, payload, encoding) => {
+      const [buffer, properties] = this.#properties(payload, encoding)
 
-      const [buffer, contentType] = this.#encode(payload, encoding)
+      await this.#events.route(exchange, key, buffer, properties)
+    })
 
-      properties.contentType = contentType
+  /**
+   * Consumes a named durable queue bound to a routed exchange under a key, where `consume`
+   * takes everything published to a fanout. The queue is named rather than derived, because
+   * what is bound to it is a key rather than an exchange.
+   */
+  subscribe = lazy(this, this.#createEventChannel,
+    /**
+     * @param {string} exchange
+     * @param {string} queue
+     * @param {string} key
+     * @param {comq.Consumer} callback
+     * @returns {Promise<void>}
+     */
+    async (exchange, queue, key, callback) => {
+      const consumer = this.#getEventConsumer(callback)
 
-      await this.#events[method](exchange, buffer, properties)
+      await this.#events.bound(exchange, queue, key, consumer)
     })
 
   process = lazy(this, this.#createEventChannel,
@@ -169,6 +201,11 @@ class IO {
     await Promise.all([this.#requests, this.#replies, this.#events]
       .map((channel) => channel?.close()))
 
+    // a connection that outlives its IO must not keep it as a listener
+    for (const [event, forwarder] of this.#forwarders) this.#connection.forget(event, forwarder)
+
+    this.#forwarders = []
+
     await this.#connection.close()
   })
 
@@ -189,6 +226,30 @@ class IO {
     this.#events = await this.#createChannel('event')
   }
 
+  /**
+   * What a publication carries, from what its caller passed as an encoding or as properties.
+   *
+   * @param {any} payload
+   * @param {comq.Encoding | comq.amqp.options.Publish} [encoding]
+   * @returns {[Buffer, comq.amqp.options.Publish]}
+   */
+  #properties (payload, encoding) {
+    /** @type {comq.amqp.options.Publish} */
+    const properties = {}
+
+    if (typeof encoding === 'object') { // properties passed
+      Object.assign(properties, encoding)
+
+      encoding = /** @type {comq.Encoding} */ properties.contentType
+    }
+
+    const [buffer, contentType] = this.#encode(payload, encoding)
+
+    properties.contentType = contentType
+
+    return [buffer, properties]
+  }
+
   async #consumeReplies (queue) {
     const emitter = io.createReplyEmitter(queue)
     const consumer = this.#getReplyConsumer(queue, emitter)
@@ -274,7 +335,7 @@ class IO {
     (message) => {
       const payload = decode(message)
 
-      emitter.emit(message.properties.correlationId, payload, message.properties)
+      emitter.emit(message.properties.correlationId, payload, message.properties, message.content.length)
     }
 
   /**
@@ -289,20 +350,21 @@ class IO {
     })
 
   /**
+   * The request holds no copy of what was sent: a retransmission encodes the
+   * payload anew, and an unanswered request would otherwise keep two of it.
+   *
    * @param {string} queue
-   * @param {any} payload
-   * @param {comq.Encoding} [encoding]
+   * @param {comq.Encoding} contentType
    * @return {comq.Request}
    */
-  #createRequest (queue, payload, encoding) {
-    const [buffer, contentType] = this.#encode(payload, encoding)
+  #createRequest (queue, contentType) {
     const emitter = this.#emitters.get(queue)
-    const correlationId = randomBytes(8).toString('hex')
+    const correlationId = emitter.next()
 
     /** @type {comq.amqp.Properties} */
     const properties = { contentType, correlationId, replyTo: emitter.queue }
 
-    return { buffer, emitter, properties }
+    return { emitter, properties }
   }
 
   /**
@@ -312,7 +374,7 @@ class IO {
   #createReply (request) {
     const reply = this.#createPendingReply(request)
 
-    request.emitter.once(request.properties.correlationId, this.#getReplyResolver(request, reply))
+    request.emitter.on(request.properties.correlationId, this.#getReplyResolver(request, reply))
 
     return reply
   }
@@ -323,12 +385,12 @@ class IO {
    */
   #createPendingReply (request) {
     const reply = new Promex()
+    const settled = () => this.#pendingReplies.delete(reply)
 
     this.#pendingReplies.set(reply, request)
 
-    reply
-      .catch(noop)
-      .finally(() => this.#pendingReplies.delete(reply))
+    // one derived promise per request, and a rejection handled with it
+    reply.then(settled, settled)
 
     return reply
   }
@@ -338,11 +400,14 @@ class IO {
    * @param reply
    */
   #getReplyResolver (request, reply) {
-    return async (payload, properties) => {
+    return async (payload, properties, size) => {
       const isStream = properties.headers?.index !== undefined
 
+      // a reply is answered once; a reply stream takes the place of this resolver
+      request.emitter.off(request.properties.correlationId)
+
       if (isStream) {
-        const stream = this.#createReplyStream(request, payload, properties)
+        const stream = this.#createReplyStream(request, payload, properties, size)
 
         try {
           await stream.confirmation
@@ -358,10 +423,10 @@ class IO {
     }
   }
 
-  #createReplyStream (request, payload, properties) {
+  #createReplyStream (request, payload, properties, size) {
     const stream = new io.ReplyStream(request, this.#reply.bind(this))
 
-    stream.arrange(payload, properties)
+    stream.arrange(payload, properties, size)
     this.#addReplyStream(/** @type {comq.Destroyable} */ stream)
 
     return stream
@@ -453,7 +518,7 @@ class IO {
     for (const [reply, request] of this.#pendingReplies) {
       // detaching this attempt alone leaves the listeners of the reply streams
       // that are still flowing over the other shards in place
-      request.emitter.removeAllListeners(request.properties.correlationId)
+      request.emitter.off(request.properties.correlationId)
 
       // trigger failsafe attribute
       reply.reject(RETRANSMISSION)
@@ -484,6 +549,4 @@ const DEFAULT = 'application/json'
 
 const RETRANSMISSION = /** @type {Error} */ Symbol('retransmission')
 
-function noop () {}
-
 exports.IO = IO

package/source/pipeline.js

@@ -5,12 +5,20 @@ const stream = require('node:stream/promises')
 
 function pipeline (source, transform, channel) {
   const destination = new Pipeline(transform)
+  const pause = source.pause.bind(source)
+  const resume = source.resume.bind(source)
 
-  // eslint-disable-next-line no-void
-  void stream.pipeline(source, destination)
+  channel.diagnose('pause', pause)
+  channel.diagnose('resume', resume)
 
-  channel.diagnose('pause', source.pause.bind(source))
-  channel.diagnose('resume', source.resume.bind(source))
+  // the source must not outlive the pipeline as a listener of the channel;
+  // a failure is delivered through the destination, the promise says nothing new
+  const detach = () => {
+    channel.forget('pause', pause)
+    channel.forget('resume', resume)
+  }
+
+  stream.pipeline(source, destination).then(detach, detach)
 
   return destination
 }

package/source/shards/connection.js

@@ -46,6 +46,10 @@ class Connection {
     this.#diagnostics.on(event, listener)
   }
 
+  forget (event, listener) {
+    this.#diagnostics.off(event, listener)
+  }
+
   /**
    * @param {comq.Connection} connection
    * @param {number} index

package/test/ReplyPipe.test.js

@@ -0,0 +1,170 @@
+'use strict'
+
+const { Readable } = require('node:stream')
+const { Promex } = require('promex')
+const { ReplyPipe } = require('../source/.io/ReplyPipe')
+const { createReplyEmitter } = require('../source/.io/createReplyEmitter')
+const { control, FLOW_HEADER } = require('../source/.io/const')
+const { timeout } = require('./helpers')
+
+/** @type {jest.Mock} */
+let reply
+
+/** @type {[any, comq.amqp.options.Publish][]} */
+let sent
+
+let channel
+
+/** @type {comq.ReplyEmitter} */
+let feedback
+
+let request
+
+beforeEach(() => {
+  global.COMQ_TESTING_HEARTBEAT_INTERVAL = 60_000
+
+  sent = []
+  reply = jest.fn(async (message, properties) => {
+    sent.push([message, properties])
+
+    return true
+  })
+
+  channel = { diagnose: jest.fn(), forget: jest.fn() }
+  feedback = createReplyEmitter('control')
+  request = { properties: { correlationId: 'test-correlation', replyTo: 'replies' } }
+})
+
+afterEach(() => {
+  delete global.COMQ_TESTING_HEARTBEAT_INTERVAL
+})
+
+const messages = () => sent.map(([message]) => message)
+
+const closing = (pipe) => new Promise((resolve) => pipe.on('close', resolve))
+
+it('should confirm, announcing flow control', async () => {
+  const pipe = await ReplyPipe.create(request, Readable.from([]), channel, feedback, reply)
+
+  expect(sent[0][0]).toStrictEqual(control.ok)
+  expect(sent[0][1].headers).toStrictEqual({ index: 0, [FLOW_HEADER]: true })
+
+  await closing(pipe)
+})
+
+it('should transmit values in order, then end', async () => {
+  const pipe = await ReplyPipe.create(request, Readable.from([1, 2, 3]), channel, feedback, reply)
+
+  await closing(pipe)
+
+  expect(messages()).toStrictEqual([control.ok, 1, 2, 3, control.end])
+  expect(sent.map(([, properties]) => properties.headers.index)).toStrictEqual([0, 1, 2, 3, 4])
+})
+
+// a source listened to runs at its own pace, and its output piles up in front of a paused channel
+it('should pull the source no faster than the channel takes', async () => {
+  const gate = new Promex()
+
+  let pulled = 0
+
+  reply.mockImplementation(async (message) => {
+    if (message !== control.ok) await gate
+
+    return true
+  })
+
+  function * source () {
+    for (let i = 0; i < 100; i++) {
+      pulled++
+
+      yield i
+    }
+  }
+
+  const pipe = await ReplyPipe.create(request, Readable.from(source()), channel, feedback, reply)
+
+  await timeout(10)
+
+  // a Readable reads ahead by no more than its high water mark
+  expect(pulled).toBeLessThanOrEqual(20)
+
+  gate.resolve()
+
+  await closing(pipe)
+
+  expect(pulled).toStrictEqual(100)
+})
+
+it('should hold the source while the consumer has asked for a pause', async () => {
+  const pipe = await ReplyPipe.create(request, Readable.from([1, 2, 3]), channel, feedback, reply)
+
+  feedback.emit(request.properties.correlationId, control.pause, {})
+
+  await timeout(10)
+
+  expect(messages()).toStrictEqual([control.ok])
+
+  feedback.emit(request.properties.correlationId, control.resume, {})
+
+  await closing(pipe)
+
+  expect(messages()).toStrictEqual([control.ok, 1, 2, 3, control.end])
+})
+
+it('should stop once the consumer has ended the stream', async () => {
+  const gate = new Promex()
+
+  reply.mockImplementation(async (message, properties) => {
+    sent.push([message, properties])
+
+    if (message !== control.ok) await gate
+
+    return true
+  })
+
+  const pipe = await ReplyPipe.create(request, Readable.from([1, 2, 3]), channel, feedback, reply)
+  const closed = closing(pipe)
+
+  feedback.emit(request.properties.correlationId, control.end, {})
+
+  await closed
+
+  gate.resolve()
+
+  await timeout(10)
+
+  expect(messages()).not.toContain(control.end)
+  expect(messages()).not.toContain(3)
+  expect(channel.forget).toHaveBeenCalledWith('return', expect.any(Function))
+  expect(feedback.pending).toStrictEqual(0)
+})
+
+it('should not heartbeat once the confirmation could not be delivered', async () => {
+  global.COMQ_TESTING_HEARTBEAT_INTERVAL = 5
+
+  reply.mockImplementation(async (message, properties) => {
+    sent.push([message, properties])
+
+    return false
+  })
+
+  const pipe = new ReplyPipe(request, Readable.from([1]), channel, feedback, reply)
+
+  await pipe.pipe()
+  await timeout(30)
+
+  expect(messages()).toStrictEqual([control.ok])
+})
+
+it('should heartbeat while idle', async () => {
+  global.COMQ_TESTING_HEARTBEAT_INTERVAL = 5
+
+  const source = new Readable({ objectMode: true, read () {} })
+  const pipe = await ReplyPipe.create(request, source, channel, feedback, reply)
+
+  await timeout(30)
+
+  expect(messages()).toContain(control.heartbeat)
+
+  pipe.destroy()
+})

package/test/ReplyStream.test.js

@@ -1,9 +1,9 @@
 'use strict'
 
-const { EventEmitter } = require('node:events')
 const { once } = require('node:events')
 const { ReplyStream, UNCONFIRMED } = require('../source/.io/ReplyStream')
-const { control } = require('../source/.io/const')
+const { createReplyEmitter } = require('../source/.io/createReplyEmitter')
+const { control, FLOW_HEADER } = require('../source/.io/const')
 
 /** @type {jest.Mock} */
 let reply
@@ -18,7 +18,7 @@ beforeEach(() => {
   reply = jest.fn().mockResolvedValue(true)
 
   const request = {
-    emitter: new EventEmitter(),
+    emitter: createReplyEmitter('test'),
     properties: { correlationId: 'test-correlation' }
   }
 
@@ -63,7 +63,7 @@ it('should reject confirmation when nothing arrives within the idle interval', a
   global.COMQ_TESTING_IDLE_INTERVAL = 10
 
   const request = {
-    emitter: new EventEmitter(),
+    emitter: createReplyEmitter('test'),
     properties: { correlationId: 'idle-correlation' }
   }
 
@@ -81,3 +81,81 @@ it('should keep confirmation resolved after control.ok', async () => {
 
   await expect(stream.confirmation).resolves.toBeUndefined()
 })
+
+describe('flow', () => {
+  const TOTAL = 40 // well beyond the 16 values a Readable holds in object mode
+
+  /**
+   * @param {boolean} flow whether the producer honours pause and resume
+   */
+  const ok = (flow) => ({ headers: { index: 0, [FLOW_HEADER]: flow }, type: 'control' })
+
+  const send = () => {
+    for (let index = 1; index <= TOTAL; index++) stream.arrange(index, { headers: { index } })
+
+    stream.arrange(control.end, { headers: { index: TOTAL + 1 }, type: 'control' })
+  }
+
+  // dropping the listener once behind would leave the stream waiting for values that have passed
+  it('should keep receiving while the consumer is behind', async () => {
+    stream.arrange(control.ok, ok(false))
+    send()
+
+    const received = []
+
+    for await (const value of stream) received.push(value)
+
+    expect(received).toHaveLength(TOTAL)
+    expect(received[TOTAL - 1]).toStrictEqual(TOTAL)
+  })
+
+  it('should not ask a producer that knows nothing of pause', async () => {
+    stream.arrange(control.ok, ok(false))
+    send()
+
+    expect(reply).not.toHaveBeenCalled()
+  })
+
+  it('should ask the producer to pause once behind, and to resume once caught up', async () => {
+    stream.arrange(control.ok, ok(true))
+
+    for (let index = 1; index <= TOTAL; index++) stream.arrange(index, { headers: { index } })
+
+    expect(reply).toHaveBeenCalledTimes(1)
+    expect(reply).toHaveBeenCalledWith({ properties: ok(true) }, control.pause)
+
+    const drained = new Promise((resolve) => stream.on('data', (value) => { if (value === TOTAL) resolve() }))
+
+    await drained
+
+    expect(reply).toHaveBeenCalledTimes(2)
+    expect(reply).toHaveBeenLastCalledWith({ properties: ok(true) }, control.resume)
+  })
+
+  it('should bound the values held out of order by their size', async () => {
+    global.COMQ_TESTING_MAX_BUFFER_SIZE = 1000
+    global.COMQ_TESTING_MAX_BUFFER_BYTES = 100
+
+    try {
+      const request = {
+        emitter: createReplyEmitter('test'),
+        properties: { correlationId: 'bytes-correlation' }
+      }
+
+      const held = new ReplyStream(request, reply)
+      const closed = once(held, 'close')
+
+      held.arrange(1, { headers: { index: 1 } }, 60)
+
+      expect(held.destroyed).toBe(false)
+
+      held.arrange(2, { headers: { index: 2 } }, 60)
+
+      await closed
+
+      expect(held.destroyed).toBe(true)
+    } finally {
+      delete global.COMQ_TESTING_MAX_BUFFER_BYTES
+    }
+  })
+})

package/test/channel.routed.test.js

@@ -0,0 +1,119 @@
+'use strict'
+
+const { generate } = require('randomstring')
+
+const { amqplib } = require('./amqplib.mock')
+const fixtures = require('./channel.fixtures')
+const { create } = require('../source/channel')
+
+/** @type {jest.MockedObject<comq.amqp.Connection>} */
+let connection
+
+/** @type {comq.Topology} */
+let topology
+
+/** @type {comq.Channel} */
+let channel
+
+/** @type {jest.MockedObject<comq.amqp.Channel>} */
+let chan
+
+const exchange = generate()
+const queue = generate()
+const key = generate()
+
+const getCreatedChannel = () => {
+  const method = `create${topology.confirms ? 'Confirm' : ''}Channel`
+
+  return connection[method].mock.results[0].value
+}
+
+beforeEach(async () => {
+  jest.clearAllMocks()
+
+  connection = await amqplib.connect()
+  topology = fixtures.preset()
+  channel = await create(connection, topology)
+  chan = await getCreatedChannel()
+})
+
+describe('route', () => {
+  const buffer = Buffer.from(generate())
+
+  it('should assert a direct exchange', async () => {
+    await channel.route(exchange, key, buffer)
+
+    expect(chan.assertExchange).toHaveBeenCalledTimes(1)
+
+    const [name, type] = chan.assertExchange.mock.calls[0]
+
+    expect(name).toStrictEqual(exchange)
+    expect(type).toStrictEqual('direct')
+  })
+
+  it('should publish with the key as the routing key', async () => {
+    await channel.route(exchange, key, buffer)
+
+    const [name, routingKey, published] = chan.publish.mock.calls[0]
+
+    expect(name).toStrictEqual(exchange)
+    expect(routingKey).toStrictEqual(key)
+    expect(published).toStrictEqual(buffer)
+  })
+
+  it('should assert the exchange once for many keys', async () => {
+    await channel.route(exchange, key, buffer)
+    await channel.route(exchange, generate(), buffer)
+
+    expect(chan.assertExchange).toHaveBeenCalledTimes(1)
+    expect(chan.publish).toHaveBeenCalledTimes(2)
+  })
+})
+
+describe('bound', () => {
+  const consumer = /** @type {comq.channels.Consumer} */ jest.fn(() => undefined)
+
+  it('should assert a direct exchange', async () => {
+    await channel.bound(exchange, queue, key, consumer)
+
+    const [name, type] = chan.assertExchange.mock.calls[0]
+
+    expect(name).toStrictEqual(exchange)
+    expect(type).toStrictEqual('direct')
+  })
+
+  it('should assert the queue by name', async () => {
+    await channel.bound(exchange, queue, key, consumer)
+
+    expect(chan.assertQueue).toHaveBeenCalledWith(queue, expect.anything())
+  })
+
+  it('should bind the queue under the key', async () => {
+    await channel.bound(exchange, queue, key, consumer)
+
+    const { queue: asserted } = await chan.assertQueue.mock.results[0].value
+
+    expect(chan.bindQueue).toHaveBeenCalledTimes(1)
+    expect(chan.bindQueue).toHaveBeenCalledWith(asserted, exchange, key)
+  })
+
+  it('should consume the queue', async () => {
+    await channel.bound(exchange, queue, key, consumer)
+
+    expect(chan.consume).toHaveBeenCalledWith(queue, expect.any(Function), expect.anything())
+  })
+
+  it('should bind once per key', async () => {
+    await channel.bound(exchange, queue, key, consumer)
+    await channel.bound(exchange, queue, key, consumer)
+
+    expect(chan.bindQueue).toHaveBeenCalledTimes(1)
+  })
+
+  it('should bind again for another key', async () => {
+    await channel.bound(exchange, queue, key, consumer)
+    await channel.bound(exchange, generate(), generate(), consumer)
+
+    expect(chan.bindQueue).toHaveBeenCalledTimes(2)
+  })
+})