comq 0.16.0 → 0.16.2

package/source/attributes/recall.js

@@ -36,7 +36,24 @@ async function replay (context) {
   await Promise.all(promises)
 }
 
+/**
+ * Forgets the recorded calls, and the arguments they hold on to.
+ *
+ * @param {object} context
+ */
+function reset (context) {
+  const methods = context[METHODS]
+
+  if (methods === undefined) return
+
+  for (const method of methods) method[CALLS] = undefined
+
+  context[METHODS] = undefined
+}
+
 const METHODS = Symbol('context methods')
 const CALLS = Symbol('method calls')
 
+recall.reset = reset
+
 exports.recall = recall

package/source/attributes/recall.test.js

@@ -90,3 +90,46 @@ it('should not re-call those thrown exceptions', async () => {
 
   expect(method).toHaveBeenCalledTimes(3)
 })
+
+describe('reset', () => {
+  it('should be', async () => {
+    expect(recall.reset).toBeDefined()
+  })
+
+  it('should forget recorded calls', async () => {
+    const context = {}
+    const func = recall(context, method)
+
+    await func(generate())
+
+    recall.reset(context)
+    method.mockClear()
+
+    await recall(context)
+
+    expect(method).not.toHaveBeenCalled()
+  })
+
+  it('should record again after reset', async () => {
+    const context = {}
+    const func = recall(context, method)
+
+    await func(generate())
+
+    recall.reset(context)
+
+    const args = [generate()]
+
+    await func(...args)
+    method.mockClear()
+
+    await recall(context)
+
+    expect(method).toHaveBeenCalledTimes(1)
+    expect(method).toHaveBeenCalledWith(...args)
+  })
+
+  it('should not throw on a context without records', async () => {
+    expect(() => recall.reset({})).not.toThrow()
+  })
+})

package/source/channel.js

@@ -44,17 +44,22 @@ class Channel {
 
   #closed = false
 
+  /** @type {(channel: comq.Channel) => void} */
+  #release
+
   /**
    * @param {comq.amqp.Connection} connection
    * @param {comq.Topology} topology
-   * @param {number} index
+   * @param {number} [index]
+   * @param {(channel: comq.Channel) => void} [release] called once the channel is given back
    */
-  constructor (connection, topology, index) {
+  constructor (connection, topology, index, release = noop) {
     this.index = index
 
     this.#connection = connection
     this.#topology = topology
     this.#failfast = index !== undefined
+    this.#release = release
 
     if (this.#failfast) failsafe.disable(this.send, this.publish)
   }
@@ -63,6 +68,9 @@ class Channel {
     if (this.#topology.confirms) this.#channel = await this.#connection.createConfirmChannel()
     else this.#channel = await this.#connection.createChannel()
 
+    // the consumers of the previous channel went down with it, their tags mean nothing here
+    this.#tags = []
+
     await this.#channel.prefetch(this.#topology.prefetch)
 
     this.#channel.on('drain', this.#unpause)
@@ -141,6 +149,8 @@ class Channel {
 
     // a channel that went down with its connection is the outcome this asks for
     await this.#channel?.close().catch(noop)
+
+    this.#release(this)
   }
 
   async seal () {
@@ -149,6 +159,10 @@ class Channel {
     const cancellations = this.#tags.map((tag) => this.#channel.cancel(tag))
 
     await Promise.all(cancellations).catch(noop) // won't recover anyway
+
+    // a sealed channel is not going to consume again, so what it has consumed
+    // and the callbacks it was given are of no use anymore
+    recall.reset(this)
   }
 
   diagnose (event, listener) {
@@ -260,12 +274,12 @@ class Channel {
    */
   #confirmation () {
     const confirmation = new Promex()
+    const settled = () => this.#confirmations.delete(confirmation)
 
     this.#confirmations.add(confirmation)
 
-    confirmation
-      .catch(noop) // have no idea why, but this is required
-      .finally(() => this.#confirmations.delete(confirmation))
+    // one derived promise per publish, and a rejection handled with it
+    confirmation.then(settled, settled)
 
     return confirmation
   }
@@ -372,10 +386,11 @@ class Channel {
  * @param {comq.amqp.Connection} connection
  * @param {comq.Topology} topology
  * @param {number} [index]
+ * @param {(channel: comq.Channel) => void} [release]
  * @return {Promise<comq.Channel>}
  */
-async function create (connection, topology, index) {
-  const channel = new Channel(connection, topology, index)
+async function create (connection, topology, index, release) {
+  const channel = new Channel(connection, topology, index, release)
 
   await channel.create()
 

package/source/connection.js

@@ -20,8 +20,8 @@ class Connection {
   /** @type {comq.amqp.Connection} */
   #connection
 
-  /** @type {comq.Channel[]} */
-  #channels = []
+  /** @type {Set<comq.Channel>} */
+  #channels = new Set()
 
   /** @type {Promex} */
   #recovery = new Promex()
@@ -64,17 +64,10 @@ class Connection {
   async open () {
     this.#closed = false
 
-    if (this.#opening !== null) return this.#opening
-
-    this.#opening = retry(this.#open).finally(() => { this.#opening = null })
-
-    await this.#opening
+    await this.#reopen()
 
     if (this.#closed) return
 
-    // close may have landed after this attempt succeeded but before `#opening` was cleared
-    if (this.#connection === undefined) return this.open()
-
     this.#running = true
   }
 
@@ -97,10 +90,13 @@ class Connection {
       if (this.#connection === undefined) await this.#recovery
 
       const topology = presets[type]
-      const channel = await channels.create(this.#connection, topology, index)
 
-      this.#channels = this.#channels.filter((channel) => !channel.closed)
-      this.#channels.push(channel)
+      // a closed channel held here would hold its IO, and a shared connection is
+      // in no hurry to make another channel that would have swept it out
+      const release = (channel) => this.#channels.delete(channel)
+      const channel = await channels.create(this.#connection, topology, index, release)
+
+      this.#channels.add(channel)
 
       return channel
     })
@@ -109,6 +105,10 @@ class Connection {
     this.#diagnostics.on(event, listener)
   }
 
+  forget (event, listener) {
+    this.#diagnostics.off(event, listener)
+  }
+
   #open = async (retry) => {
     if (this.#closed) return
 
@@ -141,9 +141,10 @@ class Connection {
     this.#diagnostics.emit('open')
 
     try {
-      this.#channels = this.#channels.filter((channel) => !channel.closed)
-
-      for (const channel of this.#channels) await channel.recover(connection)
+      for (const channel of this.#channels) {
+        if (channel.closed) this.#channels.delete(channel)
+        else await channel.recover(connection)
+      }
     } catch (exception) {
       this.#diagnostics.emit('error', exception)
       this.#drop(connection)
@@ -172,10 +173,29 @@ class Connection {
     this.#connection = undefined
 
     if (error !== undefined && !this.#closed) {
-      this.open().catch((exception) => this.#diagnostics.emit('error', exception))
+      this.#reopen().catch((exception) => this.#diagnostics.emit('error', exception))
     }
   }
 
+  /**
+   * Establishes the connection, or joins the attempt under way. A lost connection is
+   * restored through here and not through `open()`: what a subclass makes of `open()` is
+   * its own — the singleton answers it with the first opening, made once and remembered,
+   * which is no way to make a second one.
+   *
+   * @return {Promise<void>}
+   */
+  async #reopen () {
+    this.#opening ??= retry(this.#open).finally(() => { this.#opening = null })
+
+    await this.#opening
+
+    if (this.#closed) return
+
+    // close may have landed after this attempt succeeded but before `#opening` was cleared
+    if (this.#connection === undefined) return this.#reopen()
+  }
+
   /**
    * An AMQP connection is only closed once the broker has replied with Close-Ok,
    * which never happens on a connection that has already been lost.

package/source/io.js

@@ -1,7 +1,6 @@
 'use strict'
 
 const stream = require('node:stream')
-const { randomBytes } = require('node:crypto')
 const { setTimeout } = require('node:timers/promises')
 const { Promex } = require('promex')
 const { memo, failsafe, lazy, track } = require('./attributes')
@@ -38,6 +37,9 @@ class IO {
   /** @type {Map<Promex, comq.Request>} */
   #pendingReplies = new Map()
 
+  /** @type {[comq.diagnostics.Event, Function][]} */
+  #forwarders = []
+
   /** @type {Set<comq.Destroyable>} */
   #replyStreams = new Set()
 
@@ -53,7 +55,10 @@ class IO {
     this.#connection = connection
 
     for (const event of events.connection) {
-      this.#connection.diagnose(event, (...args) => this.#diagnostics.emit(event, ...args))
+      const forwarder = (...args) => this.#diagnostics.emit(event, ...args)
+
+      this.#connection.diagnose(event, forwarder)
+      this.#forwarders.push([event, forwarder])
     }
   }
 
@@ -87,10 +92,11 @@ class IO {
           )
         }
 
-        const request = this.#createRequest(queue, payload, encoding)
+        const [buffer, contentType] = this.#encode(payload, encoding)
+        const request = this.#createRequest(queue, contentType)
         const reply = this.#createReply(request)
 
-        await this.#requests.send(queue, request.buffer, request.properties)
+        await this.#requests.send(queue, buffer, request.properties)
 
         return reply
       }))
@@ -169,6 +175,11 @@ class IO {
     await Promise.all([this.#requests, this.#replies, this.#events]
       .map((channel) => channel?.close()))
 
+    // a connection that outlives its IO must not keep it as a listener
+    for (const [event, forwarder] of this.#forwarders) this.#connection.forget(event, forwarder)
+
+    this.#forwarders = []
+
     await this.#connection.close()
   })
 
@@ -274,7 +285,7 @@ class IO {
     (message) => {
       const payload = decode(message)
 
-      emitter.emit(message.properties.correlationId, payload, message.properties)
+      emitter.emit(message.properties.correlationId, payload, message.properties, message.content.length)
     }
 
   /**
@@ -289,20 +300,21 @@ class IO {
     })
 
   /**
+   * The request holds no copy of what was sent: a retransmission encodes the
+   * payload anew, and an unanswered request would otherwise keep two of it.
+   *
    * @param {string} queue
-   * @param {any} payload
-   * @param {comq.Encoding} [encoding]
+   * @param {comq.Encoding} contentType
    * @return {comq.Request}
    */
-  #createRequest (queue, payload, encoding) {
-    const [buffer, contentType] = this.#encode(payload, encoding)
+  #createRequest (queue, contentType) {
     const emitter = this.#emitters.get(queue)
-    const correlationId = randomBytes(8).toString('hex')
+    const correlationId = emitter.next()
 
     /** @type {comq.amqp.Properties} */
     const properties = { contentType, correlationId, replyTo: emitter.queue }
 
-    return { buffer, emitter, properties }
+    return { emitter, properties }
   }
 
   /**
@@ -312,7 +324,7 @@ class IO {
   #createReply (request) {
     const reply = this.#createPendingReply(request)
 
-    request.emitter.once(request.properties.correlationId, this.#getReplyResolver(request, reply))
+    request.emitter.on(request.properties.correlationId, this.#getReplyResolver(request, reply))
 
     return reply
   }
@@ -323,12 +335,12 @@ class IO {
    */
   #createPendingReply (request) {
     const reply = new Promex()
+    const settled = () => this.#pendingReplies.delete(reply)
 
     this.#pendingReplies.set(reply, request)
 
-    reply
-      .catch(noop)
-      .finally(() => this.#pendingReplies.delete(reply))
+    // one derived promise per request, and a rejection handled with it
+    reply.then(settled, settled)
 
     return reply
   }
@@ -338,11 +350,14 @@ class IO {
    * @param reply
    */
   #getReplyResolver (request, reply) {
-    return async (payload, properties) => {
+    return async (payload, properties, size) => {
       const isStream = properties.headers?.index !== undefined
 
+      // a reply is answered once; a reply stream takes the place of this resolver
+      request.emitter.off(request.properties.correlationId)
+
       if (isStream) {
-        const stream = this.#createReplyStream(request, payload, properties)
+        const stream = this.#createReplyStream(request, payload, properties, size)
 
         try {
           await stream.confirmation
@@ -358,10 +373,10 @@ class IO {
     }
   }
 
-  #createReplyStream (request, payload, properties) {
+  #createReplyStream (request, payload, properties, size) {
     const stream = new io.ReplyStream(request, this.#reply.bind(this))
 
-    stream.arrange(payload, properties)
+    stream.arrange(payload, properties, size)
     this.#addReplyStream(/** @type {comq.Destroyable} */ stream)
 
     return stream
@@ -453,7 +468,7 @@ class IO {
     for (const [reply, request] of this.#pendingReplies) {
       // detaching this attempt alone leaves the listeners of the reply streams
       // that are still flowing over the other shards in place
-      request.emitter.removeAllListeners(request.properties.correlationId)
+      request.emitter.off(request.properties.correlationId)
 
       // trigger failsafe attribute
       reply.reject(RETRANSMISSION)
@@ -484,6 +499,4 @@ const DEFAULT = 'application/json'
 
 const RETRANSMISSION = /** @type {Error} */ Symbol('retransmission')
 
-function noop () {}
-
 exports.IO = IO

package/source/pipeline.js

@@ -5,12 +5,20 @@ const stream = require('node:stream/promises')
 
 function pipeline (source, transform, channel) {
   const destination = new Pipeline(transform)
+  const pause = source.pause.bind(source)
+  const resume = source.resume.bind(source)
 
-  // eslint-disable-next-line no-void
-  void stream.pipeline(source, destination)
+  channel.diagnose('pause', pause)
+  channel.diagnose('resume', resume)
 
-  channel.diagnose('pause', source.pause.bind(source))
-  channel.diagnose('resume', source.resume.bind(source))
+  // the source must not outlive the pipeline as a listener of the channel;
+  // a failure is delivered through the destination, the promise says nothing new
+  const detach = () => {
+    channel.forget('pause', pause)
+    channel.forget('resume', resume)
+  }
+
+  stream.pipeline(source, destination).then(detach, detach)
 
   return destination
 }

package/source/shards/connection.js

@@ -46,6 +46,10 @@ class Connection {
     this.#diagnostics.on(event, listener)
   }
 
+  forget (event, listener) {
+    this.#diagnostics.off(event, listener)
+  }
+
   /**
    * @param {comq.Connection} connection
    * @param {number} index

package/test/ReplyPipe.test.js

@@ -0,0 +1,170 @@
+'use strict'
+
+const { Readable } = require('node:stream')
+const { Promex } = require('promex')
+const { ReplyPipe } = require('../source/.io/ReplyPipe')
+const { createReplyEmitter } = require('../source/.io/createReplyEmitter')
+const { control, FLOW_HEADER } = require('../source/.io/const')
+const { timeout } = require('./helpers')
+
+/** @type {jest.Mock} */
+let reply
+
+/** @type {[any, comq.amqp.options.Publish][]} */
+let sent
+
+let channel
+
+/** @type {comq.ReplyEmitter} */
+let feedback
+
+let request
+
+beforeEach(() => {
+  global.COMQ_TESTING_HEARTBEAT_INTERVAL = 60_000
+
+  sent = []
+  reply = jest.fn(async (message, properties) => {
+    sent.push([message, properties])
+
+    return true
+  })
+
+  channel = { diagnose: jest.fn(), forget: jest.fn() }
+  feedback = createReplyEmitter('control')
+  request = { properties: { correlationId: 'test-correlation', replyTo: 'replies' } }
+})
+
+afterEach(() => {
+  delete global.COMQ_TESTING_HEARTBEAT_INTERVAL
+})
+
+const messages = () => sent.map(([message]) => message)
+
+const closing = (pipe) => new Promise((resolve) => pipe.on('close', resolve))
+
+it('should confirm, announcing flow control', async () => {
+  const pipe = await ReplyPipe.create(request, Readable.from([]), channel, feedback, reply)
+
+  expect(sent[0][0]).toStrictEqual(control.ok)
+  expect(sent[0][1].headers).toStrictEqual({ index: 0, [FLOW_HEADER]: true })
+
+  await closing(pipe)
+})
+
+it('should transmit values in order, then end', async () => {
+  const pipe = await ReplyPipe.create(request, Readable.from([1, 2, 3]), channel, feedback, reply)
+
+  await closing(pipe)
+
+  expect(messages()).toStrictEqual([control.ok, 1, 2, 3, control.end])
+  expect(sent.map(([, properties]) => properties.headers.index)).toStrictEqual([0, 1, 2, 3, 4])
+})
+
+// a source listened to runs at its own pace, and its output piles up in front of a paused channel
+it('should pull the source no faster than the channel takes', async () => {
+  const gate = new Promex()
+
+  let pulled = 0
+
+  reply.mockImplementation(async (message) => {
+    if (message !== control.ok) await gate
+
+    return true
+  })
+
+  function * source () {
+    for (let i = 0; i < 100; i++) {
+      pulled++
+
+      yield i
+    }
+  }
+
+  const pipe = await ReplyPipe.create(request, Readable.from(source()), channel, feedback, reply)
+
+  await timeout(10)
+
+  // a Readable reads ahead by no more than its high water mark
+  expect(pulled).toBeLessThanOrEqual(20)
+
+  gate.resolve()
+
+  await closing(pipe)
+
+  expect(pulled).toStrictEqual(100)
+})
+
+it('should hold the source while the consumer has asked for a pause', async () => {
+  const pipe = await ReplyPipe.create(request, Readable.from([1, 2, 3]), channel, feedback, reply)
+
+  feedback.emit(request.properties.correlationId, control.pause, {})
+
+  await timeout(10)
+
+  expect(messages()).toStrictEqual([control.ok])
+
+  feedback.emit(request.properties.correlationId, control.resume, {})
+
+  await closing(pipe)
+
+  expect(messages()).toStrictEqual([control.ok, 1, 2, 3, control.end])
+})
+
+it('should stop once the consumer has ended the stream', async () => {
+  const gate = new Promex()
+
+  reply.mockImplementation(async (message, properties) => {
+    sent.push([message, properties])
+
+    if (message !== control.ok) await gate
+
+    return true
+  })
+
+  const pipe = await ReplyPipe.create(request, Readable.from([1, 2, 3]), channel, feedback, reply)
+  const closed = closing(pipe)
+
+  feedback.emit(request.properties.correlationId, control.end, {})
+
+  await closed
+
+  gate.resolve()
+
+  await timeout(10)
+
+  expect(messages()).not.toContain(control.end)
+  expect(messages()).not.toContain(3)
+  expect(channel.forget).toHaveBeenCalledWith('return', expect.any(Function))
+  expect(feedback.pending).toStrictEqual(0)
+})
+
+it('should not heartbeat once the confirmation could not be delivered', async () => {
+  global.COMQ_TESTING_HEARTBEAT_INTERVAL = 5
+
+  reply.mockImplementation(async (message, properties) => {
+    sent.push([message, properties])
+
+    return false
+  })
+
+  const pipe = new ReplyPipe(request, Readable.from([1]), channel, feedback, reply)
+
+  await pipe.pipe()
+  await timeout(30)
+
+  expect(messages()).toStrictEqual([control.ok])
+})
+
+it('should heartbeat while idle', async () => {
+  global.COMQ_TESTING_HEARTBEAT_INTERVAL = 5
+
+  const source = new Readable({ objectMode: true, read () {} })
+  const pipe = await ReplyPipe.create(request, source, channel, feedback, reply)
+
+  await timeout(30)
+
+  expect(messages()).toContain(control.heartbeat)
+
+  pipe.destroy()
+})