comq 0.16.1 → 0.16.2

package/.github/dependabot.yml

@@ -14,3 +14,8 @@ updates:
     target-branch: "dev"
     labels:
       - "Bump"
+    ignore:
+      # a newer preset needs conventional-changelog-writer@9, which the
+      # semantic-release plugins do not use yet; the release fails to render its notes
+      - dependency-name: "conventional-changelog-conventionalcommits"
+        update-types: ["version-update:semver-major"]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.16.1",
+  "version": "0.16.2",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",

package/readme.md

@@ -255,6 +255,11 @@ These intervals are not configurable.
 
 An "end stream" message is sent to the `replyTo` queue when the Reply stream is finished.
 
+When the Consumer falls behind, a "pause" message is sent to the Producer's control queue,
+and the Producer stops pulling values from the iterator until a "resume" message follows.
+The Producer announces its support of these messages in the confirmation message,
+so they are never sent to a Producer that would not understand them.
+
 See also [Reply stream shutdown](#reply-stream-shutdown).
 
 ### Loss of tail
@@ -278,7 +283,8 @@ At the same time, there is no guarantee that the stream will be transmitted to t
 
 > When using the [Sharded connection](#sharded-connection), the order of yielded values is maintained through buffering.
 > However, there is a scenario in which some of the yielded values may be lost if a broker crashes.
-> In this case, the Reply stream will be destroyed once the buffer's maximum size is exceeded.
+> In this case, the Reply stream will be destroyed once the buffer's maximum size is exceeded
+> (1000 values or 16 MiB of encoded messages).
 > Also, buffered control messages can result in [stream idling](#stream-control).
 
 ## Encoding

package/source/.io/ReplyPipe.js

@@ -1,18 +1,23 @@
 'use strict'
 
 const { EventEmitter } = require('node:events')
-const { control, HEARTBEAT_INTERVAL } = require('./const')
+const { Promex } = require('promex')
+const { control, FLOW_HEADER, HEARTBEAT_INTERVAL } = require('./const')
 
 /** @typedef {(message: any, properties?: comq.amqp.options.Publish) => Promise<void>} Reply */
 
 class ReplyPipe extends EventEmitter {
   #index = -1
   #interrupted = false
+  #closed = false
   #heartbeatInterval = global['COMQ_TESTING_HEARTBEAT_INTERVAL'] || HEARTBEAT_INTERVAL
 
   /** @type {ReturnType<setInterval> | null} */
   #interval = null
 
+  /** Closed while the consumer has asked for a pause. */
+  #gate = null
+
   /** @type {string} */
   #replyTo
 
@@ -51,7 +56,8 @@ class ReplyPipe extends EventEmitter {
 
     this.#properties = {
       chunk: { correlationId, ...CHUNK },
-      control: { correlationId, replyTo: feedback.queue, ...CONTROL }
+      control: { correlationId, replyTo: feedback.queue, ...CONTROL },
+      ok: { correlationId, replyTo: feedback.queue, ...CONTROL, headers: FLOW }
     }
 
     channel.diagnose('return', this.#onReturn)
@@ -59,11 +65,13 @@ class ReplyPipe extends EventEmitter {
   }
 
   async pipe () {
-    await this.#transmit(control.ok, this.#properties.control)
+    await this.#transmit(control.ok, this.#properties.ok)
+
+    if (this.#closed) return
 
-    this.#stream.on('data', this.#onData)
-    this.#stream.on('close', this.#close)
     this.#heartbeat()
+
+    void this.#pump()
   }
 
   destroy () {
@@ -71,11 +79,35 @@ class ReplyPipe extends EventEmitter {
     this.#stream.destroy()
   }
 
+  /**
+   * The source is pulled rather than listened to: a value is asked for once the
+   * previous one has been handed to the channel, so a paused channel or a paused
+   * consumer holds the source back instead of piling its output up here.
+   */
+  async #pump () {
+    try {
+      for await (const chunk of this.#stream) {
+        if (this.#gate !== null) await this.#gate
+        if (this.#closed) break
+
+        await this.#transmit(chunk, this.#properties.chunk)
+
+        if (this.#closed) break
+
+        this.#heartbeat()
+      }
+    } catch {
+      // the source has been destroyed, by this pipe or by whoever made it
+    }
+
+    this.#close()
+  }
+
   async #transmit (data, properties) {
     this.#index++
 
-    const ok = await this.#reply(data,
-      { ...properties, headers: { index: this.#index } })
+    const headers = { ...properties.headers, index: this.#index }
+    const ok = await this.#reply(data, { ...properties, headers })
 
     if (!ok) this.#interrupt()
   }
@@ -97,12 +129,21 @@ class ReplyPipe extends EventEmitter {
   #clear () {
     clearInterval(this.#interval)
 
-    this.#stream.removeAllListeners()
     this.#channel.forget('return', this.#onReturn)
     this.#feedback.off(this.#properties.control.correlationId, this.#control)
+    this.#resume()
+  }
+
+  #resume () {
+    this.#gate?.resolve()
+    this.#gate = null
   }
 
   #close = () => {
+    if (this.#closed) return
+
+    this.#closed = true
+
     this.emit('close')
     this.#clear()
 
@@ -111,12 +152,6 @@ class ReplyPipe extends EventEmitter {
     }
   }
 
-  #onData = async (chunk) => {
-    await this.#transmit(chunk, this.#properties.chunk)
-
-    this.#heartbeat()
-  }
-
   #onReturn = (message) => {
     if (message.fields.routingKey === this.#replyTo)
       this.#interrupt()
@@ -127,6 +162,12 @@ class ReplyPipe extends EventEmitter {
       case control.end:
         this.#interrupt()
         break
+      case control.pause:
+        this.#gate ??= new Promex()
+        break
+      case control.resume:
+        this.#resume()
+        break
       default:
         throw new Error(`Unknown control message: ${message}`)
     }
@@ -155,4 +196,6 @@ const CHUNK = { mandatory: true }
 /** @type {comq.amqp.options.Publish} */
 const CONTROL = { type: 'control', mandatory: true }
 
+const FLOW = { [FLOW_HEADER]: true }
+
 exports.ReplyPipe = ReplyPipe

package/source/.io/ReplyStream.js

@@ -2,7 +2,7 @@
 
 const { Readable } = require('node:stream')
 const { Promex } = require('promex')
-const { IDLE_INTERVAL, control } = require('./const')
+const { IDLE_INTERVAL, FLOW_HEADER, control } = require('./const')
 
 class ReplyStream extends Readable {
   confirmation = new Promex()
@@ -16,6 +16,7 @@ class ReplyStream extends Readable {
   /** @type {string} */
   #correlationId
 
+  /** The confirmation message, addressing the control queue of the producer. */
   #control
 
   #reply
@@ -26,9 +27,19 @@ class ReplyStream extends Readable {
 
   #buffered = 0
 
+  #bufferedBytes = 0
+
   #maxBufferSize
 
-  /** @type {Map<number, unknown>} */
+  #maxBufferBytes
+
+  /** Whether the producer honours `pause` and `resume`. */
+  #flow = false
+
+  /** Whether the producer has been asked to pause. */
+  #throttled = false
+
+  /** @type {Map<number, { payload: unknown, properties: comq.amqp.Properties, size: number }>} */
   #queue = new Map()
 
   /**
@@ -42,6 +53,7 @@ class ReplyStream extends Readable {
     this.#correlationId = request.properties.correlationId
     this.#idleInterval = global['COMQ_TESTING_IDLE_INTERVAL'] || IDLE_INTERVAL
     this.#maxBufferSize = global['COMQ_TESTING_MAX_BUFFER_SIZE'] || MAX_BUFFER_SIZE
+    this.#maxBufferBytes = global['COMQ_TESTING_MAX_BUFFER_BYTES'] || MAX_BUFFER_BYTES
     this.#reply = reply
 
     this.confirmation.catch(noop) // it is not awaited until the stream is handed over
@@ -65,15 +77,25 @@ class ReplyStream extends Readable {
     super._destroy(error, callback)
   }
 
-  _read (_) {}
+  /**
+   * Called once the consumer has room again.
+   */
+  _read (_) {
+    if (!this.#throttled) return
+
+    this.#throttled = false
+
+    void this.#reply(this.#control, control.resume)
+  }
 
   /**
    * @param {unknown} payload
    * @param {comq.amqp.Properties} properties
+   * @param {number} [size] of the encoded payload
    */
-  arrange (payload, properties) {
+  arrange (payload, properties, size = 0) {
     if (properties.headers.index !== this.#index) {
-      this._buffer(payload, properties)
+      this._buffer(payload, properties, size)
 
       return
     }
@@ -85,6 +107,7 @@ class ReplyStream extends Readable {
 
       while ((message = this.#queue.get(this.#index))) {
         this.#queue.delete(this.#index)
+        this.#bufferedBytes -= message.size
         this._add(message.payload, message.properties)
       }
 
@@ -104,18 +127,45 @@ class ReplyStream extends Readable {
     if (properties.type === 'control')
       this._control(payload, properties)
     else if (!this.push(payload))
-      this._clear()
+      this._throttle()
   }
 
-  _buffer (payload, properties) {
-    if (this.#buffered > this.#maxBufferSize) {
+  /**
+   * Values arriving out of order are held until the gap is filled. The hold is
+   * bounded by their number and, roughly, by their size: the size is that of
+   * the encoded message, which is what is known of a decoded value.
+   *
+   * @param {unknown} payload
+   * @param {comq.amqp.Properties} properties
+   * @param {number} size
+   * @private
+   */
+  _buffer (payload, properties, size) {
+    if (this.#buffered > this.#maxBufferSize || this.#bufferedBytes + size > this.#maxBufferBytes) {
       this.destroy()
 
       return
     }
 
     this.#buffered++
-    this.#queue.set(properties.headers.index, { payload, properties })
+    this.#bufferedBytes += size
+    this.#queue.set(properties.headers.index, { payload, properties, size })
+  }
+
+  /**
+   * The consumer is behind. Values in flight keep coming, and a producer that
+   * knows nothing of `pause` keeps going, in which case they pile up here rather
+   * than get lost: dropping the listener would leave the stream waiting for a
+   * value that has already passed.
+   *
+   * @private
+   */
+  _throttle () {
+    if (this.#throttled || !this.#flow) return
+
+    this.#throttled = true
+
+    void this.#reply(this.#control, control.pause)
   }
 
   /**
@@ -127,6 +177,7 @@ class ReplyStream extends Readable {
     switch (message) {
       case control.ok:
         this.#control = { properties }
+        this.#flow = properties.headers?.[FLOW_HEADER] === true
         this.confirmation.resolve()
         break
       case control.heartbeat:
@@ -147,12 +198,15 @@ class ReplyStream extends Readable {
 
   _clear () {
     clearTimeout(this.#timeout)
-    this.#emitter.removeAllListeners(this.#correlationId)
+    this.#emitter.off(this.#correlationId)
   }
 }
 
 const MAX_BUFFER_SIZE = 1000
 
+/** @type {number} */
+const MAX_BUFFER_BYTES = 16 * 1024 * 1024
+
 const UNCONFIRMED = 'Reply stream has been destroyed before confirmation'
 
 function noop () {}

package/source/.io/const.js

@@ -3,8 +3,16 @@
 exports.control = {
   ok: 'ok',
   heartbeat: 'heartbeat',
-  end: 'end'
+  end: 'end',
+  pause: 'pause',
+  resume: 'resume'
 }
 
+/**
+ * Set on the confirmation message by a producer that honours `pause` and
+ * `resume`, so that a consumer never sends them to one that does not.
+ */
+exports.FLOW_HEADER = 'x-flow'
+
 exports.HEARTBEAT_INTERVAL = 5_000
 exports.IDLE_INTERVAL = 12_000

package/source/.io/createReplyEmitter.js

@@ -1,22 +1,97 @@
 'use strict'
 
 const { randomBytes } = require('node:crypto')
-const { EventEmitter } = require('node:events')
 const { concat } = require('./concat')
 
+/**
+ * Routes the replies arriving on a queue to whoever is waiting for them.
+ *
+ * A Map keyed by correlation identifier rather than an EventEmitter: with
+ * thousands of requests in flight the emitter's event table degrades into a
+ * dictionary, and every `once` costs a wrapper.
+ *
+ * @implements {comq.ReplyEmitter}
+ */
+class ReplyEmitter {
+  /** @type {string} */
+  queue
+
+  /** @type {string} */
+  #prefix
+
+  /** @type {number} */
+  #sequence = 0
+
+  /** @type {Map<string, comq.ReplyHandler>} */
+  #handlers = new Map()
+
+  /**
+   * @param {string} label
+   */
+  constructor (label) {
+    const id = randomBytes(8).toString('hex')
+
+    this.queue = concat(label, id)
+    this.#prefix = id
+  }
+
+  /**
+   * A correlation identifier that is unique across processes as well: the
+   * control queue of a producer sees the requests of every consumer, and tells
+   * them apart by nothing else. Random bytes per request would cost more than
+   * everything else on the way to the broker put together.
+   *
+   * @return {string}
+   */
+  next () {
+    return this.#prefix + (++this.#sequence).toString(36)
+  }
+
+  /**
+   * @param {string} id
+   * @param {comq.ReplyHandler} handler
+   */
+  on (id, handler) {
+    this.#handlers.set(id, handler)
+  }
+
+  /**
+   * @param {string} id
+   * @param {comq.ReplyHandler} [handler] the one to remove, or whichever is there
+   */
+  off (id, handler) {
+    if (handler === undefined || this.#handlers.get(id) === handler) this.#handlers.delete(id)
+  }
+
+  /**
+   * @param {string} id
+   * @param {any} payload
+   * @param {comq.amqp.Properties} properties
+   * @param {number} [size] of the encoded payload
+   * @return {boolean} whether anyone was waiting
+   */
+  emit (id, payload, properties, size) {
+    const handler = this.#handlers.get(id)
+
+    if (handler === undefined) return false
+
+    handler(payload, properties, size)
+
+    return true
+  }
+
+  /** How many replies are being waited for. */
+  get pending () {
+    return this.#handlers.size
+  }
+}
+
 /**
  * @param {string} label
  * @returns {comq.ReplyEmitter}
  */
 function createReplyEmitter (label) {
-  const id = randomBytes(8).toString('hex')
-  const queue = concat(label, id)
-  const emitter = new EventEmitter()
-
-  emitter.setMaxListeners(0)
-  emitter.queue = queue
-
-  return /** @type {comq.ReplyEmitter} */ emitter
+  return new ReplyEmitter(label)
 }
 
 exports.createReplyEmitter = createReplyEmitter

package/source/attributes/lazy.js

@@ -1,8 +1,18 @@
 'use strict'
 
+/**
+ * Runs the initializers once per distinct set of arguments before the method.
+ *
+ * The record of what has been initialized is kept per context, so that a context
+ * that is dropped takes its record with it, and it is looked up by argument rather
+ * than scanned — a channel publishing to a thousand queues asks for the thousandth
+ * as fast as for the first.
+ *
+ * @param {object} context
+ * @param {Function | Function[]} initializers
+ * @param {(...args: unknown[]) => Promise<unknown>} method
+ */
 function lazy (context, initializers, method) {
-  if (context[LOCK] === undefined) context[LOCK] = Symbol('methods locking key')
-
   if (!Array.isArray(initializers)) initializers = [initializers]
 
   return async function (...args) {
@@ -24,7 +34,7 @@ async function call (context, initializers, args) {
   let override
 
   for (const init of initializers) {
-    const result = await resolve(context, init, args)
+    const result = await lock(context, init, args)
 
     if (result !== undefined) override = result
   }
@@ -33,46 +43,76 @@ async function call (context, initializers, args) {
 }
 
 /**
+ * An initializer expecting N arguments is run once per distinct N leading
+ * arguments, which are the keys of a trie of Maps N levels deep.
+ *
  * @param {object} context
- * @param {Function} init
- * @param {any[]} args
- * @returns {Promise}
+ * @param {(...args: unknown[]) => Promise<unknown>} init
+ * @param {unknown[]} args
+ * @returns {Promise<unknown>}
  */
-function resolve (context, init, args) {
-  const key = context[LOCK]
-  const expected = args.slice(0, init.length)
+function lock (context, init, args) {
+  const arity = init.length
+
+  let node = table(context, init)
+
+  for (let i = 0; i < arity - 1; i++) node = branch(node, args[i])
+
+  const key = arity === 0 ? NONE : args[arity - 1]
+  const found = node.get(key)
+
+  if (found !== undefined) return found
+
+  const promise = init.apply(context, args.slice(0, arity))
+
+  node.set(key, promise)
 
-  return lock(context, init, expected, key)
+  return promise
 }
 
 /**
  * @param {object} context
- * @param {(...args: unknown[]) => Promise<unknown>} init
- * @param {unknown[]} args
- * @param {Symbol} key
- * @returns {Promise<unknown>}
+ * @param {Function} init
+ * @return {Map<unknown, unknown>}
  */
-function lock (context, init, args, key) {
-  if (init[key] === undefined) init[key] = []
+function table (context, init) {
+  let tables = TABLES.get(context)
 
-  const locks = init[key]
-  const found = locks.find((lock) => lock.args.reduce((match, argument, i) => match && argument === args[i], true))
+  if (tables === undefined) {
+    tables = new Map()
+    TABLES.set(context, tables)
+  }
 
-  if (found !== undefined) return found.promise
+  return branch(tables, init)
+}
 
-  const promise = init.apply(context, args)
-  const lock = { args, promise }
+/**
+ * @param {Map<unknown, unknown>} node
+ * @param {unknown} key
+ * @return {Map<unknown, unknown>}
+ */
+function branch (node, key) {
+  let next = node.get(key)
 
-  locks.push(lock)
+  if (next === undefined) {
+    next = new Map()
+    node.set(key, next)
+  }
 
-  return promise
+  return /** @type {Map<unknown, unknown>} */ next
 }
 
+/**
+ * @param {object} context
+ */
 function reset (context) {
-  context[LOCK] = context[LOCK] = Symbol('methods locking key')
+  TABLES.delete(context)
 }
 
-const LOCK = Symbol('context locking key')
+/** @type {WeakMap<object, Map<Function, Map<unknown, unknown>>>} */
+const TABLES = new WeakMap()
+
+const NONE = Symbol('no arguments')
 
 lazy.reset = reset
 

package/source/attributes/lazy.test.js

@@ -297,3 +297,42 @@ describe('reset', () => {
     expect(initialize2).toHaveBeenCalledTimes(2)
   })
 })
+
+class Exposed {
+  do = lazy(this, this.initialize, async () => undefined)
+
+  async initialize (_) {}
+}
+
+// an initializer is a method shared by every instance of its class, so a record
+// kept on it would outlive the instances it was made for
+it('should keep no record on the initializer or the context', async () => {
+  const instance = new Exposed()
+
+  await instance.do(generate())
+  lazy.reset(instance)
+  await instance.do(generate())
+
+  expect(Object.getOwnPropertySymbols(Exposed.prototype.initialize)).toHaveLength(0)
+  expect(Object.getOwnPropertySymbols(instance)).toHaveLength(0)
+})
+
+it('should tell arguments apart by identity', async () => {
+  let calls = 0
+
+  class Counted {
+    do = lazy(this, this.initialize, async () => undefined)
+
+    async initialize (_) {
+      calls++
+    }
+  }
+
+  const instance = new Counted()
+
+  await instance.do(undefined)
+  await instance.do('undefined')
+  await instance.do(undefined)
+
+  expect(calls).toStrictEqual(2)
+})

package/source/attributes/recall.js

@@ -36,7 +36,24 @@ async function replay (context) {
   await Promise.all(promises)
 }
 
+/**
+ * Forgets the recorded calls, and the arguments they hold on to.
+ *
+ * @param {object} context
+ */
+function reset (context) {
+  const methods = context[METHODS]
+
+  if (methods === undefined) return
+
+  for (const method of methods) method[CALLS] = undefined
+
+  context[METHODS] = undefined
+}
+
 const METHODS = Symbol('context methods')
 const CALLS = Symbol('method calls')
 
+recall.reset = reset
+
 exports.recall = recall