comq 0.16.1 → 0.17.0

package/.github/dependabot.yml

@@ -14,3 +14,8 @@ updates:
     target-branch: "dev"
     labels:
       - "Bump"
+    ignore:
+      # a newer preset needs conventional-changelog-writer@9, which the
+      # semantic-release plugins do not use yet; the release fails to render its notes
+      - dependency-name: "conventional-changelog-conventionalcommits"
+        update-types: ["version-update:semver-major"]

package/features/events.routed.feature

@@ -0,0 +1,19 @@
+Feature: Routed events
+
+  An exchange that routes by key, where an event exchange fans out: a message reaches the
+  queues bound under the key it was published with, and none of the others.
+
+  Background:
+    Given an active connection to the broker
+
+  Scenario: Receiving an event by its routing key
+    Given that `orders` is bound to the `records` exchange under the `placed` key
+    When a message is routed to the `records` exchange under the `placed` key
+    Then `orders` receives the event
+
+  Scenario: Not receiving an event bound under another key
+    Given that `orders` is bound to the `records` exchange under the `placed` key
+    And `customers` is bound to the `records` exchange under the `shipped` key
+    When a message is routed to the `records` exchange under the `placed` key
+    Then `orders` receives the event
+    And `customers` receives nothing

package/features/steps/events.routed.js

@@ -0,0 +1,55 @@
+'use strict'
+
+const assert = require('node:assert')
+const { randomBytes } = require('node:crypto')
+const { timeout } = require('../../test/helpers')
+const { Given, When, Then } = require('@cucumber/cucumber')
+
+Given('(that ){token} is bound to the {token} exchange under the {token} key',
+  /**
+   * @param {string} queue
+   * @param {string} exchange
+   * @param {string} key
+   * @this {comq.features.Context}
+   */
+  async function (queue, exchange, key) {
+    await subscribe.call(this, queue, exchange, key)
+  })
+
+When('a message is routed to the {token} exchange under the {token} key',
+  /**
+   * @param {string} exchange
+   * @param {string} key
+   * @this {comq.features.Context}
+   */
+  async function (exchange, key) {
+    const message = randomBytes(8)
+
+    await this.io.route(exchange, key, message)
+
+    this.published = message
+  })
+
+Then('{token} receives nothing',
+  /**
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  async function (queue) {
+    await timeout(100) // let it not consume
+
+    assert.equal(this.consumed[queue], undefined, `'${queue}' has received an event`)
+  })
+
+/**
+ * @param {string} queue
+ * @param {string} exchange
+ * @param {string} key
+ * @this {comq.features.Context}
+ */
+async function subscribe (queue, exchange, key) {
+  await this.io.subscribe(exchange, queue, key, async (payload, properties) => {
+    this.consumed[queue] = { payload, properties }
+    this.eventsConsumedCount++
+  })
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.16.1",
+  "version": "0.17.0",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",

package/readme.md

@@ -7,7 +7,7 @@ for distributed, eventually consistent systems running on Node.js.
 
 - [Dynamic topology](#topology)
 - [Request](#request)-[reply](#reply) (RPC)
-- Events ([pub](#emission)/[sub](#consumption))
+- Events ([pub](#emission)/[sub](#consumption)), fanned out or [routed](#routing)
 - [Tasks](#tasks)
 - [Pipelines](#pipelines)
 - [Reply streams](#reply-streams)
@@ -67,6 +67,9 @@ value as in the Request.
 
 **Event** is an AMQP message published to an exchange.
 
+**Key** is the routing key an Event is published with, and the one a queue is bound under. An
+Event published to a fanout exchange carries none.
+
 **Task** is an AMQP message sent to a queue without a `replyTo` property set.
 
 **Producer** is an application role that receives Requests and Tasks, and produces Replies and Events.
@@ -158,6 +161,36 @@ asserted.
 await io.emit('numbers_added', { a: 1, b: 2 })
 ```
 
+## Routing
+
+`async IO.route(exchange: string, key: string, payload: any, encoding?: string): void`
+
+`async IO.subscribe(exchange: string, queue: string, key: string, consumer): void`
+
+Publish and consume Events addressed by a Key, where [Emission](#emission)
+and [Consumption](#consumption) fan out.
+
+`route` asserts a
+[direct exchange](https://www.rabbitmq.com/tutorials/amqp-concepts.html#exchange-direct) (once per
+unique `exchange`) and publishes the encoded Event to it under the `key`. `subscribe` asserts the
+same exchange and a durable `queue`, binds it under the `key` (once per unique `exchange`, `queue`
+and `key`), and starts consuming. **An Event reaches the queues bound under the Key it carries,
+and no others.**
+
+The queue is named rather than derived from a Consumer group, because what identifies it here is
+the Key it is bound under rather than the exchange it belongs to. It is also durable, so what is
+published while nothing is consuming is held rather than dropped.
+
+### Example
+
+```javascript
+await io.subscribe('records', 'records.orders', 'store.orders',
+  (record) => console.log(record))
+
+await io.route('records', 'store.orders', { id: 1, status: 'paid' })
+await io.route('records', 'store.customers', { id: 2 }) // not delivered to the above
+```
+
 ## Tasks
 
 `async IO.enqueue(queue: string, payload: any, encoding?: string): void`
@@ -255,6 +288,11 @@ These intervals are not configurable.
 
 An "end stream" message is sent to the `replyTo` queue when the Reply stream is finished.
 
+When the Consumer falls behind, a "pause" message is sent to the Producer's control queue,
+and the Producer stops pulling values from the iterator until a "resume" message follows.
+The Producer announces its support of these messages in the confirmation message,
+so they are never sent to a Producer that would not understand them.
+
 See also [Reply stream shutdown](#reply-stream-shutdown).
 
 ### Loss of tail
@@ -278,7 +316,8 @@ At the same time, there is no guarantee that the stream will be transmitted to t
 
 > When using the [Sharded connection](#sharded-connection), the order of yielded values is maintained through buffering.
 > However, there is a scenario in which some of the yielded values may be lost if a broker crashes.
-> In this case, the Reply stream will be destroyed once the buffer's maximum size is exceeded.
+> In this case, the Reply stream will be destroyed once the buffer's maximum size is exceeded
+> (1000 values or 16 MiB of encoded messages).
 > Also, buffered control messages can result in [stream idling](#stream-control).
 
 ## Encoding
@@ -403,6 +442,9 @@ requests and are expecting replies.
 
 - Exchanges and queues for Events, and queues for Requests
   are _durable_.
+- An exchange is asserted as _fanout_ for [Emission](#emission)
+  and [Consumption](#consumption), and as _direct_ for [Routing](#routing). One name is one or
+  the other: asserting it as both is what the broker refuses.
 - Queues for Replies are _exclusive_ and _auto deleted_.
 
 See [queue assertion options](https://amqp-node.github.io/amqplib/channel_api.html#channel_assertQueue).

package/source/.io/ReplyPipe.js

@@ -1,18 +1,23 @@
 'use strict'
 
 const { EventEmitter } = require('node:events')
-const { control, HEARTBEAT_INTERVAL } = require('./const')
+const { Promex } = require('promex')
+const { control, FLOW_HEADER, HEARTBEAT_INTERVAL } = require('./const')
 
 /** @typedef {(message: any, properties?: comq.amqp.options.Publish) => Promise<void>} Reply */
 
 class ReplyPipe extends EventEmitter {
   #index = -1
   #interrupted = false
+  #closed = false
   #heartbeatInterval = global['COMQ_TESTING_HEARTBEAT_INTERVAL'] || HEARTBEAT_INTERVAL
 
   /** @type {ReturnType<setInterval> | null} */
   #interval = null
 
+  /** Closed while the consumer has asked for a pause. */
+  #gate = null
+
   /** @type {string} */
   #replyTo
 
@@ -51,7 +56,8 @@ class ReplyPipe extends EventEmitter {
 
     this.#properties = {
       chunk: { correlationId, ...CHUNK },
-      control: { correlationId, replyTo: feedback.queue, ...CONTROL }
+      control: { correlationId, replyTo: feedback.queue, ...CONTROL },
+      ok: { correlationId, replyTo: feedback.queue, ...CONTROL, headers: FLOW }
     }
 
     channel.diagnose('return', this.#onReturn)
@@ -59,11 +65,13 @@ class ReplyPipe extends EventEmitter {
   }
 
   async pipe () {
-    await this.#transmit(control.ok, this.#properties.control)
+    await this.#transmit(control.ok, this.#properties.ok)
+
+    if (this.#closed) return
 
-    this.#stream.on('data', this.#onData)
-    this.#stream.on('close', this.#close)
     this.#heartbeat()
+
+    void this.#pump()
   }
 
   destroy () {
@@ -71,11 +79,35 @@ class ReplyPipe extends EventEmitter {
     this.#stream.destroy()
   }
 
+  /**
+   * The source is pulled rather than listened to: a value is asked for once the
+   * previous one has been handed to the channel, so a paused channel or a paused
+   * consumer holds the source back instead of piling its output up here.
+   */
+  async #pump () {
+    try {
+      for await (const chunk of this.#stream) {
+        if (this.#gate !== null) await this.#gate
+        if (this.#closed) break
+
+        await this.#transmit(chunk, this.#properties.chunk)
+
+        if (this.#closed) break
+
+        this.#heartbeat()
+      }
+    } catch {
+      // the source has been destroyed, by this pipe or by whoever made it
+    }
+
+    this.#close()
+  }
+
   async #transmit (data, properties) {
     this.#index++
 
-    const ok = await this.#reply(data,
-      { ...properties, headers: { index: this.#index } })
+    const headers = { ...properties.headers, index: this.#index }
+    const ok = await this.#reply(data, { ...properties, headers })
 
     if (!ok) this.#interrupt()
   }
@@ -97,12 +129,21 @@ class ReplyPipe extends EventEmitter {
   #clear () {
     clearInterval(this.#interval)
 
-    this.#stream.removeAllListeners()
     this.#channel.forget('return', this.#onReturn)
     this.#feedback.off(this.#properties.control.correlationId, this.#control)
+    this.#resume()
+  }
+
+  #resume () {
+    this.#gate?.resolve()
+    this.#gate = null
   }
 
   #close = () => {
+    if (this.#closed) return
+
+    this.#closed = true
+
     this.emit('close')
     this.#clear()
 
@@ -111,12 +152,6 @@ class ReplyPipe extends EventEmitter {
     }
   }
 
-  #onData = async (chunk) => {
-    await this.#transmit(chunk, this.#properties.chunk)
-
-    this.#heartbeat()
-  }
-
   #onReturn = (message) => {
     if (message.fields.routingKey === this.#replyTo)
       this.#interrupt()
@@ -127,6 +162,12 @@ class ReplyPipe extends EventEmitter {
       case control.end:
         this.#interrupt()
         break
+      case control.pause:
+        this.#gate ??= new Promex()
+        break
+      case control.resume:
+        this.#resume()
+        break
       default:
         throw new Error(`Unknown control message: ${message}`)
     }
@@ -155,4 +196,6 @@ const CHUNK = { mandatory: true }
 /** @type {comq.amqp.options.Publish} */
 const CONTROL = { type: 'control', mandatory: true }
 
+const FLOW = { [FLOW_HEADER]: true }
+
 exports.ReplyPipe = ReplyPipe

package/source/.io/ReplyStream.js

@@ -2,7 +2,7 @@
 
 const { Readable } = require('node:stream')
 const { Promex } = require('promex')
-const { IDLE_INTERVAL, control } = require('./const')
+const { IDLE_INTERVAL, FLOW_HEADER, control } = require('./const')
 
 class ReplyStream extends Readable {
   confirmation = new Promex()
@@ -16,6 +16,7 @@ class ReplyStream extends Readable {
   /** @type {string} */
   #correlationId
 
+  /** The confirmation message, addressing the control queue of the producer. */
   #control
 
   #reply
@@ -26,9 +27,19 @@ class ReplyStream extends Readable {
 
   #buffered = 0
 
+  #bufferedBytes = 0
+
   #maxBufferSize
 
-  /** @type {Map<number, unknown>} */
+  #maxBufferBytes
+
+  /** Whether the producer honours `pause` and `resume`. */
+  #flow = false
+
+  /** Whether the producer has been asked to pause. */
+  #throttled = false
+
+  /** @type {Map<number, { payload: unknown, properties: comq.amqp.Properties, size: number }>} */
   #queue = new Map()
 
   /**
@@ -42,6 +53,7 @@ class ReplyStream extends Readable {
     this.#correlationId = request.properties.correlationId
     this.#idleInterval = global['COMQ_TESTING_IDLE_INTERVAL'] || IDLE_INTERVAL
     this.#maxBufferSize = global['COMQ_TESTING_MAX_BUFFER_SIZE'] || MAX_BUFFER_SIZE
+    this.#maxBufferBytes = global['COMQ_TESTING_MAX_BUFFER_BYTES'] || MAX_BUFFER_BYTES
     this.#reply = reply
 
     this.confirmation.catch(noop) // it is not awaited until the stream is handed over
@@ -65,15 +77,25 @@ class ReplyStream extends Readable {
     super._destroy(error, callback)
   }
 
-  _read (_) {}
+  /**
+   * Called once the consumer has room again.
+   */
+  _read (_) {
+    if (!this.#throttled) return
+
+    this.#throttled = false
+
+    void this.#reply(this.#control, control.resume)
+  }
 
   /**
    * @param {unknown} payload
    * @param {comq.amqp.Properties} properties
+   * @param {number} [size] of the encoded payload
    */
-  arrange (payload, properties) {
+  arrange (payload, properties, size = 0) {
     if (properties.headers.index !== this.#index) {
-      this._buffer(payload, properties)
+      this._buffer(payload, properties, size)
 
       return
     }
@@ -85,6 +107,7 @@ class ReplyStream extends Readable {
 
       while ((message = this.#queue.get(this.#index))) {
         this.#queue.delete(this.#index)
+        this.#bufferedBytes -= message.size
         this._add(message.payload, message.properties)
       }
 
@@ -104,18 +127,45 @@ class ReplyStream extends Readable {
     if (properties.type === 'control')
       this._control(payload, properties)
     else if (!this.push(payload))
-      this._clear()
+      this._throttle()
   }
 
-  _buffer (payload, properties) {
-    if (this.#buffered > this.#maxBufferSize) {
+  /**
+   * Values arriving out of order are held until the gap is filled. The hold is
+   * bounded by their number and, roughly, by their size: the size is that of
+   * the encoded message, which is what is known of a decoded value.
+   *
+   * @param {unknown} payload
+   * @param {comq.amqp.Properties} properties
+   * @param {number} size
+   * @private
+   */
+  _buffer (payload, properties, size) {
+    if (this.#buffered > this.#maxBufferSize || this.#bufferedBytes + size > this.#maxBufferBytes) {
       this.destroy()
 
       return
     }
 
     this.#buffered++
-    this.#queue.set(properties.headers.index, { payload, properties })
+    this.#bufferedBytes += size
+    this.#queue.set(properties.headers.index, { payload, properties, size })
+  }
+
+  /**
+   * The consumer is behind. Values in flight keep coming, and a producer that
+   * knows nothing of `pause` keeps going, in which case they pile up here rather
+   * than get lost: dropping the listener would leave the stream waiting for a
+   * value that has already passed.
+   *
+   * @private
+   */
+  _throttle () {
+    if (this.#throttled || !this.#flow) return
+
+    this.#throttled = true
+
+    void this.#reply(this.#control, control.pause)
   }
 
   /**
@@ -127,6 +177,7 @@ class ReplyStream extends Readable {
     switch (message) {
       case control.ok:
         this.#control = { properties }
+        this.#flow = properties.headers?.[FLOW_HEADER] === true
         this.confirmation.resolve()
         break
       case control.heartbeat:
@@ -147,12 +198,15 @@ class ReplyStream extends Readable {
 
   _clear () {
     clearTimeout(this.#timeout)
-    this.#emitter.removeAllListeners(this.#correlationId)
+    this.#emitter.off(this.#correlationId)
   }
 }
 
 const MAX_BUFFER_SIZE = 1000
 
+/** @type {number} */
+const MAX_BUFFER_BYTES = 16 * 1024 * 1024
+
 const UNCONFIRMED = 'Reply stream has been destroyed before confirmation'
 
 function noop () {}

package/source/.io/const.js

@@ -3,8 +3,16 @@
 exports.control = {
   ok: 'ok',
   heartbeat: 'heartbeat',
-  end: 'end'
+  end: 'end',
+  pause: 'pause',
+  resume: 'resume'
 }
 
+/**
+ * Set on the confirmation message by a producer that honours `pause` and
+ * `resume`, so that a consumer never sends them to one that does not.
+ */
+exports.FLOW_HEADER = 'x-flow'
+
 exports.HEARTBEAT_INTERVAL = 5_000
 exports.IDLE_INTERVAL = 12_000

package/source/.io/createReplyEmitter.js

@@ -1,22 +1,97 @@
 'use strict'
 
 const { randomBytes } = require('node:crypto')
-const { EventEmitter } = require('node:events')
 const { concat } = require('./concat')
 
+/**
+ * Routes the replies arriving on a queue to whoever is waiting for them.
+ *
+ * A Map keyed by correlation identifier rather than an EventEmitter: with
+ * thousands of requests in flight the emitter's event table degrades into a
+ * dictionary, and every `once` costs a wrapper.
+ *
+ * @implements {comq.ReplyEmitter}
+ */
+class ReplyEmitter {
+  /** @type {string} */
+  queue
+
+  /** @type {string} */
+  #prefix
+
+  /** @type {number} */
+  #sequence = 0
+
+  /** @type {Map<string, comq.ReplyHandler>} */
+  #handlers = new Map()
+
+  /**
+   * @param {string} label
+   */
+  constructor (label) {
+    const id = randomBytes(8).toString('hex')
+
+    this.queue = concat(label, id)
+    this.#prefix = id
+  }
+
+  /**
+   * A correlation identifier that is unique across processes as well: the
+   * control queue of a producer sees the requests of every consumer, and tells
+   * them apart by nothing else. Random bytes per request would cost more than
+   * everything else on the way to the broker put together.
+   *
+   * @return {string}
+   */
+  next () {
+    return this.#prefix + (++this.#sequence).toString(36)
+  }
+
+  /**
+   * @param {string} id
+   * @param {comq.ReplyHandler} handler
+   */
+  on (id, handler) {
+    this.#handlers.set(id, handler)
+  }
+
+  /**
+   * @param {string} id
+   * @param {comq.ReplyHandler} [handler] the one to remove, or whichever is there
+   */
+  off (id, handler) {
+    if (handler === undefined || this.#handlers.get(id) === handler) this.#handlers.delete(id)
+  }
+
+  /**
+   * @param {string} id
+   * @param {any} payload
+   * @param {comq.amqp.Properties} properties
+   * @param {number} [size] of the encoded payload
+   * @return {boolean} whether anyone was waiting
+   */
+  emit (id, payload, properties, size) {
+    const handler = this.#handlers.get(id)
+
+    if (handler === undefined) return false
+
+    handler(payload, properties, size)
+
+    return true
+  }
+
+  /** How many replies are being waited for. */
+  get pending () {
+    return this.#handlers.size
+  }
+}
+
 /**
  * @param {string} label
  * @returns {comq.ReplyEmitter}
  */
 function createReplyEmitter (label) {
-  const id = randomBytes(8).toString('hex')
-  const queue = concat(label, id)
-  const emitter = new EventEmitter()
-
-  emitter.setMaxListeners(0)
-  emitter.queue = queue
-
-  return /** @type {comq.ReplyEmitter} */ emitter
+  return new ReplyEmitter(label)
 }
 
 exports.createReplyEmitter = createReplyEmitter