comq 0.8.0 → 0.10.0

package/features/steps/rpc.js

@@ -1,10 +1,12 @@
 'use strict'
 
+const stream = require('node:stream')
 const assert = require('node:assert')
 const { randomBytes } = require('node:crypto')
 const { parse } = require('@toa.io/yaml')
 const { match, timeout, quantity } = require('@toa.io/generic')
 const { Given, When, Then } = require('@cucumber/cucumber')
+const { Readable } = require('node:stream')
 
 Given('function replying {token} queue:',
   /**
@@ -19,6 +21,70 @@ Given('function replying {token} queue:',
     await this.io.reply(queue, producer)
   })
 
+Given('a generator replying {token} queue:',
+  /**
+   * @param {string} queue
+   * @param {string} javascript
+   * @this {comq.features.Context}
+   */
+  async function (queue, javascript) {
+    // eslint-disable-next-line no-new-func
+    const generator = new Function('return ' + javascript)()
+
+    function producer (input) {
+      return stream.Readable.from(generator(input))
+    }
+
+    await this.io.reply(queue, producer)
+  })
+
+Given('a number generator with {number}ms increasing delay replying {token} queue',
+  /**
+   * @param {number} delay
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  async function (delay, queue) {
+    const that = this
+
+    class Stream extends Readable {
+      #MAX = 10
+      #count = 0
+
+      constructor () {
+        super({ objectMode: true })
+      }
+
+      async _read (size) {
+        await timeout(delay * (1 + this.#count / 5))
+
+        if (this.#count === this.#MAX) { this.push(null) } else this.push(this.#count++)
+      }
+
+      _destroy (error, callback) {
+        that.generatorDestroyed = true
+        this.push(null)
+        super._destroy(error, callback)
+      }
+    }
+
+    function producer () {
+      return new Stream()
+    }
+
+    await this.io.reply(queue, producer)
+  })
+
+Given('heartbeat interval is set to {number}ms',
+  function (value) {
+    global.COMQ_TESTING_HEARTBEAT_INTERVAL = value
+  })
+
+Given('idle timeout is set to {number}ms',
+  function (value) {
+    global.COMQ_TESTING_IDLE_INTERVAL = value
+  })
+
 Given('a producer replying {token} queue',
   /**
    * @param {string} queue
@@ -70,6 +136,37 @@ When('the consumer sends a request to the {token} queue',
     await send.call(this, queue, payload)
   })
 
+When('the consumer fetches a stream with the following request to the {token} queue:',
+  /**
+   * @param {string} queue
+   * @param {string} yaml
+   * @this {comq.features.Context}
+   */
+  async function (queue, yaml) {
+    const payload = parse(yaml)
+
+    await fetch.call(this, queue, payload)
+  })
+
+When('the consumer fetches a stream with request to the {token} queue',
+  /**
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  async function (queue) {
+    await fetch.call(this, queue, null)
+  })
+
+When('the consumer{number} fetches a stream with request to the {token} queue',
+  /**
+   * @param {number} number
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  async function (number, queue) {
+    await fetch.call(this, queue, null, number)
+  })
+
 Then('the consumer receives the reply:',
   /**
    * @param {string} yaml
@@ -106,6 +203,162 @@ Then('the consumer does not receive the reply',
     assert.equal(reply, undefined, 'The reply was received')
   })
 
+Then('the consumer receives the stream:',
+  /**
+   * @param {string} yaml
+   * @this {comq.features.Context}
+   */
+  async function (yaml) {
+    const values = parse(yaml)
+    const replies = []
+
+    for await (const reply of this.stream) replies.push(reply)
+
+    assert.equal(values.length, replies.length, `Stream length mismatch: expected ${values.length}, received ${replies.length}`)
+  })
+
+Then('the consumer receives the stream',
+  /**
+   * @this {comq.features.Context}
+   */
+  async function () {
+    this.stream.on('data', (data) => this.streamValues.push(data))
+    this.stream.on('end', () => (this.streamEnded = true))
+  })
+
+Then('the consumer{number} receives the stream',
+  /**
+   * @param {number} number
+   * @this {comq.features.Context}
+   */
+  async function (number) {
+    this.streams[number].on('data', (data) => this.streamsValues[number].push(data))
+    this.streams[number].on('end', () => (this.streamsEnded[number] = true))
+  })
+
+Then('the consumer has received the stream:',
+  /**
+   * @param {string} yaml
+   * @this {comq.features.Context}
+   */
+  async function (yaml) {
+    const values = parse(yaml)
+
+    assert.equal(this.streamEnded, true, 'The stream was not closed')
+
+    assert.equal(values.length, this.streamValues.length,
+      `Stream values count mismatch: expected ${values.length}, received ${this.streamValues.length}`)
+
+    for (let i = 0; i < values.length; i++) {
+      assert.equal(values[i], this.streamValues[i], `Stream value mismatch at index ${i}`)
+    }
+  })
+
+Then('the consumer{number} has received the stream:',
+  /**
+   * @param {number} number
+   * @param {string} yaml
+   * @this {comq.features.Context}
+   */
+  async function (number, yaml) {
+    const expected = parse(yaml)
+    const values = this.streamsValues[number]
+
+    assert.equal(this.streamsEnded[number], true, 'The stream was not closed')
+
+    assert.equal(expected.length, values.length,
+      `Stream values count mismatch: expected ${expected.length}, received ${values.length}`)
+
+    for (let i = 0; i < expected.length; i++) {
+      assert.equal(expected[i], values[i], `Stream value mismatch at index ${i}`)
+    }
+  })
+
+Then('the consumer has received the stream containing:',
+  /**
+   * @param {string} yaml
+   * @this {comq.features.Context}
+   */
+  async function (yaml) {
+    const values = parse(yaml)
+
+    assert.equal(this.streamEnded, true, 'The stream was not closed')
+
+    assert.equal(values.length <= this.streamValues.length, true,
+      `Received less values: expected ${values.length}, received ${this.streamValues.length}`)
+
+    for (const value of values) {
+      assert.notEqual(this.streamValues.indexOf(value), -1,
+        `Received values does not contain ${value}`)
+    }
+  })
+
+Then('the consumer interrupts the stream after {number} replies',
+  /**
+   * @param {number} replies
+   * @this {comq.features.Context}
+   */
+  async function (replies) {
+    // eslint-disable-next-line no-unused-vars
+    for await (const _ of this.stream) {
+      if (--replies === 0) break // this destroys the stream
+    }
+
+    assert.equal(replies, 0, 'Stream was prematurely interrupted')
+  })
+
+Then('the consumer interrupts the stream',
+  /**
+   * @this {comq.features.Context}
+   */
+  async function () {
+    this.stream.destroy()
+    await timeout(0)
+  })
+
+Then('the generator is destroyed',
+  async function () {
+    assert.equal(this.generatorDestroyed, true, 'The generator was not destroyed')
+  })
+
+When('the consumer sends {quantity} requests to the {token} queue as a stream',
+  /**
+   * @param {string} amountQ
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  async function (amountQ, queue) {
+    const amount = quantity(amountQ)
+
+    function * generate () {
+      for (let i = 0; i < amount; i++) {
+        yield randomBytes(8)
+      }
+    }
+
+    const input = stream.Readable.from(generate())
+
+    await send.call(this, queue, input)
+  })
+
+Then('the consumer receives {quantity} replies as a stream',
+  /**
+   * @this {comq.features.Context}
+   */
+  async function (amountQ) {
+    const expected = quantity(amountQ)
+    const output = await this.reply
+
+    let actual = 0
+
+    // eslint-disable-next-line no-unused-vars
+    for await (const _ of output) {
+      actual++
+    }
+
+    assert.equal(actual, expected, 'The amount of replies mismatches')
+  })
+
 Given('I\'m sending {quantity}B requests to the {token} queue at {quantity}Hz',
   /**
    * @param {string} bytesQ
@@ -145,3 +398,23 @@ async function send (queue, payload) {
 
   this.reply = this.io.request(queue, payload)
 }
+
+/**
+ * @param {string} queue
+ * @param {any} payload
+ * @param {number} number
+ * @this {comq.features.Context}
+ * @return {Promise<void>}
+ */
+async function fetch (queue, payload, number) {
+  if (number === undefined) {
+    this.stream = await this.io.fetch(queue, payload)
+  } else {
+    this.streams[number] = await this.io.fetch(queue, payload)
+    this.streamsValues[number] = []
+    this.streamsEnded[number] = false
+  }
+}
+
+global.COMQ_TESTING_IDLE_INTERVAL = 150
+global.COMQ_TESTING_HEARTBEAT_INTERVAL = 100

package/lerna.json

@@ -1,5 +1,4 @@
 {
-  "useWorkspaces": true,
-  "version": "0.8.0",
+  "version": "0.10.0",
   "conventionalCommits": true
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.8.0",
+  "version": "0.10.0",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",
@@ -45,7 +45,7 @@
     "msgpackr": "^1.9.5"
   },
   "scripts": {
-    "prepare": "husky install",
+    "prepare": "echo run: husky install",
     "test": "standard && jest",
     "features": "cucumber-js",
     "lint": "standard --fix --verbose | snazzy"

package/readme.md

@@ -1,26 +1,30 @@
 # ComQ
 
-Distributed system communications over [AMQP](https://github.com/amqp-node/amqplib) for
-Node.js.
+Production-grade communication via [AMQP](https://github.com/amqp-node/amqplib)
+for distributed, eventually consistent systems running on Node.js.
 
 ## Features
 
-1. [Dynamic topology](#topology)
-2. [Request](#request)-[reply](#reply) (RPC)
-3. Events ([pub](#emission)/[sub](#consumption))
-4. [Content encoding](#encoding)
-5. [Flow control](#flow-control) and back pressure handling
-6. [Consumer acknowledgments](#messages) and [publisher confirms](#channels)
-7. [Poison message handling](#messages)
-8. [Connection tolerance](#connection-tolerance) and broker restart resilience
-9. [Sharded connection](#sharded-connection)
-10. [Graceful shutdown](#graceful-shutdown)
-
-> CommonJS, ECMAScript and TypeScript compatible (types included).
+- [Dynamic topology](#topology)
+- [Request](#request)-[reply](#reply) (RPC)
+- Events ([pub](#emission)/[sub](#consumption))
+- [Pipelines](#pipelines)
+- [Reply streams](#reply-streams)
+- [Content encoding](#encoding)
+- [Flow control](#flow-control) and back pressure handling
+- [Consumer acknowledgments](#messages) and [publisher confirms](#channels)
+- [Poison message handling](#messages)
+- [Connection tolerance](#connection-tolerance) and broker restart resilience
+- [Sharded connection](#sharded-connection) :rocket:
+- [Singleton connection](#singleton-connection)
+- [Graceful shutdown](#graceful-shutdown)
+
+> CommonJS, ECMAScript, and TypeScript compatible (types included).
 
 ## TL;DR
 
-[Code examples](examples)
+- [Code examples](examples)
+- [Scenarios](features)
 
 ## Installation
 
@@ -73,7 +77,7 @@ value as in the Request.
 `producer` function's signature is `async? (message: any): any`
 
 Assert a `queue` and start consuming Requests. Received messages are decoded and the resulting
-content is passed to the `producer`. The result generated by the `producer` is then encoded and sent
+content is passed to the `producer`. The result returned by the `producer` is then encoded and sent
 back to the queue specified in the `replyTo` property of the Request, along with a `correlationId`
 that has the same value as in the Request.
 
@@ -96,7 +100,7 @@ await io.reply('add_numbers', ({ a, b }) => (a + b))
 
 ## Request
 
-`async IO.request(queue: string, payload: any, [encoding: string]): any`
+`async IO.request(queue: string, payload: any, encoding?: string): any`
 
 Send encoded Request message with `replyTo` and `correlationId` properties set and
 return decoded Reply content.
@@ -111,7 +115,7 @@ const sum = await io.request('add_numbers', { a: 1, b: 2 })
 
 ## Consumption
 
-`async IO.consume(exchange: string, [group: string], consumer): void`
+`async IO.consume(exchange: string, group?: string, consumer): void`
 
 `consumer` function's signature is `async? (payload: any): void`
 
@@ -130,14 +134,20 @@ auto-generated name.
 ### Example
 
 ```javascript
-await io.consume('numbers_added', 'logger', ({ a, b }) => {
+// with a consumer function
+await io.consume('numbers_added', 'logger',
+  ({ a, b }) => console.log(`${a} was added to ${b}`))
+
+// as a stream
+const events = await io.consume('numbers_added', 'logger')
+
+for await (const { a, b } of events)
   console.log(`${a} was added to ${b}`)
-})
 ```
 
 ## Emission
 
-`async IO.emit(exchange: string, payload: any, [encoding: string]): void`
+`async IO.emit(exchange: string, payload: any, encoding?: string): void`
 
 Publish encoded Event to the `exchange`.
 
@@ -151,6 +161,111 @@ asserted.
 await io.emit('numbers_added', { a: 1, b: 2 })
 ```
 
+## Pipelines
+
+Payloads for Requests and Events can be passed as a readable stream
+in [object mode](https://nodejs.org/api/stream.html#object-mode), enabling the handling of large amounts of data with
+the benefits of RabbitMQ back pressure and flow control.
+
+`async IO.request(queue: string, stream: Readable, encoding?: string): Readable`
+
+Returns a readable stream of replies.
+
+`async IO.emit(exchange: string, stream: Readable, encoding?: string): void`
+
+```javascript
+function * generate () {
+  yield { a: 1, b: 2 };
+  yield { a: 3, b: 4 };
+}
+
+const events = Readable.from(generate())
+
+await io.emit('numbers_added', events)
+
+const requests = Readable.from(generate())
+
+for await (const reply of io.request('add_numbers', requests))
+  console.log(reply)
+```
+
+## Reply streams
+
+The `producer` function of [`IO.reply`](#reply) may return a non-array
+(Async)[Iterator](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Iteration_protocols).
+In this case, the yielded values will be sent to the `replyTo` queue until the iterator is finished,
+or a [cancellation message](#stream-control) is received, or the `replyTo` queue is deleted.
+
+```javascript
+await io.reply('get_numbers', function * ({ amount }) {
+  for (let i = 0; i < amount; i++) yield i
+})
+```
+
+The Reply stream may be consumed by using the `IO.fetch` function:
+
+`async IO.fetch(queue: string, payload: any, encoding?: string): Readable`
+
+```javascript
+const stream = await io.fetch('get_numbers', { amount: 10 })
+
+for await (const number of stream)
+  console.log(number)
+```
+
+### Stream topology
+
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="./docs/reply-stream-topology-dark.jpg">
+  <img alt="Reply topology" width="500" height="411" src="./docs/reply-stream-topology-light.jpg">
+</picture>
+
+On the first call of the `IO.fetch` for each request queue, an exclusive queue for replies is asserted (a stream queue).
+
+When the producer function of `IO.reply` returns an Iterator for the first time across all request queues,
+a control queue is asserted on the [Reply channel](#channels)
+using the [reply topology](#exchanges-and-queues).
+When the Consumer destroys the Reply stream, a stream cancellation message is sent to the Producer's control queue.
+
+### Stream control
+
+Upon receiving a request, the Producer sends a confirmation message to the `replyTo` queue.
+If the underlying connection is lost before the Consumer receives the confirmation message,
+the request will be retransmitted upon reconnection.
+
+A heartbeat message is sent to the `replyTo` queue whenever a Reply stream idles for 5 seconds.
+If the Consumer of the reply stream doesn't receive a reply or a heartbeat message for 12 seconds, the stream returned
+by `IO.fetch` is destroyed.
+These intervals are not configurable.
+
+An "end stream" message is sent to the `replyTo` queue when the Reply stream is finished.
+
+See also [Reply stream shutdown](#reply-stream-shutdown).
+
+### Loss of tail
+
+:warning:
+
+While consuming the Reply stream if the broker connection is lost,
+or if the Consumer crashes or destroys the stream returned by `IO.fetch`,
+some of the values yielded by the Reply stream may be lost.
+
+To avoid inconsistency, it is strongly recommended to use the Reply stream only with _safe_ Producers, which do not
+change the application state.
+
+### Stream guarantees
+
+:warning:
+
+The reply topology guarantees
+that the order of yielded values is [preserved](https://www.rabbitmq.com/queues.html#message-ordering).
+At the same time, there is no guarantee that the stream will be transmitted to the end.
+
+> When using the [Sharded connection](#sharded-connection), the order of yielded values is maintained through buffering.
+> However, there is a scenario in which some of the yielded values may be lost if a broker crashes.
+> In this case, the Reply stream will be destroyed once the buffer's maximum size is exceeded.
+> Also, buffered control messages can result in [stream idling](#stream-control).
+
 ## Encoding
 
 By default, outgoing message contents are encoded with [msgpack](https://msgpack.org) and
@@ -270,8 +385,10 @@ requests and are expecting replies.
 ### Exchanges and queues
 
 - Exchanges and queues for Events, and queues for Requests
-  are [durable](https://amqp-node.github.io/amqplib/channel_api.html#channel_assertQueue).
-- Queues for Replies are *exclusive*.
+  are _durable_.
+- Queues for Replies are _exclusive_ and _auto deleted_.
+
+See [queue assertion options](https://amqp-node.github.io/amqplib/channel_api.html#channel_assertQueue).
 
 ### Messages
 
@@ -282,16 +399,16 @@ requests and are expecting replies.
   manual [acknowledgment mode](https://www.rabbitmq.com/confirms.html#acknowledgment-modes),
   and Replies are consumed using automatic mode.
 
-If incoming message causes an exception, then it is "negative acknowledged" and requeued. If it
+If an incoming message causes an exception, then it is "negatively acknowledged" and requeued. If it
 causes an exception again, it will be discarded.
 
-> It is highly recommended to set up a dead letter exchange policy to analyse messages that caused
+> It is highly recommended to set up a dead letter exchange policy to analyze messages that caused
 > exceptions. Note that in some cases, if the problematic message is a Request, a Consumer will
 > never receive a Reply, and this can result in a prefetch deadlock of a Consumer.
 
 See:
 
-- [Consumer Acknowledgments and Publisher Confirms](https://www.rabbitmq.com/confirms.html).
+- [Consumer Acknowledgments and Publisher Confirms](https://www.rabbitmq.com/confirms.html)
 - [Negative Acknowledgment and Requeuing of Deliveries](https://www.rabbitmq.com/confirms.html#consumer-nacks-requeue)
 - [Dead Letter Exchanges](https://www.rabbitmq.com/dlx.html)
 
@@ -311,14 +428,14 @@ See:
 
 [Stop receiving](https://amqp-node.github.io/amqplib/channel_api.html#channel_cancel) new Events and
 Requests.
+Sending Requests, receiving Replies, and emitting Events will still be available.
 
 ### Disconnection
 
 `async IO.close(): void`
 
 1. Call `IO.seal()`.
-2. Wait for any outstanding messages to be processed[^2] and acknowledged. Sending Requests,
-   receiving Replies, and emitting Events will still be available.
+2. Wait for any outstanding messages to be processed[^2] and acknowledged.
 3. Close the connection.
 
 [^2]: Therefore, if the underlying connection is lost, `.close()` will only be completed once the
@@ -341,6 +458,13 @@ In these or other similar scenarios, it is recommended to call `IO.seal()` to st
 messages, ensure that any code execution that may send outgoing messages is completed before
 calling `IO.close()`.
 
+### Reply stream shutdown
+
+All current [Reply streams](#reply-streams) of the corresponding Producer or Consumer instance are destroyed when:
+
+- the `IO.seal` function is called on the Consumer
+- the `IO.close` function is called on the Producer
+
 ## Diagnostics
 
 `IO` emits events for testing, diagnostics, or logging purposes.
@@ -362,12 +486,16 @@ Subscribe to one of the diagnostic events:
   exceptions. Channel type,
   raw [amqp message object](https://amqp-node.github.io/amqplib/channel_api.html#channel_consume)
   and the exception are passed as arguments.
+- `pause`: channel is paused. Channel type is passed.
+- `resume`: channel is resumed. Channel type is passed.
 
 In the case of a [sharded connection](#sharded-connection), an additional argument specifying the
-shard number will be passed to listeners. The shard number corresponds to the number of the argument
-used in the `connect` function call.
+shard number will be passed to listeners.
+This is applicable except for the `pause` and `resume` events,
+which are emitted when the associated channels are paused or resumed across all shards.
+The shard number corresponds to the position of the argument used in the `connect` function call.
 
-[^3]: As [`connect`](#connect) function returns an instance of `IO` *after* the connection has been
+[^3]: As the [`connect`](#connect) function returns an instance of `IO` *after* the connection has been
 established, there is no way to capture the initial `open` event.
 
 ### Example