comq 0.10.0 → 0.10.1

package/features/rpc.streams.feature

@@ -5,14 +5,14 @@ Feature: Reply streams
     And heartbeat interval is set to 100ms
     And idle timeout is set to 150ms
 
-  Scenario: Fetching a stream
+  Scenario: Requesting a stream
     Given function replying `get_numbers` queue:
       """
       function * ({ limit }) {
         for (let i = 0; i < limit; i++) yield i
       }
       """
-    When the consumer fetches a stream with the following request to the `get_numbers` queue:
+    When the consumer requests a stream with the following request to the `get_numbers` queue:
       """yaml
       limit: 5
       """
@@ -21,11 +21,11 @@ Feature: Reply streams
       [0, 1, 2, 3, 4]
       """
 
-  Scenario: Fetching streams concurrently
+  Scenario: Requesting streams concurrently
     Given a number generator with 10ms increasing delay replying `get_numbers` queue
-    When the consumer0 fetches a stream with request to the `get_numbers` queue
+    When the consumer0 requests a stream with request to the `get_numbers` queue
     And the consumer0 receives the stream
-    And the consumer1 fetches a stream with request to the `get_numbers` queue
+    And the consumer1 requests a stream with request to the `get_numbers` queue
     And the consumer1 receives the stream
     And after 300ms
     Then the consumer0 has received the stream:
@@ -39,7 +39,7 @@ Feature: Reply streams
 
   Scenario: Interrupting a stream
     Given a number generator with 20ms increasing delay replying `get_numbers` queue
-    When the consumer fetches a stream with request to the `get_numbers` queue
+    When the consumer requests a stream with request to the `get_numbers` queue
     And the consumer interrupts the stream after 3 replies
     # feedback message must be delivered
     And after 10ms
@@ -48,7 +48,7 @@ Feature: Reply streams
   Scenario: Reply stream idle timeout
     Given heartbeat interval is set to 300ms
     And a number generator with 100ms increasing delay replying `get_numbers` queue
-    When the consumer fetches a stream with request to the `get_numbers` queue
+    When the consumer requests a stream with request to the `get_numbers` queue
     Then the consumer receives the stream:
       """yaml
       [0, 1, 3]
@@ -59,14 +59,14 @@ Feature: Reply streams
   Scenario: Reply stream heartbeat
     # delay will exceed the idle timeout on the 3rd reply
     Given a number generator with 90ms increasing delay replying `get_numbers` queue
-    When the consumer fetches a stream with request to the `get_numbers` queue
+    When the consumer requests a stream with request to the `get_numbers` queue
     Then the consumer interrupts the stream after 5 replies
     And after 200ms
     Then the generator is destroyed
 
   Scenario: Broker crashes while streaming reply
     Given a number generator with 90ms increasing delay replying `get_numbers` queue
-    When the consumer fetches a stream with request to the `get_numbers` queue
+    When the consumer requests a stream with request to the `get_numbers` queue
     Then the consumer receives the stream
     And after 300ms
     Then the broker has crashed

package/features/rpc.streams.shards.feature

@@ -5,14 +5,14 @@ Feature: Reply streams over shards
     And heartbeat interval is set to 100ms
     And idle timeout is set to 150ms
 
-  Scenario: Fetching a stream using sharded connection
+  Scenario: Requesting a stream using sharded connection
     Given a generator replying `get_numbers` queue:
       """
       function * ({ amount }) {
         for (let i = 0; i < amount; i++) yield i
       }
       """
-    When the consumer fetches a stream with the following request to the `get_numbers` queue:
+    When the consumer requests a stream with the following request to the `get_numbers` queue:
       """yaml
       amount: 5
       """
@@ -23,7 +23,7 @@ Feature: Reply streams over shards
 
   Scenario: Broker crashes while fetching a stream
     Given a number generator with 90ms increasing delay replying `get_numbers` queue
-    When the consumer fetches a stream with request to the `get_numbers` queue
+    When the consumer requests a stream with request to the `get_numbers` queue
     And the consumer receives the stream
     And after 300ms
     And the broker has crashed

package/features/steps/broker.js

@@ -50,6 +50,8 @@ async function healthy () {
 
     process = await execute('docker inspect -f {{.State.Health.Status}} comq-rmq-0')
   } while (process.output !== 'healthy')
+
+  await timeout(500)
 }
 
 const BROKERS_AMOUNT = 2

package/features/steps/rpc.js

@@ -136,7 +136,7 @@ When('the consumer sends a request to the {token} queue',
     await send.call(this, queue, payload)
   })
 
-When('the consumer fetches a stream with the following request to the {token} queue:',
+When('the consumer requests a stream with the following request to the {token} queue:',
   /**
    * @param {string} queue
    * @param {string} yaml
@@ -148,7 +148,7 @@ When('the consumer fetches a stream with the following request to the {token} qu
     await fetch.call(this, queue, payload)
   })
 
-When('the consumer fetches a stream with request to the {token} queue',
+When('the consumer requests a stream with request to the {token} queue',
   /**
    * @param {string} queue
    * @this {comq.features.Context}
@@ -157,7 +157,7 @@ When('the consumer fetches a stream with request to the {token} queue',
     await fetch.call(this, queue, null)
   })
 
-When('the consumer{number} fetches a stream with request to the {token} queue',
+When('the consumer{number} requests a stream with request to the {token} queue',
   /**
    * @param {number} number
    * @param {string} queue
@@ -408,9 +408,9 @@ async function send (queue, payload) {
  */
 async function fetch (queue, payload, number) {
   if (number === undefined) {
-    this.stream = await this.io.fetch(queue, payload)
+    this.stream = await this.io.request(queue, payload)
   } else {
-    this.streams[number] = await this.io.fetch(queue, payload)
+    this.streams[number] = await this.io.request(queue, payload)
     this.streamsValues[number] = []
     this.streamsEnded[number] = false
   }

package/lerna.json

@@ -1,4 +1,4 @@
 {
-  "version": "0.10.0",
+  "version": "0.10.1",
   "conventionalCommits": true
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.10.0",
+  "version": "0.10.1",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",
@@ -47,7 +47,7 @@
   "scripts": {
     "prepare": "echo run: husky install",
     "test": "standard && jest",
-    "features": "cucumber-js",
+    "features": "cucumber-js --publish-quiet",
     "lint": "standard --fix --verbose | snazzy"
   },
   "standard": {

package/readme.md

@@ -202,12 +202,10 @@ await io.reply('get_numbers', function * ({ amount }) {
 })
 ```
 
-The Reply stream may be consumed by using the `IO.fetch` function:
-
-`async IO.fetch(queue: string, payload: any, encoding?: string): Readable`
+The Reply stream may be consumed by using the `IO.request` function:
 
 ```javascript
-const stream = await io.fetch('get_numbers', { amount: 10 })
+const stream = await io.request('get_numbers', { amount: 10 })
 
 for await (const number of stream)
   console.log(number)
@@ -220,8 +218,6 @@ for await (const number of stream)
   <img alt="Reply topology" width="500" height="411" src="./docs/reply-stream-topology-light.jpg">
 </picture>
 
-On the first call of the `IO.fetch` for each request queue, an exclusive queue for replies is asserted (a stream queue).
-
 When the producer function of `IO.reply` returns an Iterator for the first time across all request queues,
 a control queue is asserted on the [Reply channel](#channels)
 using the [reply topology](#exchanges-and-queues).
@@ -235,7 +231,7 @@ the request will be retransmitted upon reconnection.
 
 A heartbeat message is sent to the `replyTo` queue whenever a Reply stream idles for 5 seconds.
 If the Consumer of the reply stream doesn't receive a reply or a heartbeat message for 12 seconds, the stream returned
-by `IO.fetch` is destroyed.
+by `IO.request` is destroyed.
 These intervals are not configurable.
 
 An "end stream" message is sent to the `replyTo` queue when the Reply stream is finished.
@@ -247,7 +243,7 @@ See also [Reply stream shutdown](#reply-stream-shutdown).
 :warning:
 
 While consuming the Reply stream if the broker connection is lost,
-or if the Consumer crashes or destroys the stream returned by `IO.fetch`,
+or if the Consumer crashes or destroys the stream returned by `IO.request`,
 some of the values yielded by the Reply stream may be lost.
 
 To avoid inconsistency, it is strongly recommended to use the Reply stream only with _safe_ Producers, which do not

package/source/.io/ReplyStream.js

@@ -1,11 +1,11 @@
 'use strict'
 
-const { IDLE_INTERVAL, control } = require('./const')
-
 const { Readable } = require('node:stream')
+const { promex } = require('@toa.io/generic')
+const { IDLE_INTERVAL, control } = require('./const')
 
 class ReplyStream extends Readable {
-  #confirmation
+  confirmation = promex()
 
   /** @type {ReturnType<setTimeout> | null} */
   #timeout = null
@@ -36,14 +36,12 @@ class ReplyStream extends Readable {
   constructor (request, reply) {
     super({ objectMode: true })
 
-    this.#confirmation = request.reply
     this.#emitter = request.emitter
     this.#correlationId = request.properties.correlationId
     this.#idleInterval = global['COMQ_TESTING_IDLE_INTERVAL'] || IDLE_INTERVAL
     this.#reply = reply
 
-    this.#emitter.on(this.#correlationId, this._arrange.bind(this))
-    this.#confirmation.catch(this._clear.bind(this))
+    this.#emitter.on(this.#correlationId, this.arrange.bind(this))
   }
 
   _destroy (error, callback) {
@@ -59,9 +57,8 @@ class ReplyStream extends Readable {
   /**
    * @param {unknown} payload
    * @param {comq.amqp.Properties} properties
-   * @private
    */
-  _arrange (payload, properties) {
+  arrange (payload, properties) {
     if (properties.headers.index !== this.#index) {
       this._buffer(payload, properties)
 
@@ -113,7 +110,7 @@ class ReplyStream extends Readable {
     switch (message) {
       case control.ok:
         this.#control = { properties }
-        this.#confirmation.resolve()
+        this.confirmation.resolve()
         break
       case control.heartbeat:
         break

package/source/io.js

@@ -32,7 +32,7 @@ class IO {
   #emitters = new Map()
 
   /** @type {comq.ReplyEmitter | null} */
-  #feedback = null
+  #control = null
 
   /** @type {Set<toa.generic.Promex>} */
   #pendingReplies = new Set()
@@ -86,33 +86,12 @@ class IO {
           )
         }
 
-        const { buffer, properties, emitter, reply } = this.#createRequest(queue, payload, encoding)
-
-        emitter.once(properties.correlationId, reply.resolve)
-
-        await this.#requests.send(queue, buffer, properties)
-
-        return reply
-      }))
-
-  fetch = lazy(this, [this.#createRequestReplyChannels, this.#consumeReplies],
-    failsafe(this, this.#recover,
-      /**
-       * @param {string} queue
-       * @param {any} payload
-       * @param {comq.Encoding} [encoding]
-       * @returns {Promise<Readable>}
-       */
-      async (queue, payload, encoding) => {
         const request = this.#createRequest(queue, payload, encoding)
-        const stream = new io.ReplyStream(request, this.#reply.bind(this))
-
-        this.#addReplyStream(stream)
+        const reply = this.#createReply(request)
 
         await this.#requests.send(queue, request.buffer, request.properties)
-        await request.reply // wait for the confirmation
 
-        return stream
+        return reply
       }))
 
   consume = lazy(this, this.#createEventChannel,
@@ -247,9 +226,9 @@ class IO {
             ? reply
             : stream.Readable.from(reply)
 
-          this.#feedback ??= await this.#createFeedback()
+          this.#control ??= await this.#createControl()
 
-          const pipe = await io.ReplyPipe.create(request, readable, this.#replies, this.#feedback,
+          const pipe = await io.ReplyPipe.create(request, readable, this.#replies, this.#control,
             (message, properties) => this.#reply(request, message, properties))
 
           this.#addReplyPipe(pipe)
@@ -291,18 +270,29 @@ class IO {
     const [buffer, contentType] = this.#encode(payload, encoding)
     const emitter = this.#emitters.get(queue)
     const correlationId = randomBytes(8).toString('hex')
-    const reply = this.#createReply()
 
     /** @type {comq.amqp.Properties} */
     const properties = { contentType, correlationId, replyTo: emitter.queue }
 
-    return { buffer, emitter, reply, properties }
+    return { buffer, emitter, properties }
+  }
+
+  /**
+   * @param {comq.Request} request
+   * @return {Promise<any>}
+   */
+  #createReply (request) {
+    const reply = this.#createPendingReply()
+
+    request.emitter.once(request.properties.correlationId, this.#getReplyResolver(request, reply))
+
+    return reply
   }
 
   /**
    * @return {toa.generic.Promex}
    */
-  #createReply () {
+  #createPendingReply () {
     const reply = promex()
 
     this.#pendingReplies.add(reply)
@@ -314,11 +304,40 @@ class IO {
     return reply
   }
 
+  /**
+   * @param {comq.Request} request
+   * @param reply
+   */
+  #getReplyResolver (request, reply) {
+    return async (payload, properties) => {
+      const isStream = properties.headers?.index !== undefined
+
+      if (isStream) {
+        const stream = this.#createReplyStream(request, payload, properties)
+
+        await stream.confirmation
+
+        reply.resolve(stream)
+      } else {
+        reply.resolve(payload)
+      }
+    }
+  }
+
+  #createReplyStream (request, payload, properties) {
+    const stream = new io.ReplyStream(request, this.#reply.bind(this))
+
+    stream.arrange(payload, properties)
+    this.#addReplyStream(/** @type {comq.Destroyable} */ stream)
+
+    return stream
+  }
+
   /**
    * @return {Promise<comq.ReplyEmitter>}
    */
-  async #createFeedback () {
-    const queue = 'feedback'
+  async #createControl () {
+    const queue = 'control'
 
     await this.#consumeReplies(queue)
 

package/types/io.d.ts

@@ -22,18 +22,15 @@ declare namespace comq {
     buffer: Buffer
     emitter: ReplyEmitter
     properties: _amqp.Properties
-    reply: any
   }
 
   interface IO extends _diagnostics.Diagnosable {
     reply (queue: string, produce: Producer): Promise<void>
 
-    request<Reply = any, Request = any> (queue: string, payload: Request, encoding?: _encoding.Encoding): Promise<Reply>
+    request<Reply = any, Request = any> (queue: string, payload: Request, encoding?: _encoding.Encoding): Promise<Reply> | Promise<Readable>
 
     request (queue: string, stream: Readable, encoding?: _encoding.Encoding): Promise<Readable>
 
-    fetch (queue: string, payload: any, encoding?: _encoding.Encoding): Promise<Readable>
-
     consume<T = any> (exchange: string, group: string, consumer: Consumer<T>): Promise<void>
 
     consume<T = any> (exchange: string, consumer: Consumer<T>): Promise<void>