comq 0.10.0 → 0.10.2

package/.github/workflows/release.yaml

@@ -21,7 +21,7 @@ jobs:
 
     steps:
       - name: Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 16
           registry-url: 'https://registry.npmjs.org'

package/features/rpc.streams.feature

@@ -5,14 +5,14 @@ Feature: Reply streams
     And heartbeat interval is set to 100ms
     And idle timeout is set to 150ms
 
-  Scenario: Fetching a stream
+  Scenario: Requesting a stream
     Given function replying `get_numbers` queue:
       """
       function * ({ limit }) {
         for (let i = 0; i < limit; i++) yield i
       }
       """
-    When the consumer fetches a stream with the following request to the `get_numbers` queue:
+    When the consumer requests a stream with the following request to the `get_numbers` queue:
       """yaml
       limit: 5
       """
@@ -21,11 +21,11 @@ Feature: Reply streams
       [0, 1, 2, 3, 4]
       """
 
-  Scenario: Fetching streams concurrently
+  Scenario: Requesting streams concurrently
     Given a number generator with 10ms increasing delay replying `get_numbers` queue
-    When the consumer0 fetches a stream with request to the `get_numbers` queue
+    When the consumer0 requests a stream with request to the `get_numbers` queue
     And the consumer0 receives the stream
-    And the consumer1 fetches a stream with request to the `get_numbers` queue
+    And the consumer1 requests a stream with request to the `get_numbers` queue
     And the consumer1 receives the stream
     And after 300ms
     Then the consumer0 has received the stream:
@@ -39,7 +39,7 @@ Feature: Reply streams
 
   Scenario: Interrupting a stream
     Given a number generator with 20ms increasing delay replying `get_numbers` queue
-    When the consumer fetches a stream with request to the `get_numbers` queue
+    When the consumer requests a stream with request to the `get_numbers` queue
     And the consumer interrupts the stream after 3 replies
     # feedback message must be delivered
     And after 10ms
@@ -48,7 +48,7 @@ Feature: Reply streams
   Scenario: Reply stream idle timeout
     Given heartbeat interval is set to 300ms
     And a number generator with 100ms increasing delay replying `get_numbers` queue
-    When the consumer fetches a stream with request to the `get_numbers` queue
+    When the consumer requests a stream with request to the `get_numbers` queue
     Then the consumer receives the stream:
       """yaml
       [0, 1, 3]
@@ -59,14 +59,14 @@ Feature: Reply streams
   Scenario: Reply stream heartbeat
     # delay will exceed the idle timeout on the 3rd reply
     Given a number generator with 90ms increasing delay replying `get_numbers` queue
-    When the consumer fetches a stream with request to the `get_numbers` queue
+    When the consumer requests a stream with request to the `get_numbers` queue
     Then the consumer interrupts the stream after 5 replies
     And after 200ms
     Then the generator is destroyed
 
   Scenario: Broker crashes while streaming reply
     Given a number generator with 90ms increasing delay replying `get_numbers` queue
-    When the consumer fetches a stream with request to the `get_numbers` queue
+    When the consumer requests a stream with request to the `get_numbers` queue
     Then the consumer receives the stream
     And after 300ms
     Then the broker has crashed

package/features/rpc.streams.shards.feature

@@ -5,14 +5,14 @@ Feature: Reply streams over shards
     And heartbeat interval is set to 100ms
     And idle timeout is set to 150ms
 
-  Scenario: Fetching a stream using sharded connection
+  Scenario: Requesting a stream using sharded connection
     Given a generator replying `get_numbers` queue:
       """
       function * ({ amount }) {
         for (let i = 0; i < amount; i++) yield i
       }
       """
-    When the consumer fetches a stream with the following request to the `get_numbers` queue:
+    When the consumer requests a stream with the following request to the `get_numbers` queue:
       """yaml
       amount: 5
       """
@@ -23,7 +23,7 @@ Feature: Reply streams over shards
 
   Scenario: Broker crashes while fetching a stream
     Given a number generator with 90ms increasing delay replying `get_numbers` queue
-    When the consumer fetches a stream with request to the `get_numbers` queue
+    When the consumer requests a stream with request to the `get_numbers` queue
     And the consumer receives the stream
     And after 300ms
     And the broker has crashed

package/features/steps/broker.js

@@ -32,7 +32,7 @@ Given('one of the brokers is/has {status}',
 const actions = {
   up: async (n = 0) => {
     await execute('docker start comq-rmq-' + n)
-    await healthy()
+    await healthy(n)
   },
   down: async (n = 0) => {
     await execute('docker stop comq-rmq-' + n)
@@ -42,13 +42,13 @@ const actions = {
   }
 }
 
-async function healthy () {
+async function healthy (n) {
   let process
 
   do {
     await timeout(HEALTHCHECK_INTERVAL)
 
-    process = await execute('docker inspect -f {{.State.Health.Status}} comq-rmq-0')
+    process = await execute('docker inspect -f {{.State.Health.Status}} comq-rmq-' + n)
   } while (process.output !== 'healthy')
 }
 

package/features/steps/events.js

@@ -4,7 +4,7 @@ const stream = require('node:stream')
 const assert = require('node:assert')
 const { randomBytes } = require('node:crypto')
 const { timeout, quantity, match } = require('@toa.io/generic')
-const { parse } = require('@toa.io/yaml')
+const { load: parse } = require('js-yaml')
 const { Given, When, Then } = require('@cucumber/cucumber')
 
 Given('(that ){token} is consuming events from the {token} exchange',
@@ -64,6 +64,7 @@ When('an event is emitted to the {token} exchange with properties:',
 
 When('a stream of {quantity} events is emitted to the {token} exchange',
   /**
+   * @param {string} amountQ
    * @param {string} exchange
    * @this {comq.features.Context}
    */

package/features/steps/rpc.js

@@ -3,7 +3,7 @@
 const stream = require('node:stream')
 const assert = require('node:assert')
 const { randomBytes } = require('node:crypto')
-const { parse } = require('@toa.io/yaml')
+const { load: parse } = require('js-yaml')
 const { match, timeout, quantity } = require('@toa.io/generic')
 const { Given, When, Then } = require('@cucumber/cucumber')
 const { Readable } = require('node:stream')
@@ -136,7 +136,7 @@ When('the consumer sends a request to the {token} queue',
     await send.call(this, queue, payload)
   })
 
-When('the consumer fetches a stream with the following request to the {token} queue:',
+When('the consumer requests a stream with the following request to the {token} queue:',
   /**
    * @param {string} queue
    * @param {string} yaml
@@ -148,7 +148,7 @@ When('the consumer fetches a stream with the following request to the {token} qu
     await fetch.call(this, queue, payload)
   })
 
-When('the consumer fetches a stream with request to the {token} queue',
+When('the consumer requests a stream with request to the {token} queue',
   /**
    * @param {string} queue
    * @this {comq.features.Context}
@@ -157,7 +157,7 @@ When('the consumer fetches a stream with request to the {token} queue',
     await fetch.call(this, queue, null)
   })
 
-When('the consumer{number} fetches a stream with request to the {token} queue',
+When('the consumer{number} requests a stream with request to the {token} queue',
   /**
    * @param {number} number
    * @param {string} queue
@@ -408,9 +408,9 @@ async function send (queue, payload) {
  */
 async function fetch (queue, payload, number) {
   if (number === undefined) {
-    this.stream = await this.io.fetch(queue, payload)
+    this.stream = await this.io.request(queue, payload)
   } else {
-    this.streams[number] = await this.io.fetch(queue, payload)
+    this.streams[number] = await this.io.request(queue, payload)
     this.streamsValues[number] = []
     this.streamsEnded[number] = false
   }

package/lerna.json

@@ -1,4 +1,4 @@
 {
-  "version": "0.10.0",
+  "version": "0.10.2",
   "conventionalCommits": true
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.10.0",
+  "version": "0.10.2",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",
@@ -27,27 +27,28 @@
   "main": "source/index.js",
   "types": "types/index.d.ts",
   "devDependencies": {
-    "@cucumber/cucumber": "^9.1.2",
+    "@cucumber/cucumber": "^10.0.1",
     "@toa.io/command": "^0.4.0",
-    "@toa.io/tomato": "^0.4.0",
-    "@toa.io/yaml": "^0.4.0",
-    "@types/amqplib": "^0.10.1",
-    "@types/jest": "^29.5.2",
+    "@toa.io/generic": "^1.0.0-alpha.0",
+    "@types/amqplib": "^0.10.3",
+    "@types/jest": "^29.5.8",
     "husky": "^8.0.3",
     "jest": "^29.5.0",
+    "js-yaml": "^4.1.0",
     "randomstring": "^1.3.0",
     "snazzy": "^9.0.0",
     "standard": "^17.1.0"
   },
   "dependencies": {
-    "@toa.io/generic": "^0.4.0",
     "amqplib": "^0.10.3",
-    "msgpackr": "^1.9.5"
+    "msgpackr": "^1.9.5",
+    "promex": "^0.1.1",
+    "reretry": "^0.1.0"
   },
   "scripts": {
     "prepare": "echo run: husky install",
     "test": "standard && jest",
-    "features": "cucumber-js",
+    "features": "cucumber-js --fail-fast",
     "lint": "standard --fix --verbose | snazzy"
   },
   "standard": {

package/readme.md

@@ -137,12 +137,6 @@ auto-generated name.
 // with a consumer function
 await io.consume('numbers_added', 'logger',
   ({ a, b }) => console.log(`${a} was added to ${b}`))
-
-// as a stream
-const events = await io.consume('numbers_added', 'logger')
-
-for await (const { a, b } of events)
-  console.log(`${a} was added to ${b}`)
 ```
 
 ## Emission
@@ -202,12 +196,10 @@ await io.reply('get_numbers', function * ({ amount }) {
 })
 ```
 
-The Reply stream may be consumed by using the `IO.fetch` function:
-
-`async IO.fetch(queue: string, payload: any, encoding?: string): Readable`
+The Reply stream may be consumed by using the `IO.request` function:
 
 ```javascript
-const stream = await io.fetch('get_numbers', { amount: 10 })
+const stream = await io.request('get_numbers', { amount: 10 })
 
 for await (const number of stream)
   console.log(number)
@@ -220,8 +212,6 @@ for await (const number of stream)
   <img alt="Reply topology" width="500" height="411" src="./docs/reply-stream-topology-light.jpg">
 </picture>
 
-On the first call of the `IO.fetch` for each request queue, an exclusive queue for replies is asserted (a stream queue).
-
 When the producer function of `IO.reply` returns an Iterator for the first time across all request queues,
 a control queue is asserted on the [Reply channel](#channels)
 using the [reply topology](#exchanges-and-queues).
@@ -235,7 +225,7 @@ the request will be retransmitted upon reconnection.
 
 A heartbeat message is sent to the `replyTo` queue whenever a Reply stream idles for 5 seconds.
 If the Consumer of the reply stream doesn't receive a reply or a heartbeat message for 12 seconds, the stream returned
-by `IO.fetch` is destroyed.
+by `IO.request` is destroyed.
 These intervals are not configurable.
 
 An "end stream" message is sent to the `replyTo` queue when the Reply stream is finished.
@@ -247,7 +237,7 @@ See also [Reply stream shutdown](#reply-stream-shutdown).
 :warning:
 
 While consuming the Reply stream if the broker connection is lost,
-or if the Consumer crashes or destroys the stream returned by `IO.fetch`,
+or if the Consumer crashes or destroys the stream returned by `IO.request`,
 some of the values yielded by the Reply stream may be lost.
 
 To avoid inconsistency, it is strongly recommended to use the Reply stream only with _safe_ Producers, which do not
@@ -354,9 +344,6 @@ and it will be closed when the last instance of `IO` using that connection is [d
 Topology is designed to deliver maximum performance while ensuring that the **at least once**
 guarantee provided by RabbitMQ is maintained.
 
-> Topology design assumes that unanswered Requests and unconfirmed Events will be retransmitted
-> upon reconnection.
-
 ### Dynamic
 
 Static topology refers to the process of defining the complete topology declaration along with the

package/source/.io/ReplyStream.js

@@ -1,11 +1,11 @@
 'use strict'
 
-const { IDLE_INTERVAL, control } = require('./const')
-
 const { Readable } = require('node:stream')
+const { Promex } = require('promex')
+const { IDLE_INTERVAL, control } = require('./const')
 
 class ReplyStream extends Readable {
-  #confirmation
+  confirmation = new Promex()
 
   /** @type {ReturnType<setTimeout> | null} */
   #timeout = null
@@ -26,8 +26,8 @@ class ReplyStream extends Readable {
 
   #buffered = 0
 
-  /** @type {Record<number, unknown>} */
-  #queue = {}
+  /** @type {Map<number, unknown>} */
+  #queue = new Map()
 
   /**
    * @param {comq.Request} request
@@ -36,14 +36,12 @@ class ReplyStream extends Readable {
   constructor (request, reply) {
     super({ objectMode: true })
 
-    this.#confirmation = request.reply
     this.#emitter = request.emitter
     this.#correlationId = request.properties.correlationId
     this.#idleInterval = global['COMQ_TESTING_IDLE_INTERVAL'] || IDLE_INTERVAL
     this.#reply = reply
 
-    this.#emitter.on(this.#correlationId, this._arrange.bind(this))
-    this.#confirmation.catch(this._clear.bind(this))
+    this.#emitter.on(this.#correlationId, this.arrange.bind(this))
   }
 
   _destroy (error, callback) {
@@ -59,9 +57,8 @@ class ReplyStream extends Readable {
   /**
    * @param {unknown} payload
    * @param {comq.amqp.Properties} properties
-   * @private
    */
-  _arrange (payload, properties) {
+  arrange (payload, properties) {
     if (properties.headers.index !== this.#index) {
       this._buffer(payload, properties)
 
@@ -73,12 +70,12 @@ class ReplyStream extends Readable {
     if (this.#buffered > 0) {
       let message
 
-      while ((message = this.#queue[this.#index])) {
-        delete this.#queue[this.#index]
+      while ((message = this.#queue.get(this.#index))) {
+        this.#queue.delete(this.#index)
         this._add(message.payload, message.properties)
       }
 
-      this.#buffered = Object.keys(this.#queue).length
+      this.#buffered = this.#queue.size
     }
   }
 
@@ -101,7 +98,7 @@ class ReplyStream extends Readable {
     if (this.#buffered > MAX_BUFFER_SIZE) this.destroy()
 
     this.#buffered++
-    this.#queue[properties.headers.index] = { payload, properties }
+    this.#queue.set(properties.headers.index, { payload, properties })
   }
 
   /**
@@ -113,7 +110,7 @@ class ReplyStream extends Readable {
     switch (message) {
       case control.ok:
         this.#control = { properties }
-        this.#confirmation.resolve()
+        this.confirmation.resolve()
         break
       case control.heartbeat:
         break

package/source/channel.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { promex, timeout } = require('@toa.io/generic')
+const { Promex } = require('promex')
 const { failsafe, lazy, recall } = require('./attributes')
 const emitter = require('./emitter')
 
@@ -25,16 +25,16 @@ class Channel {
   /** @type {string[]} */
   #tags = []
 
-  /** @type {toa.generic.Promex | null} */
+  /** @type {Promex | null} */
   #paused = null
 
   /** @type {boolean} */
   #sealed = false
 
-  /** @type {toa.generic.Promex} */
-  #recovery = promex()
+  /** @type {Promex} */
+  #recovery = new Promex()
 
-  /** @type {Set<toa.generic.Promex>} */
+  /** @type {Set<Promex>} */
   #confirmations = new Set()
 
   #diagnostics = emitter.create()
@@ -149,10 +149,11 @@ class Channel {
 
     for (const confirmation of this.#confirmations) confirmation.reject(INTERRUPTION)
 
-    await timeout(0) // handle interruptions
+    // handle interruptions
+    await new Promise(resolve => setTimeout(resolve, 0))
 
     this.#recovery.resolve()
-    this.#recovery = promex()
+    this.#recovery = new Promex()
     this.#diagnostics.emit('recover')
   }
 
@@ -222,10 +223,10 @@ class Channel {
   }
 
   /**
-   * @return {toa.generic.Promex}
+   * @return {Promex}
    */
   #confirmation () {
-    const confirmation = promex()
+    const confirmation = new Promex()
 
     this.#confirmations.add(confirmation)
 
@@ -292,7 +293,7 @@ class Channel {
   #pause () {
     if (this.#paused !== null) return
 
-    this.#paused = promex()
+    this.#paused = new Promex()
     this.#diagnostics.emit('flow')
     this.#diagnostics.emit('pause')
   }

package/source/connection.js

@@ -1,7 +1,8 @@
 'use strict'
 
 const amqp = require('amqplib')
-const { retry, promex } = require('@toa.io/generic')
+const { Promex } = require('promex')
+const { retry } = require('reretry')
 
 const { failsafe } = require('./attributes')
 const presets = require('./topology')
@@ -21,8 +22,8 @@ class Connection {
   /** @type {comq.Channel[]} */
   #channels = []
 
-  /** @type {toa.generic.Promex} */
-  #recovery = promex()
+  /** @type {Promex} */
+  #recovery = new Promex()
 
   /** @type {boolean} */
   #running = false
@@ -37,7 +38,7 @@ class Connection {
   }
 
   async open () {
-    await retry(this.#open, RETRY)
+    await retry(this.#open)
 
     this.#running = true
   }
@@ -69,14 +70,11 @@ class Connection {
     this.#diagnostics.on(event, listener)
   }
 
-  /**
-   * @type {toa.generic.retry.Task}
-   */
   #open = async (retry) => {
     try {
       this.#connection = await amqp.connect(this.#url)
     } catch (exception) {
-      if (this.#transient(exception)) return retry()
+      if (this.#transient(exception)) return retry
       else throw exception
     }
 
@@ -90,7 +88,7 @@ class Connection {
     for (const channel of this.#channels) await channel.recover(this.#connection)
 
     this.#recovery.resolve()
-    this.#recovery = promex()
+    this.#recovery = new Promex()
   }
 
   /**
@@ -117,9 +115,4 @@ class Connection {
 
 function noop () {}
 
-/** @type {toa.generic.retry.Options} */
-const RETRY = {
-  retries: Infinity
-}
-
 exports.Connection = Connection

package/source/io.js

@@ -3,7 +3,8 @@
 const stream = require('node:stream')
 const { EventEmitter } = require('node:events')
 const { randomBytes } = require('node:crypto')
-const { promex, timeout } = require('@toa.io/generic')
+const { setTimeout } = require('node:timers/promises')
+const { Promex } = require('promex')
 const { memo, failsafe, lazy, track } = require('./attributes')
 
 const { decode } = require('./decode')
@@ -32,9 +33,9 @@ class IO {
   #emitters = new Map()
 
   /** @type {comq.ReplyEmitter | null} */
-  #feedback = null
+  #control = null
 
-  /** @type {Set<toa.generic.Promex>} */
+  /** @type {Set<Promex>} */
   #pendingReplies = new Set()
 
   /** @type {Set<comq.Destroyable>} */
@@ -86,33 +87,12 @@ class IO {
           )
         }
 
-        const { buffer, properties, emitter, reply } = this.#createRequest(queue, payload, encoding)
-
-        emitter.once(properties.correlationId, reply.resolve)
-
-        await this.#requests.send(queue, buffer, properties)
-
-        return reply
-      }))
-
-  fetch = lazy(this, [this.#createRequestReplyChannels, this.#consumeReplies],
-    failsafe(this, this.#recover,
-      /**
-       * @param {string} queue
-       * @param {any} payload
-       * @param {comq.Encoding} [encoding]
-       * @returns {Promise<Readable>}
-       */
-      async (queue, payload, encoding) => {
         const request = this.#createRequest(queue, payload, encoding)
-        const stream = new io.ReplyStream(request, this.#reply.bind(this))
-
-        this.#addReplyStream(stream)
+        const reply = this.#createReply(request)
 
         await this.#requests.send(queue, request.buffer, request.properties)
-        await request.reply // wait for the confirmation
 
-        return stream
+        return reply
       }))
 
   consume = lazy(this, this.#createEventChannel,
@@ -247,9 +227,9 @@ class IO {
             ? reply
             : stream.Readable.from(reply)
 
-          this.#feedback ??= await this.#createFeedback()
+          this.#control ??= await this.#createControl()
 
-          const pipe = await io.ReplyPipe.create(request, readable, this.#replies, this.#feedback,
+          const pipe = await io.ReplyPipe.create(request, readable, this.#replies, this.#control,
             (message, properties) => this.#reply(request, message, properties))
 
           this.#addReplyPipe(pipe)
@@ -291,19 +271,30 @@ class IO {
     const [buffer, contentType] = this.#encode(payload, encoding)
     const emitter = this.#emitters.get(queue)
     const correlationId = randomBytes(8).toString('hex')
-    const reply = this.#createReply()
 
     /** @type {comq.amqp.Properties} */
     const properties = { contentType, correlationId, replyTo: emitter.queue }
 
-    return { buffer, emitter, reply, properties }
+    return { buffer, emitter, properties }
+  }
+
+  /**
+   * @param {comq.Request} request
+   * @return {Promex<any>}
+   */
+  #createReply (request) {
+    const reply = this.#createPendingReply()
+
+    request.emitter.once(request.properties.correlationId, this.#getReplyResolver(request, reply))
+
+    return reply
   }
 
   /**
-   * @return {toa.generic.Promex}
+   * @return {Promex}
    */
-  #createReply () {
-    const reply = promex()
+  #createPendingReply () {
+    const reply = new Promex()
 
     this.#pendingReplies.add(reply)
 
@@ -314,11 +305,40 @@ class IO {
     return reply
   }
 
+  /**
+   * @param {comq.Request} request
+   * @param reply
+   */
+  #getReplyResolver (request, reply) {
+    return async (payload, properties) => {
+      const isStream = properties.headers?.index !== undefined
+
+      if (isStream) {
+        const stream = this.#createReplyStream(request, payload, properties)
+
+        await stream.confirmation
+
+        reply.resolve(stream)
+      } else {
+        reply.resolve(payload)
+      }
+    }
+  }
+
+  #createReplyStream (request, payload, properties) {
+    const stream = new io.ReplyStream(request, this.#reply.bind(this))
+
+    stream.arrange(payload, properties)
+    this.#addReplyStream(/** @type {comq.Destroyable} */ stream)
+
+    return stream
+  }
+
   /**
    * @return {Promise<comq.ReplyEmitter>}
    */
-  async #createFeedback () {
-    const queue = 'feedback'
+  async #createControl () {
+    const queue = 'control'
 
     await this.#consumeReplies(queue)
 
@@ -364,7 +384,7 @@ class IO {
     Even if these messages are lost, the reply stream will be closed anyway,
     either due to missing heartbeat or the deletion of the stream queue.
     */
-    await timeout(50)
+    await setTimeout(50)
   }
 
   /**