comq 0.8.0 → 0.10.0

package/source/channel.js

@@ -1,7 +1,8 @@
 'use strict'
 
-const { EventEmitter } = require('node:events')
-const { lazy, recall, promex, failsafe, timeout } = require('@toa.io/generic')
+const { promex, timeout } = require('@toa.io/generic')
+const { failsafe, lazy, recall } = require('./attributes')
+const emitter = require('./emitter')
 
 /**
  * @implements {comq.Channel}
@@ -24,8 +25,8 @@ class Channel {
   /** @type {string[]} */
   #tags = []
 
-  /** @type {toa.generic.Promex} */
-  #paused
+  /** @type {toa.generic.Promex | null} */
+  #paused = null
 
   /** @type {boolean} */
   #sealed = false
@@ -36,7 +37,7 @@ class Channel {
   /** @type {Set<toa.generic.Promex>} */
   #confirmations = new Set()
 
-  #diagnostics = new EventEmitter()
+  #diagnostics = emitter.create()
 
   /**
    * @param {comq.amqp.Connection} connection
@@ -56,6 +57,9 @@ class Channel {
   async create () {
     if (this.#topology.confirms) this.#channel = await this.#connection.createConfirmChannel()
     else this.#channel = await this.#connection.createChannel()
+
+    this.#channel.on('drain', this.#unpause)
+    this.#channel.on('return', (message) => this.#diagnostics.emit('return', message))
   }
 
   consume = recall(this,
@@ -63,10 +67,10 @@ class Channel {
       lazy(this, this.#assertQueue,
         /**
          * @param {string} queue
-         * @param {comq.channels.consumer} callback
+         * @param {comq.channels.Consumer} callback
          */
         async (queue, callback) => {
-          if (!this.#sealed) await this.#consume(queue, callback)
+          if (!this.#sealed) return await this.#consume(queue, callback)
         })))
 
   subscribe = recall(this,
@@ -75,7 +79,7 @@ class Channel {
         /**
          * @param {string} exchange
          * @param {string} queue
-         * @param {comq.channels.consumer} callback
+         * @param {comq.channels.Consumer} callback
          * @returns {Promise<void>}
          */
         async (exchange, queue, callback) => {
@@ -107,9 +111,13 @@ class Channel {
   async fire (queue, buffer, options) {
     try {
       await this.#publish(DEFAULT, queue, buffer, options)
+
+      return true
     } catch (exception) {
       if (this.#failfast) throw exception
       // ignore otherwise
+
+      return false
     }
   }
 
@@ -125,6 +133,10 @@ class Channel {
     this.#diagnostics.on(event, listener)
   }
 
+  forget (event, listener) {
+    this.#diagnostics.off(event, listener)
+  }
+
   async recover (connection) {
     this.#connection = connection
 
@@ -137,7 +149,7 @@ class Channel {
 
     for (const confirmation of this.#confirmations) confirmation.reject(INTERRUPTION)
 
-    await timeout(0) // let unpause and confirmation interruptions be handled
+    await timeout(0) // handle interruptions
 
     this.#recovery.resolve()
     this.#recovery = promex()
@@ -197,7 +209,7 @@ class Channel {
    * @param {comq.amqp.options.Publish} options
    */
   async #publish (exchange, queue, buffer, options) {
-    if (this.#paused !== undefined) await this.#unpaused()
+    if (this.#paused !== null) await this.#unpaused()
 
     options = Object.assign({ persistent: this.#topology.persistent }, options)
 
@@ -226,8 +238,8 @@ class Channel {
 
   /**
    * @param {string} queue
-   * @param {comq.channels.consumer} consumer
-   * @returns {Promise<void>}
+   * @param {comq.channels.Consumer} consumer
+   * @returns {Promise<string>}
    */
   async #consume (queue, consumer) {
     /** @type {comq.amqp.options.Consume} */
@@ -239,11 +251,13 @@ class Channel {
     const response = await this.#channel.consume(queue, consumer, options)
 
     this.#tags.push(response.consumerTag)
+
+    return response.consumerTag
   }
 
   /**
-   * @param {comq.channels.consumer} consumer
-   * @returns {comq.channels.consumer}
+   * @param {comq.channels.Consumer} consumer
+   * @returns {comq.channels.Consumer}
    */
   #getAcknowledgingConsumer = (consumer) =>
     async (message) => {
@@ -276,24 +290,25 @@ class Channel {
   }
 
   #pause () {
-    if (this.#paused !== undefined) return
+    if (this.#paused !== null) return
 
     this.#paused = promex()
-    this.#channel.once('drain', this.#unpause)
     this.#diagnostics.emit('flow')
+    this.#diagnostics.emit('pause')
   }
 
   /**
    * @param {Error} [exception]
    */
   #unpause = (exception) => {
-    if (this.#paused === undefined) return
+    if (this.#paused === null) return
 
     if (exception === undefined) this.#paused.resolve()
     else this.#paused.reject(exception)
 
-    this.#paused = undefined
+    this.#paused = null
     this.#diagnostics.emit('drain')
+    this.#diagnostics.emit('resume')
   }
 
   async #unpaused () {

package/source/connect.js

@@ -5,12 +5,12 @@ const { Connection } = require('./connection')
 const { SingletonConnection } = require('./singleton')
 const shards = require('./shards')
 
-/** @type {comq.connect} */
+/** @type {comq.Connect} */
 const connect = async (...urls) => {
   return create(urls, connectionConstructor(Connection))
 }
 
-/** @type {comq.connect} */
+/** @type {comq.Connect} */
 const assert = async (...urls) => {
   return create(urls, connectionConstructor(SingletonConnection))
 }

package/source/connection.js

@@ -1,11 +1,12 @@
 'use strict'
 
-const { EventEmitter } = require('node:events')
 const amqp = require('amqplib')
-const { retry, promex, failsafe } = require('@toa.io/generic')
+const { retry, promex } = require('@toa.io/generic')
 
+const { failsafe } = require('./attributes')
 const presets = require('./topology')
 const channels = require('./channel')
+const emitter = require('./emitter')
 
 /**
  * @implements {comq.Connection}
@@ -26,7 +27,7 @@ class Connection {
   /** @type {boolean} */
   #running = false
 
-  #diagnostics = createEmitter()
+  #diagnostics = emitter.create()
 
   /**
    * @param {string} url
@@ -114,14 +115,6 @@ class Connection {
   }
 }
 
-function createEmitter () {
-  const emitter = new EventEmitter()
-
-  emitter.setMaxListeners(10000)
-
-  return emitter
-}
-
 function noop () {}
 
 /** @type {toa.generic.retry.Options} */

package/source/emitter.js

@@ -0,0 +1,13 @@
+'use strict'
+
+const { EventEmitter } = require('node:events')
+
+function create () {
+  const emitter = new EventEmitter()
+
+  emitter.setMaxListeners(0)
+
+  return emitter
+}
+
+exports.create = create

package/source/encode.js

@@ -4,7 +4,7 @@ const encoders = require('./encoders')
 
 /**
  * @param {any} payload
- * @param {comq.encoding} encoding
+ * @param {comq.Encoding} encoding
  * @returns {Buffer}
  */
 const encode = (payload, encoding) => {

package/source/events.js

@@ -1,7 +1,7 @@
 'use strict'
 
-/** @type {comq.diagnostics.event[]} */
+/** @type {comq.diagnostics.Event[]} */
 exports.connection = ['open', 'close']
 
-/** @type {comq.diagnostics.event[]} */
-exports.channel = ['flow', 'drain', 'recover', 'discard']
+/** @type {comq.diagnostics.Event[]} */
+exports.channel = ['flow', 'drain', 'recover', 'discard', 'pause', 'resume']

package/source/io.js

@@ -1,11 +1,14 @@
 'use strict'
 
+const stream = require('node:stream')
 const { EventEmitter } = require('node:events')
 const { randomBytes } = require('node:crypto')
-const { lazy, track, failsafe, promex } = require('@toa.io/generic')
+const { promex, timeout } = require('@toa.io/generic')
+const { memo, failsafe, lazy, track } = require('./attributes')
 
 const { decode } = require('./decode')
 const { encode } = require('./encode')
+const { pipeline, transform } = require('./pipeline')
 const events = require('./events')
 const io = require('./.io')
 
@@ -28,9 +31,18 @@ class IO {
   /** @type {Map<string, comq.ReplyEmitter>} */
   #emitters = new Map()
 
+  /** @type {comq.ReplyEmitter | null} */
+  #feedback = null
+
   /** @type {Set<toa.generic.Promex>} */
   #pendingReplies = new Set()
 
+  /** @type {Set<comq.Destroyable>} */
+  #replyStreams = new Set()
+
+  /** @type {Set<comq.Destroyable>} */
+  #replyPipes = new Set()
+
   #diagnostics = new EventEmitter()
 
   /**
@@ -47,7 +59,7 @@ class IO {
   reply = lazy(this, this.#createRequestReplyChannels,
     /**
      * @param {string} queue
-     * @param {comq.producer} callback
+     * @param {comq.Producer} callback
      * @returns {Promise<void>}
      */
     async (queue, callback) => {
@@ -61,30 +73,51 @@ class IO {
     failsafe(this, this.#recover,
       /**
        * @param {string} queue
-       * @param {any} payload
-       * @param {comq.encoding} [encoding]
-       * @param {comq.ReplyToPropertyFormatter} [replyToFormatter]
-       * @returns {Promise<void>}
+       * @param {any | Readable} payload
+       * @param {comq.Encoding} [encoding]
+       * @returns {Promise<any | Readable>}
        */
-      async (queue, payload, encoding, replyToFormatter) => {
-        const [buffer, contentType] = this.#encode(payload, encoding)
-        const emitter = this.#emitters.get(queue)
-        const reply = this.#createReply()
-        const correlationId = randomBytes(8).toString('hex')
-        const properties = { contentType, correlationId }
+      async (queue, payload, encoding) => {
+        if (payload instanceof stream.Readable) {
+          return pipeline(
+            payload,
+            (payload) => this.request(queue, payload, encoding),
+            this.#requests
+          )
+        }
 
-        properties.replyTo = replyToFormatter?.(emitter.queue) ?? emitter.queue
+        const { buffer, properties, emitter, reply } = this.#createRequest(queue, payload, encoding)
 
-        emitter.once(correlationId, reply.resolve)
+        emitter.once(properties.correlationId, reply.resolve)
 
         await this.#requests.send(queue, buffer, properties)
 
         return reply
       }))
 
+  fetch = lazy(this, [this.#createRequestReplyChannels, this.#consumeReplies],
+    failsafe(this, this.#recover,
+      /**
+       * @param {string} queue
+       * @param {any} payload
+       * @param {comq.Encoding} [encoding]
+       * @returns {Promise<Readable>}
+       */
+      async (queue, payload, encoding) => {
+        const request = this.#createRequest(queue, payload, encoding)
+        const stream = new io.ReplyStream(request, this.#reply.bind(this))
+
+        this.#addReplyStream(stream)
+
+        await this.#requests.send(queue, request.buffer, request.properties)
+        await request.reply // wait for the confirmation
+
+        return stream
+      }))
+
   consume = lazy(this, this.#createEventChannel,
     async (exchange, group, callback) => {
-      if (callback === undefined) { // 2 arguments passed
+      if (callback === undefined) { // two arguments passed
         callback = group
         group = undefined
       }
@@ -100,17 +133,25 @@ class IO {
     /**
      * @param {string} exchange
      * @param {any} payload
-     * @param {comq.encoding | comq.amqp.options.Publish} [encoding]
+     * @param {comq.Encoding | comq.amqp.options.Publish} [encoding]
      * @returns {Promise<void>}
      */
     async (exchange, payload, encoding) => {
+      if (payload instanceof stream.Readable) {
+        return transform(
+          payload,
+          (payload) => this.emit(exchange, payload, encoding),
+          this.#events
+        )
+      }
+
       /** @type {comq.amqp.options.Publish} */
       const properties = {}
 
       if (typeof encoding === 'object') { // properties passed
         Object.assign(properties, encoding)
 
-        encoding = /** @type {comq.encoding} */ properties.contentType
+        encoding = /** @type {comq.Encoding} */ properties.contentType
       }
 
       const [buffer, contentType] = this.#encode(payload, encoding)
@@ -120,16 +161,18 @@ class IO {
       await this.#events.publish(exchange, buffer, properties)
     })
 
-  async seal () {
+  seal = memo(async () => {
     await this.#requests?.seal()
     await this.#events?.seal()
-  }
+    await this.#destroyStreams(this.#replyStreams)
+  })
 
-  async close () {
+  close = memo(async () => {
     await this.seal()
+    await this.#destroyStreams(this.#replyPipes)
     await track(this)
     await this.#connection.close()
-  }
+  })
 
   diagnose (event, listener) {
     this.#diagnostics.on(event, listener)
@@ -180,8 +223,8 @@ class IO {
   }
 
   /**
-   * @param {comq.producer} producer
-   * @returns {comq.channels.consumer}
+   * @param {comq.Producer} producer
+   * @returns {comq.channels.Consumer}
    */
   #getRequestConsumer = (producer) =>
     track(this,
@@ -193,43 +236,43 @@ class IO {
         const payload = decode(request)
         const reply = await producer(payload)
 
-        if ('replyTo' in request.properties) await this.#reply(request, reply)
-      })
+        if (request.properties.replyTo === undefined) return
 
-  /**
-   * @param {comq.amqp.Message} request
-   * @param {any} reply
-   * @returns {Promise<void>}
-   */
-  async #reply (request, reply) {
-    if (reply === undefined) throw new Error('The `producer` function must return a value')
+        const iterator = typeof reply === 'object' &&
+          (Symbol.asyncIterator in reply ||
+            (Symbol.iterator in reply && !Array.isArray(reply) && !Buffer.isBuffer(reply)))
 
-    let { replyTo, correlationId, contentType } = request.properties
+        if (iterator) {
+          const readable = reply instanceof stream.Readable
+            ? reply
+            : stream.Readable.from(reply)
 
-    if (Buffer.isBuffer(reply)) contentType = OCTETS
-    if (contentType === undefined) throw new Error('Reply to a Request without the `contentType` property must be of type `Buffer`')
+          this.#feedback ??= await this.#createFeedback()
 
-    const buffer = contentType === OCTETS ? reply : encode(reply, contentType)
-    const properties = { contentType, correlationId }
+          const pipe = await io.ReplyPipe.create(request, readable, this.#replies, this.#feedback,
+            (message, properties) => this.#reply(request, message, properties))
 
-    await this.#replies.fire(replyTo, buffer, properties)
-  }
+          this.#addReplyPipe(pipe)
+        } else {
+          await this.#reply(request, reply)
+        }
+      })
 
   /**
    * @param {string} queue
    * @param {comq.ReplyEmitter} emitter
-   * @returns {comq.channels.consumer}
+   * @returns {comq.channels.Consumer}
    */
   #getReplyConsumer = (queue, emitter) =>
     (message) => {
       const payload = decode(message)
 
-      emitter.emit(message.properties.correlationId, payload)
+      emitter.emit(message.properties.correlationId, payload, message.properties)
     }
 
   /**
-   * @param {comq.consumer} callback
-   * @returns {comq.channels.consumer}
+   * @param {comq.Consumer} callback
+   * @returns {comq.channels.Consumer}
    */
   #getEventConsumer = (callback) =>
     track(this, async (message) => {
@@ -238,6 +281,24 @@ class IO {
       await callback(payload, message.properties)
     })
 
+  /**
+   * @param {string} queue
+   * @param {any} payload
+   * @param {comq.Encoding} [encoding]
+   * @return {comq.Request}
+   */
+  #createRequest (queue, payload, encoding) {
+    const [buffer, contentType] = this.#encode(payload, encoding)
+    const emitter = this.#emitters.get(queue)
+    const correlationId = randomBytes(8).toString('hex')
+    const reply = this.#createReply()
+
+    /** @type {comq.amqp.Properties} */
+    const properties = { contentType, correlationId, replyTo: emitter.queue }
+
+    return { buffer, emitter, reply, properties }
+  }
+
   /**
    * @return {toa.generic.Promex}
    */
@@ -253,12 +314,87 @@ class IO {
     return reply
   }
 
+  /**
+   * @return {Promise<comq.ReplyEmitter>}
+   */
+  async #createFeedback () {
+    const queue = 'feedback'
+
+    await this.#consumeReplies(queue)
+
+    return this.#emitters.get(queue)
+  }
+
+  /**
+   * @param {comq.Destroyable} stream
+   */
+  #addReplyStream (stream) {
+    this.#addStream(stream, this.#replyStreams)
+  }
+
+  /**
+   * @param {comq.Destroyable} pipe
+   */
+  #addReplyPipe (pipe) {
+    this.#addStream(pipe, this.#replyPipes)
+  }
+
+  /**
+   * @param {comq.Destroyable} stream
+   * @param {Set<comq.Destroyable>} streams
+   */
+  #addStream (stream, streams) {
+    streams.add(stream)
+    stream.on('close', () => streams.delete(stream))
+  }
+
+  /**
+   * @param {Set<comq.Destroyable>} streams
+   * @return {Promise<void>}
+   */
+  async #destroyStreams (streams) {
+    if (streams.size === 0) return
+
+    for (const stream of streams) stream.destroy()
+
+    /*
+    When streams are destroyed, they attempt to send an 'end' control message.
+    Since these messages are sent without an acknowledgment,
+    it is needed to wait briefly before closing the connection.
+    Even if these messages are lost, the reply stream will be closed anyway,
+    either due to missing heartbeat or the deletion of the stream queue.
+    */
+    await timeout(50)
+  }
+
+  /**
+   * @param {Pick<comq.amqp.Message, 'properties'>} request
+   * @param {any} reply
+   * @param {comq.amqp.options.Publish} [properties]
+   * @returns {Promise<boolean>}
+   */
+  async #reply (request, reply, properties = {}) {
+    if (reply === undefined) throw new Error('The `producer` function must return a value')
+
+    let { replyTo, contentType } = request.properties
+
+    if (Buffer.isBuffer(reply)) contentType = OCTETS
+    if (contentType === undefined) throw new Error('Reply to a Request without the `contentType` property must be of type `Buffer`')
+
+    const buffer = contentType === OCTETS ? reply : encode(reply, contentType)
+
+    properties.contentType = contentType
+    properties.correlationId = request.properties.correlationId
+
+    return await this.#replies.fire(replyTo, buffer, properties)
+  }
+
   #recover (exception) {
     if (exception !== RETRANSMISSION) return false
   }
 
   #retransmit = () => {
-    for (const emitter of this.#emitters.values()) emitter.clear()
+    for (const emitter of this.#emitters.values()) emitter.removeAllListeners()
 
     // trigger failsafe attribute
     for (const reply of this.#pendingReplies) reply.reject(RETRANSMISSION)
@@ -266,8 +402,8 @@ class IO {
 
   /**
    * @param {any} payload
-   * @param {comq.encoding} [contentType]
-   * @returns [Buffer, comq.encoding]
+   * @param {comq.Encoding} [contentType]
+   * @returns {[Buffer, comq.Encoding]}
    */
   #encode (payload, contentType) {
     const raw = Buffer.isBuffer(payload)
@@ -280,10 +416,10 @@ class IO {
   }
 }
 
-/** @type {comq.encoding} */
+/** @type {comq.Encoding} */
 const OCTETS = 'application/octet-stream'
 
-/** @type {comq.encoding} */
+/** @type {comq.Encoding} */
 const DEFAULT = 'application/msgpack'
 
 const RETRANSMISSION = /** @type {Error} */ Symbol('retransmission')

package/source/pipeline.js

@@ -0,0 +1,43 @@
+'use strict'
+
+const { Transform } = require('node:stream')
+const stream = require('node:stream/promises')
+
+function pipeline (source, transform, channel) {
+  const destination = new Pipeline(transform)
+
+  // eslint-disable-next-line no-void
+  void stream.pipeline(source, destination)
+
+  channel.diagnose('pause', source.pause.bind(source))
+  channel.diagnose('resume', source.resume.bind(source))
+
+  return destination
+}
+
+async function transform (source, transform, channel) {
+  const readable = pipeline(source, transform, channel)
+
+  // eslint-disable-next-line no-void, no-unused-vars
+  for await (const _ of readable) void 0
+}
+
+class Pipeline extends Transform {
+  #transform
+
+  constructor (transform) {
+    super({ objectMode: true })
+
+    this.#transform = transform
+  }
+
+  _transform (request, _, callback) {
+    const promise = this.#transform(request)
+
+    this.push(promise)
+    callback()
+  }
+}
+
+exports.pipeline = pipeline
+exports.transform = transform