comq 0.10.2 → 0.11.0

package/.github/workflows/codeql.yml

@@ -46,7 +46,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -60,7 +60,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -73,6 +73,6 @@ jobs:
     #     ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3
       with:
         category: "/language:${{matrix.language}}"

package/.github/workflows/release.yaml

@@ -23,7 +23,7 @@ jobs:
       - name: Setup Node
         uses: actions/setup-node@v4
         with:
-          node-version: 16
+          node-version: 20
           registry-url: 'https://registry.npmjs.org'
 
       - name: Checkout

package/docker-compose.yaml

@@ -1,4 +1,3 @@
-version: "3.9"
 services:
   rabbitmq-0:
     image: rabbitmq:3.10.0-management

package/features/events.feature

@@ -23,4 +23,4 @@ Feature: Events
   Scenario: Streaming requests
     Given that events are exclusively consumed from the `numbers_added` exchange
     When a stream of 1k events is emitted to the `numbers_added` exchange
-    Then 1k events is received
+    Then 1k events have been received

package/features/rpc.feature

@@ -18,6 +18,19 @@ Feature: Request-reply (RPC)
       3
       """
 
+  Scenario: Getting `null` reply
+    Given function replying `add_numbers` queue:
+      """
+      () => { return null }
+      """
+    When the consumer sends the following request to the `add_numbers` queue:
+      """yaml
+      """
+    Then the consumer receives the reply:
+      """yaml
+      null
+      """
+
   Scenario: Request causing an exception is discarded
     Given function replying `throw_error` queue:
     """

package/features/steps/.types/context.d.ts

@@ -14,9 +14,12 @@ declare namespace comq.features {
     published?: Buffer
     eventsPublishedCount: number
     eventsConsumedCount: number
-    consumed?: Record<string, { payload: any, properties?: _amqp.Properties }>
+    consumed: Record<string, { payload: any, properties?: _amqp.Properties }>
     consumedCount: number
     events?: { [K in _diagnostics.Event]?: boolean }
+    enqueued?: any
+    processed: any
+    tasksProcessedCount: number
     exception?: Error
     consumptionPromise?: Promise<any>
     sharded: boolean

package/features/steps/context.js

@@ -12,11 +12,14 @@ class Context extends World {
   connecting
   requestsSent = []
   reply
-  consumed
+  consumed = {}
   published
   eventsPublishedCount = 0
   eventsConsumedCount = 0
   events = {}
+  processed
+  enqueued
+  tasksProcessedCount = 0
   exception
   expected
   sharded

package/features/steps/events.js

@@ -112,7 +112,7 @@ Then('the event is received',
     await consumed.call(this)
   })
 
-Then('{quantity} events is received',
+Then('{quantity} events ha(ve)(s) been received',
   /**
    * @this {comq.features.Context}
    */
@@ -167,8 +167,6 @@ Then('all events have been received',
  * @return {Promise<void>}
  */
 async function consume (group, exchange) {
-  this.consumed ??= {}
-
   const consumer = async (payload, properties) => {
     this.consumed[group] = { payload, properties }
     this.eventsConsumedCount++

package/features/steps/tasks.js

@@ -0,0 +1,70 @@
+'use strict'
+
+const assert = require('node:assert')
+const { Given, Then, When } = require('@cucumber/cucumber')
+const { quantity, timeout } = require('@toa.io/generic')
+const { randomBytes } = require('node:crypto')
+const stream = require('node:stream')
+
+Given('tasks from the {token} queue are being processed',
+  /**
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  async function (queue) {
+    const process = (message) => {
+      this.processed = message
+      this.tasksProcessedCount++
+    }
+
+    await this.io.process(queue, process)
+  })
+
+Given('a task is sent to the {token} queue',
+  /**
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  async function (queue) {
+    this.enqueued = 'hello'
+
+    await this.io.enqueue(queue, this.enqueued)
+  })
+
+When('a stream of {quantity} tasks is sent to the {token} queue',
+  /**
+   * @param {string} amountQ
+   * @param {string} exchange
+   * @this {comq.features.Context}
+   */
+  async function (amountQ, exchange) {
+    const amount = quantity(amountQ)
+
+    function * generate () {
+      for (let i = 0; i < amount; i++) yield randomBytes(8)
+    }
+
+    const tasks = stream.Readable.from(generate())
+
+    await this.io.enqueue(exchange, tasks)
+  })
+
+Then('the task has been received',
+  /**
+   * @this {comq.features.Context}
+   */
+  function () {
+    assert.notEqual(this.enqueued, undefined, 'Task was not enqueued')
+    assert.equal(this.processed, this.enqueued, 'Task was not processed')
+  })
+
+Then('{quantity} tasks ha(ve)(s) been processed',
+  /**
+   * @this {comq.features.Context}
+   */
+  async function (expectedQ) {
+    const expected = quantity(expectedQ)
+
+    await timeout(100) // let it process
+    assert.equal(this.tasksProcessedCount, expected, 'Not all tasks have been processed')
+  })

package/features/tasks.feature

@@ -0,0 +1,14 @@
+Feature: Tasks
+
+  Background:
+    Given an active connection to the broker
+
+  Scenario: Sending and receiving a task
+    Given tasks from the `hello` queue are being processed
+    When a task is sent to the `hello` queue
+    Then the task has been received
+
+  Scenario: Streaming tasks
+    Given tasks from the `stream` queue are being processed
+    When a stream of 1k tasks is sent to the `stream` queue
+    Then 1k tasks have been processed

package/features/tasks.shards.feature

@@ -0,0 +1,14 @@
+Feature: Tasks over sharded connection
+
+  Background:
+    Given an active sharded connection
+
+  Scenario: Sending a task using sharded connection
+    Given tasks from the `hello` queue are being processed
+    When a task is sent to the `hello` queue
+    Then the task has been received
+
+  Scenario: Streaming tasks using sharded connection
+    Given tasks from the `sharded` queue are being processed
+    When a stream of 1k tasks is sent to the `sharded` queue
+    Then 1k tasks have been processed

package/lerna.json

@@ -1,4 +1,4 @@
 {
-  "version": "0.10.2",
+  "version": "0.11.0",
   "conventionalCommits": true
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.10.2",
+  "version": "0.11.0",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",
@@ -32,7 +32,7 @@
     "@toa.io/generic": "^1.0.0-alpha.0",
     "@types/amqplib": "^0.10.3",
     "@types/jest": "^29.5.8",
-    "husky": "^8.0.3",
+    "husky": "^9.0.11",
     "jest": "^29.5.0",
     "js-yaml": "^4.1.0",
     "randomstring": "^1.3.0",
@@ -40,8 +40,8 @@
     "standard": "^17.1.0"
   },
   "dependencies": {
-    "amqplib": "^0.10.3",
-    "msgpackr": "^1.9.5",
+    "amqplib": "^0.10.4",
+    "msgpackr": "^1.10.2",
     "promex": "^0.1.1",
     "reretry": "^0.1.0"
   },

package/readme.md

@@ -8,6 +8,7 @@ for distributed, eventually consistent systems running on Node.js.
 - [Dynamic topology](#topology)
 - [Request](#request)-[reply](#reply) (RPC)
 - Events ([pub](#emission)/[sub](#consumption))
+- [Tasks](#tasks)
 - [Pipelines](#pipelines)
 - [Reply streams](#reply-streams)
 - [Content encoding](#encoding)
@@ -64,11 +65,13 @@ properties set.
 the `replyTo` property of the Request. The `correlationId` property of the Reply is set to the same
 value as in the Request.
 
-**Event** is an AMQP message that is published to an exchange.
+**Event** is an AMQP message published to an exchange.
 
-**Producer** is an application role that receives Requests and produces Replies and Events.
+**Task** is an AMQP message sent to a queue without a `replyTo` property set.
 
-**Consumer** is an application role that sends Requests and consumes Replies and Events.
+**Producer** is an application role that receives Requests and Tasks, and produces Replies and Events.
+
+**Consumer** is an application role that sends Requests and Tasks, and consumes Replies and Events.
 
 ## Reply
 
@@ -155,6 +158,22 @@ asserted.
 await io.emit('numbers_added', { a: 1, b: 2 })
 ```
 
+## Tasks
+
+`async IO.enqueue(queue: string, payload: any, encoding?: string): void`
+
+Publish encoded Task to the `queue`.
+
+On the initial call, the `queue` is asserted on the Events channel using Event topology.
+
+`async IO.process(queue: string, processor): void`
+
+`processor` function's signature is `async? (payload: any): void`
+
+Process decoded Task from the `queue`.
+
+The `queue` is asserted on the Events channel using Event topology.
+
 ## Pipelines
 
 Payloads for Requests and Events can be passed as a readable stream

package/source/io.js

@@ -114,13 +114,14 @@ class IO {
      * @param {string} exchange
      * @param {any} payload
      * @param {comq.Encoding | comq.amqp.options.Publish} [encoding]
+     * @param {'publish' | 'send'} method
      * @returns {Promise<void>}
      */
-    async (exchange, payload, encoding) => {
+    async (exchange, payload, encoding, method = 'publish') => {
       if (payload instanceof stream.Readable) {
         return transform(
           payload,
-          (payload) => this.emit(exchange, payload, encoding),
+          (payload) => this.emit(exchange, payload, encoding, method),
           this.#events
         )
       }
@@ -138,9 +139,20 @@ class IO {
 
       properties.contentType = contentType
 
-      await this.#events.publish(exchange, buffer, properties)
+      await this.#events[method](exchange, buffer, properties)
     })
 
+  process = lazy(this, this.#createEventChannel,
+    async (queue, callback) => {
+      const consumer = this.#getEventConsumer(callback)
+
+      await this.#events.consume(queue, consumer)
+    })
+
+  enqueue (queue, payload, encoding) {
+    return this.emit(queue, payload, encoding, 'send')
+  }
+
   seal = memo(async () => {
     await this.#requests?.seal()
     await this.#events?.seal()
@@ -218,7 +230,7 @@ class IO {
 
         if (request.properties.replyTo === undefined) return
 
-        const iterator = typeof reply === 'object' &&
+        const iterator = typeof reply === 'object' && reply !== null &&
           (Symbol.asyncIterator in reply ||
             (Symbol.iterator in reply && !Array.isArray(reply) && !Buffer.isBuffer(reply)))
 

package/types/io.d.ts

@@ -43,6 +43,16 @@ declare namespace comq {
 
     emit (exchange: string, stream: Readable, properties?: _amqp.Properties): Promise<void>
 
+    enqueue (exchange: string, payload: any, encoding?: _encoding.Encoding): Promise<void>
+
+    enqueue (exchange: string, payload: any, properties?: _amqp.Properties): Promise<void>
+
+    enqueue (exchange: string, stream: Readable, encoding?: _encoding.Encoding): Promise<void>
+
+    enqueue (exchange: string, stream: Readable, properties?: _amqp.Properties): Promise<void>
+
+    process<T = any> (queue: string, consumer: Consumer<T>): Promise<void>
+
     seal (): Promise<void>
 
     close (): Promise<void>