comq 0.10.3 → 0.11.1

package/docker-compose.yaml

@@ -1,4 +1,3 @@
-version: "3.9"
 services:
   rabbitmq-0:
     image: rabbitmq:3.10.0-management

package/features/events.feature

@@ -23,4 +23,4 @@ Feature: Events
   Scenario: Streaming requests
     Given that events are exclusively consumed from the `numbers_added` exchange
     When a stream of 1k events is emitted to the `numbers_added` exchange
-    Then 1k events is received
+    Then 1k events have been received

package/features/events.poison.feature

@@ -0,0 +1,22 @@
+Feature: Poison events
+
+  Background:
+    Given an active connection to the broker
+
+  @manual
+  Scenario: Sending a poison event
+
+    This scenario will cause an exception, run it once.
+
+    Given that events from the `poison` exchange are causing exceptions
+    Then an event is emitted to the `poison` exchange
+    And after 100ms
+
+  @manual
+  Scenario: Consuming poison events
+
+    After the poison event is sent, run this scenario five times
+    and verify that the poison event was discarded.
+
+    Given that events from the `poison` exchange are causing exceptions
+    And after 100ms

package/features/rpc.feature

@@ -31,15 +31,6 @@ Feature: Request-reply (RPC)
       null
       """
 
-  Scenario: Request causing an exception is discarded
-    Given function replying `throw_error` queue:
-    """
-    () => { throw new Error() }
-    """
-    When the consumer sends a request to the `throw_error` queue
-    Then the message is discarded
-    And the consumer does not receive the reply
-
   Scenario: Streaming requests
     Given a producer replying `ping` queue
     When the consumer sends 1k requests to the `ping` queue as a stream

package/features/steps/.types/context.d.ts

@@ -14,9 +14,12 @@ declare namespace comq.features {
     published?: Buffer
     eventsPublishedCount: number
     eventsConsumedCount: number
-    consumed?: Record<string, { payload: any, properties?: _amqp.Properties }>
+    consumed: Record<string, { payload: any, properties?: _amqp.Properties }>
     consumedCount: number
     events?: { [K in _diagnostics.Event]?: boolean }
+    enqueued?: any
+    processed: any
+    tasksProcessedCount: number
     exception?: Error
     consumptionPromise?: Promise<any>
     sharded: boolean

package/features/steps/context.js

@@ -12,11 +12,14 @@ class Context extends World {
   connecting
   requestsSent = []
   reply
-  consumed
+  consumed = {}
   published
   eventsPublishedCount = 0
   eventsConsumedCount = 0
   events = {}
+  processed
+  enqueued
+  tasksProcessedCount = 0
   exception
   expected
   sharded

package/features/steps/events.js

@@ -38,6 +38,17 @@ Given('(that )events are exclusively consumed from the {token} exchange',
     this.consumptionPromise = consume.call(this, undefined, exchange)
   })
 
+Given('(that )events from the {token} exchange are causing exceptions',
+  /**
+   * @param {string} exchange
+   * @this {comq.features.Context}
+   */
+  async function (exchange) {
+    await this.io.consume(exchange, 'exceptions', () => {
+      throw new Error('Expected exception')
+    })
+  })
+
 When('an event is emitted to the {token} exchange',
   /**
    * @param {string} exchange
@@ -112,7 +123,7 @@ Then('the event is received',
     await consumed.call(this)
   })
 
-Then('{quantity} events is received',
+Then('{quantity} events ha(ve)(s) been received',
   /**
    * @this {comq.features.Context}
    */
@@ -167,8 +178,6 @@ Then('all events have been received',
  * @return {Promise<void>}
  */
 async function consume (group, exchange) {
-  this.consumed ??= {}
-
   const consumer = async (payload, properties) => {
     this.consumed[group] = { payload, properties }
     this.eventsConsumedCount++

package/features/steps/poison.js

@@ -9,7 +9,7 @@ Then('the message is discarded',
    * @this {comq.features.Context}
    */
   async function () {
-    if (this.events.discard !== true) await timeout(50)
+    if (this.events.discard !== true) await timeout(300)
 
     assert.equal(this.events.discard, true, 'The {message} was not discarded')
   })

package/features/steps/tasks.js

@@ -0,0 +1,70 @@
+'use strict'
+
+const assert = require('node:assert')
+const { Given, Then, When } = require('@cucumber/cucumber')
+const { quantity, timeout } = require('@toa.io/generic')
+const { randomBytes } = require('node:crypto')
+const stream = require('node:stream')
+
+Given('tasks from the {token} queue are being processed',
+  /**
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  async function (queue) {
+    const process = (message) => {
+      this.processed = message
+      this.tasksProcessedCount++
+    }
+
+    await this.io.process(queue, process)
+  })
+
+Given('a task is sent to the {token} queue',
+  /**
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  async function (queue) {
+    this.enqueued = 'hello'
+
+    await this.io.enqueue(queue, this.enqueued)
+  })
+
+When('a stream of {quantity} tasks is sent to the {token} queue',
+  /**
+   * @param {string} amountQ
+   * @param {string} exchange
+   * @this {comq.features.Context}
+   */
+  async function (amountQ, exchange) {
+    const amount = quantity(amountQ)
+
+    function * generate () {
+      for (let i = 0; i < amount; i++) yield randomBytes(8)
+    }
+
+    const tasks = stream.Readable.from(generate())
+
+    await this.io.enqueue(exchange, tasks)
+  })
+
+Then('the task has been received',
+  /**
+   * @this {comq.features.Context}
+   */
+  function () {
+    assert.notEqual(this.enqueued, undefined, 'Task was not enqueued')
+    assert.equal(this.processed, this.enqueued, 'Task was not processed')
+  })
+
+Then('{quantity} tasks ha(ve)(s) been processed',
+  /**
+   * @this {comq.features.Context}
+   */
+  async function (expectedQ) {
+    const expected = quantity(expectedQ)
+
+    await timeout(100) // let it process
+    assert.equal(this.tasksProcessedCount, expected, 'Not all tasks have been processed')
+  })

package/features/tasks.feature

@@ -0,0 +1,14 @@
+Feature: Tasks
+
+  Background:
+    Given an active connection to the broker
+
+  Scenario: Sending and receiving a task
+    Given tasks from the `hello` queue are being processed
+    When a task is sent to the `hello` queue
+    Then the task has been received
+
+  Scenario: Streaming tasks
+    Given tasks from the `stream` queue are being processed
+    When a stream of 1k tasks is sent to the `stream` queue
+    Then 1k tasks have been processed

package/features/tasks.shards.feature

@@ -0,0 +1,14 @@
+Feature: Tasks over sharded connection
+
+  Background:
+    Given an active sharded connection
+
+  Scenario: Sending a task using sharded connection
+    Given tasks from the `hello` queue are being processed
+    When a task is sent to the `hello` queue
+    Then the task has been received
+
+  Scenario: Streaming tasks using sharded connection
+    Given tasks from the `sharded` queue are being processed
+    When a stream of 1k tasks is sent to the `sharded` queue
+    Then 1k tasks have been processed

package/lerna.json

@@ -1,4 +1,4 @@
 {
-  "version": "0.10.3",
+  "version": "0.11.1",
   "conventionalCommits": true
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.10.3",
+  "version": "0.11.1",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",
@@ -48,7 +48,7 @@
   "scripts": {
     "prepare": "echo run: husky install",
     "test": "standard && jest",
-    "features": "cucumber-js --fail-fast",
+    "features": "cucumber-js --fail-fast -t 'not @manual'",
     "lint": "standard --fix --verbose | snazzy"
   },
   "standard": {

package/readme.md

@@ -8,6 +8,7 @@ for distributed, eventually consistent systems running on Node.js.
 - [Dynamic topology](#topology)
 - [Request](#request)-[reply](#reply) (RPC)
 - Events ([pub](#emission)/[sub](#consumption))
+- [Tasks](#tasks)
 - [Pipelines](#pipelines)
 - [Reply streams](#reply-streams)
 - [Content encoding](#encoding)
@@ -64,11 +65,13 @@ properties set.
 the `replyTo` property of the Request. The `correlationId` property of the Reply is set to the same
 value as in the Request.
 
-**Event** is an AMQP message that is published to an exchange.
+**Event** is an AMQP message published to an exchange.
 
-**Producer** is an application role that receives Requests and produces Replies and Events.
+**Task** is an AMQP message sent to a queue without a `replyTo` property set.
 
-**Consumer** is an application role that sends Requests and consumes Replies and Events.
+**Producer** is an application role that receives Requests and Tasks, and produces Replies and Events.
+
+**Consumer** is an application role that sends Requests and Tasks, and consumes Replies and Events.
 
 ## Reply
 
@@ -155,9 +158,25 @@ asserted.
 await io.emit('numbers_added', { a: 1, b: 2 })
 ```
 
+## Tasks
+
+`async IO.enqueue(queue: string, payload: any, encoding?: string): void`
+
+Publish encoded Task to the `queue`.
+
+On the initial call, the `queue` is asserted on the Events channel using [Event topology](#topology).
+
+`async IO.process(queue: string, processor): void`
+
+`processor` function's signature is `async? (payload: any): void`
+
+Process decoded Task from the `queue`.
+
+The `queue` is asserted on the Events channel using Event topology.
+
 ## Pipelines
 
-Payloads for Requests and Events can be passed as a readable stream
+Payloads for Requests, Events and Tasks can be passed as a readable stream
 in [object mode](https://nodejs.org/api/stream.html#object-mode), enabling the handling of large amounts of data with
 the benefits of RabbitMQ back pressure and flow control.
 
@@ -167,6 +186,8 @@ Returns a readable stream of replies.
 
 `async IO.emit(exchange: string, stream: Readable, encoding?: string): void`
 
+`async IO.enqueue(queue: string, stream: Readable, encoding?: string): void`
+
 ```javascript
 function * generate () {
   yield { a: 1, b: 2 };
@@ -177,6 +198,10 @@ const events = Readable.from(generate())
 
 await io.emit('numbers_added', events)
 
+const tasks = Readable.from(generate())
+
+await io.enqueue('add_numbers', tasks)
+
 const requests = Readable.from(generate())
 
 for await (const reply of io.request('add_numbers', requests))
@@ -386,8 +411,9 @@ See [queue assertion options](https://amqp-node.github.io/amqplib/channel_api.ht
   manual [acknowledgment mode](https://www.rabbitmq.com/confirms.html#acknowledgment-modes),
   and Replies are consumed using automatic mode.
 
-If an incoming message causes an exception, then it is "negatively acknowledged" and requeued. If it
-causes an exception again, it will be discarded.
+If an incoming message causes an exception, the corresponding channel is sealed, the message is republished, and the
+exception is thrown.
+If the message causes exceptions five times in a row, it is discarded.
 
 > It is highly recommended to set up a dead letter exchange policy to analyze messages that caused
 > exceptions. Note that in some cases, if the problematic message is a Request, a Consumer will

package/source/channel.js

@@ -267,18 +267,29 @@ class Channel {
 
         this.#channel.ack(message)
       } catch (exception) {
-        if (exception.message === 'Channel closed') return // the message will be requeued by the broker
+        if (exception.message === 'Channel closed') { return } // the message is requeued by the broker
 
-        if (message.fields.redelivered) this.#discard(message, exception)
-        else this.#requeue(message)
+        const redeliveries = message.properties.headers[REDELIVERY_HEADER] ?? 0
+
+        if (redeliveries >= MAX_REDELIVERIES) this.#discard(message, exception)
+        else {
+          await this.#requeue(message, redeliveries)
+          throw exception
+        }
       }
     }
 
   /**
    * @param {comq.amqp.Message} message
+   * @param {number} attempt
    */
-  #requeue (message) {
-    this.#channel.nack(message, false, true)
+  async #requeue (message, attempt) {
+    this.#channel.ack(message)
+
+    message.properties.headers[REDELIVERY_HEADER] = attempt + 1
+
+    await this.seal()
+    await this.#publish(message.fields.exchange, message.fields.routingKey, message.content, message.properties)
   }
 
   /**
@@ -358,6 +369,9 @@ const EXCLUSIVE = { exclusive: true }
 
 const INTERRUPTION = /** @type {Error} */ Symbol('internal interruption')
 
+const MAX_REDELIVERIES = 5
+const REDELIVERY_HEADER = 'x-attempt'
+
 function noop () {}
 
 exports.create = create

package/source/io.js

@@ -114,13 +114,14 @@ class IO {
      * @param {string} exchange
      * @param {any} payload
      * @param {comq.Encoding | comq.amqp.options.Publish} [encoding]
+     * @param {'publish' | 'send'} method
      * @returns {Promise<void>}
      */
-    async (exchange, payload, encoding) => {
+    async (exchange, payload, encoding, method = 'publish') => {
       if (payload instanceof stream.Readable) {
         return transform(
           payload,
-          (payload) => this.emit(exchange, payload, encoding),
+          (payload) => this.emit(exchange, payload, encoding, method),
           this.#events
         )
       }
@@ -138,9 +139,20 @@ class IO {
 
       properties.contentType = contentType
 
-      await this.#events.publish(exchange, buffer, properties)
+      await this.#events[method](exchange, buffer, properties)
     })
 
+  process = lazy(this, this.#createEventChannel,
+    async (queue, callback) => {
+      const consumer = this.#getEventConsumer(callback)
+
+      await this.#events.consume(queue, consumer)
+    })
+
+  enqueue (queue, payload, encoding) {
+    return this.emit(queue, payload, encoding, 'send')
+  }
+
   seal = memo(async () => {
     await this.#requests?.seal()
     await this.#events?.seal()

package/test/channel.test.js

@@ -134,32 +134,6 @@ describe('acknowledgments', () => {
     else expect(options).toMatchObject({ noAck: true })
   })
 
-  it.each(/** @type {[string, boolean][]} */ [
-    ['nack', true],
-    ['discard', false]
-  ])('should %s the message caused an exception', async (_, requeue) => {
-    topology.acknowledgments = true
-
-    channel = await create(connection, topology)
-    chan = await getCreatedChannel()
-
-    const consumer = /** @type {Function} */ jest.fn(async () => { throw new Error() })
-
-    await channel.consume(queue, consumer)
-
-    const callback = chan.consume.mock.calls[0][1]
-    const content = randomBytes(8)
-    const properties = {}
-    const fields = {}
-    const message = /** @type {comq.amqp.Message} */ { content, properties, fields }
-
-    if (!requeue) fields.redelivered = !requeue
-
-    await callback(message)
-
-    expect(chan.nack).toHaveBeenCalledWith(message, false, requeue)
-  })
-
   it('should ignore Channel ended exception', async () => {
     topology.acknowledgments = true
 
@@ -844,10 +818,7 @@ describe('diagnostics', () => {
     expect(resumed).toStrictEqual(true)
   })
 
-  it.each(/** @type {[string, boolean][]} */ [
-    ['', true],
-    [' not', false]
-  ])('should%s emit `discard` event', async (_, redelivered) => {
+  it('should emit `discard` event', async () => {
     jest.clearAllMocks()
 
     topology.acknowledgments = true
@@ -866,14 +837,12 @@ describe('diagnostics', () => {
 
     const callback = /** @type {Function} */ chan.consume.mock.calls[0][1]
     const content = randomBytes(8)
-    const properties = {}
-    const fields = { redelivered }
-    const message = /** @type {comq.amqp.Message} */ { content, properties, fields }
+    const properties = { headers: { 'x-attempt': 5 } }
+    const message = /** @type {comq.amqp.Message} */ { content, properties }
 
     await callback(message)
 
-    if (redelivered) expect(listener).toHaveBeenCalledWith(message, exception)
-    else expect(listener).not.toHaveBeenCalled()
+    expect(listener).toHaveBeenCalledWith(message, exception)
   })
 })
 

package/types/io.d.ts

@@ -43,6 +43,16 @@ declare namespace comq {
 
     emit (exchange: string, stream: Readable, properties?: _amqp.Properties): Promise<void>
 
+    enqueue (exchange: string, payload: any, encoding?: _encoding.Encoding): Promise<void>
+
+    enqueue (exchange: string, payload: any, properties?: _amqp.Properties): Promise<void>
+
+    enqueue (exchange: string, stream: Readable, encoding?: _encoding.Encoding): Promise<void>
+
+    enqueue (exchange: string, stream: Readable, properties?: _amqp.Properties): Promise<void>
+
+    process<T = any> (queue: string, consumer: Consumer<T>): Promise<void>
+
     seal (): Promise<void>
 
     close (): Promise<void>