comq 0.4.1 → 0.5.0

package/.github/labels.yaml

@@ -0,0 +1,15 @@
+- name: "Bug"
+  description: "Something isn't working"
+  color: "d73a4a"
+- name: "Critical"
+  description: "Immediate action required"
+  color: "ffff00"
+- name: "Docs"
+  description: "Documentation improvements"
+  color: "0075ca"
+- name: "Feature"
+  description: "Feature request"
+  color: "73E25B"
+- name: "Improvement"
+  description: "Make something better"
+  color: "6A1877"

package/.github/workflows/labels.yaml

@@ -0,0 +1,18 @@
+name: Labels
+
+on:
+  push:
+    branches:
+      - 'dev'
+    paths:
+      - '.github/labels.yml'
+      - '.github/workflows/labels.yml'
+
+jobs:
+  labeler:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Run Labeler
+        uses: crazy-max/ghaction-github-labeler@v4

package/.github/workflows/release.yaml

@@ -6,6 +6,10 @@ on:
     branches:
       - main
       - dev
+    paths:
+      - 'source/**'
+      - 'types/**'
+      - 'package-lock.json'
 
 jobs:
   publish:
@@ -27,24 +31,32 @@ jobs:
       - name: Run tests
         run: npm i && npm test
 
-      - name: Run features
-        if: github.ref == 'refs/heads/main'
-        run: docker-compose up -d && npx cucumber-js --fail-fast --exit --format summary --publish-quiet
+      - name: Import GPG key
+        id: gpg
+        uses: crazy-max/ghaction-import-gpg@v5
+        with:
+          gpg_private_key: ${{ secrets.TOABOT_GPG_PRIVATE_KEY }}
+          passphrase: ${{ secrets.TOABOT_GPG_PASSPHRASE }}
+          git_user_signingkey: true
+          git_commit_gpgsign: true
 
       - name: Bump version
         run: |
-          git config user.name "toabot"
-          git config user.email "127577558+toabot@users.noreply.github.com"
-          
           if [ ${{ github.base_ref }} = main ]; then
-            npx lerna@6.5.1 version --conventional-graduate --no-commit-hooks --no-changelog --yes --message "chore(release): %s"
+            npx lerna@6.5.1 version --conventional-graduate --no-commit-hooks --no-changelog --yes --message "ci: %s" --sign-git-commit --sign-git-tag
             git fetch
             git checkout dev
             git merge main
             git push --all origin --no-verify
           else
-            npx lerna@6.5.1 version --conventional-prerelease --preid dev --no-commit-hooks --no-changelog --yes --message "chore(prerelease): %s" --git-tag-command "echo %s"
+            npx lerna@6.5.1 version --conventional-prerelease --preid dev --no-commit-hooks --no-changelog --yes --message "ci: %s" --git-tag-command "echo %s" --sign-git-commit
           fi
+        env:
+          GITHUB_TOKEN: ${{ secrets.TOABOT_GITHUB_TOKEN }}
+          GIT_AUTHOR_NAME: ${{ steps.gpg.outputs.name }}
+          GIT_AUTHOR_EMAIL: ${{ steps.gpg.outputs.email }}
+          GIT_COMMITTER_NAME: ${{ steps.gpg.outputs.name }}
+          GIT_COMMITTER_EMAIL: ${{ steps.gpg.outputs.email }}
 
       - name: Publish packages
         env:

package/docker-compose.yaml

@@ -1,8 +1,8 @@
 version: "3.9"
 services:
-  rabbitmq:
+  rabbitmq-0:
     image: rabbitmq:3.10.0-management
-    container_name: comq-rmq
+    container_name: comq-rmq-0
     ports:
       - '5673:5672'
       - '15673:15672'
@@ -18,3 +18,21 @@ services:
       resources:
         limits:
           memory: 512MB
+  rabbitmq-1:
+    image: rabbitmq:3.10.0-management
+    container_name: comq-rmq-1
+    ports:
+      - '5674:5672'
+      - '15674:15672'
+    environment:
+      RABBITMQ_DEFAULT_USER: developer
+      RABBITMQ_DEFAULT_PASS: secret
+    healthcheck:
+      test: rabbitmq-diagnostics -q ping
+      interval: 5s
+      timeout: 1s
+      retries: 5
+    deploy:
+      resources:
+        limits:
+          memory: 512MB

package/docs/notes.md

@@ -3,8 +3,8 @@
 ## "At least once"
 
 RabbitMQ delivers a message **before** it has been written to disk, therefore provides "at least
-once" delivery guarantee. Considering that, system using communications over AMQP
-must be idempotent.
+once" delivery guarantee. Considering that, system using communications over AMQP must be
+idempotent.
 
 ## Cause-Effect Confirmation Lag
 
@@ -36,12 +36,18 @@ A lightweight Producer (one that produces responses quickly) can result in being
 from the Consumer's perspective (that is, does not consume messages with an expected rate), even
 though the Producer being idle while waiting for response publication confirmations.
 
-## `replyTo` property prefix `direct:///`
+## Flow Control Lock
 
-`IO.request` accepts `replyTo` property modifier function `(replyTo): string` as fourth argument. It
-is the temporary solution to provide an option for compatibility with
-official [RabbitMQ client for .NET](https://github.com/rabbitmq/rabbitmq-dotnet-client/blob/feceedccd6b81d4b461795390432dc3330fdf596/projects/Unit/TestPublicationAddress.cs).
+When the [flow control mechanism](../readme.md#flow-control) is applied, an application that
+consumes requests and sends other request will hit the prefetch limit of incoming messages and
+become unresponsive until the issue is resolved.
 
-It is yet to be clarified and implemented correctly. [#6](https://github.com/toa-io/comq/issues/6)
+When an application becomes unresponsive, it can cause a chain reaction that makes all of its
+clients unresponsive, up to the system's *entry point* (which is typically the API Gateway).
+Therefore, the flow control of a distributed system is managed by its entry point, giving developers
+the choice to either stop accepting new requests (e.g., by replying with
+a [`429`](https://www.rfc-editor.org/rfc/rfc6585.html#section-4) HTTP status code for an API), or to
+continue sending new requests ignoring back pressure mechanism, risking a potential RabbitMQ crash
+due to running out of memory.
 
-See [Direct Reply-to](https://www.rabbitmq.com/direct-reply-to.html).
+> ComQ does not currently provide an option to ignore back pressure mechanism.

package/examples/const.js

@@ -1,3 +1 @@
-'use strict'
-
-exports.url = 'amqp://developer:secret@localhost:5673'
+export const url = 'amqp://developer:secret@localhost:5673'

package/examples/events/consumer.js

@@ -1,10 +1,9 @@
 'use strict'
 
-const { randomBytes } = require('node:crypto')
-const { connect } = require('comq')
-const { url } = require('../const')
+import { connect } from 'comq'
+import { url } from '../const.js'
 
-const group = process.argv[2] ?? randomBytes(4).toString('hex')
+const group = process.argv[2]
 
 let io
 
@@ -15,7 +14,7 @@ async function run () {
 
   await io.consume('random_numbers', group, consume)
 
-  console.log('Waiting for random numbers as ' + group)
+  console.log('Waiting for random numbers ' + (group ? 'as ' + group : 'exclusively'))
 
   process.on('SIGINT', exit)
 }
@@ -30,4 +29,4 @@ async function exit () {
   console.log('\nDisconnected')
 }
 
-run().then()
+await run()

package/examples/events/producer.js

@@ -1,7 +1,7 @@
 'use strict'
 
-const { connect } = require('comq')
-const { url } = require('../const')
+import { connect } from 'comq'
+import { url } from '../const.js'
 
 let io
 let interval
@@ -34,4 +34,4 @@ async function exit () {
   console.log('\nDisconnected')
 }
 
-run().then()
+await run()

package/examples/package.json

@@ -0,0 +1,3 @@
+{
+  "type": "module"
+}

package/examples/rpc/consumer.js

@@ -1,7 +1,7 @@
 'use strict'
 
-const { connect } = require('comq')
-const { url } = require('../const')
+import { connect } from 'comq'
+import { url } from '../const.js'
 
 let io
 let interval
@@ -37,4 +37,4 @@ async function exit () {
   console.log('\nDisconnected')
 }
 
-run().then()
+await run()

package/examples/rpc/producer.js

@@ -1,7 +1,7 @@
 'use strict'
 
-const { connect } = require('comq')
-const { url } = require('../const')
+import { connect } from 'comq'
+import { url } from '../const.js'
 
 let io
 
@@ -29,4 +29,4 @@ async function exit () {
   console.log('\nDisconnected')
 }
 
-run().then()
+await run()

package/features/connection.feature

@@ -1,5 +1,8 @@
 Feature: Connection Tolerance
 
+  Scenario: Connect
+    Given an active connection to the broker
+
   Scenario: Connecting to a broker before it was started
     Given the broker is down
     When I attempt to connect to the broker for 0.2 seconds

package/features/events.feature

@@ -1,4 +1,4 @@
-Feature: Events (pub/sub)
+Feature: Events
 
   Background:
     Given an active connection to the broker

package/features/shards.feature

@@ -0,0 +1,61 @@
+Feature: Sharded Connection
+
+  Scenario: Establishing sharded connection
+    Given an active sharded connection
+
+  Scenario: Connecting and waiting for 'open' event
+    Given the connection to both shards is established
+
+  Scenario: Connecting while a shard is down
+    Given one of the brokers has crashed
+    When I attempt to establish sharded connection
+    Then the connection is established
+    Then the broker is up
+
+  Scenario: Sending a request to a sharded connection while one broker is down
+    Given one of the brokers has crashed
+    And an active sharded connection
+    And a producer replying `add_numbers` queue
+    When the consumer sends a request to the `add_numbers` queue
+    Then the consumer receives the reply
+    Then the broker is up
+
+  Scenario: Publishing an event to a sharded connection missing one broker
+    Given the connection to both shards is established
+    And events are exclusively consumed from the `numbers_added` exchange
+    And one of the brokers has crashed
+    When an event is emitted to the `numbers_added` exchange
+    Then the event is received
+    Then the broker is up
+
+  Scenario: Shard crashes while sending requests
+    Given the connection to both shards is established
+    And a producer replying `flood` queue
+    And I'm sending 1kB requests to the `flood` queue at 1kHz
+    When one of the brokers has crashed
+    Then no exceptions are thrown
+    And all replies have been received
+    Then the broker is up
+
+  Scenario: Shard crashes while publishing events
+    Given the connection to both shards is established
+    And events are exclusively consumed from the `flood` exchange
+    And I'm publishing 1kB events to the `flood` exchange at 100Hz
+    When one of the brokers has crashed
+    Then no exceptions are thrown
+    And all events have been received
+    Then the broker is up
+
+  Scenario: Sealing sharded connection while one broker is down
+    Given the connection to both shards is established
+    And a producer replying `echo` queue
+    And one of the brokers has crashed
+    And the connection has started sealing
+    When the consumer sends a request to the `echo` queue
+    And the broker is up
+    Then the connection is sealed
+    And the consumer does not receive the reply
+
+  Scenario: Connecting with wrong credentials
+    When I attempt to establish a sharded connection as "someone" with password "whatever"
+    Then an exception is thrown: "Handshake terminated by server: 403 (ACCESS-REFUSED)"

package/features/steps/.test/backpressure.test.js

@@ -30,7 +30,7 @@ describe('When I\'m sending {quantity}B requests to the {token} queue at {quanti
   beforeAll(async () => {
     jest.clearAllMocks()
 
-    context = /** @type {comq.features.Context} */ { io }
+    context = /** @type {comq.features.Context} */ { io, requestCount: 0, replyCount: 0 }
 
     await step.call(context, bytesQ, queue, frequencyQ, seconds)
   })
@@ -62,7 +62,7 @@ describe('Then back pressure was applied', () => {
   const step = tomato.steps.Th('back pressure was applied')
 
   beforeEach(() => {
-    context = /** @type {comq.features.Context} */ { io }
+    context = /** @type {comq.features.Context} */ { io, events: {} }
   })
 
   it('should be', async () => undefined)

package/features/steps/.test/broker.test.js

@@ -12,19 +12,26 @@ jest.setTimeout(10000)
 
 require('../broker')
 
-const context = {}
+const context = /** @type {comq.features.Context} */ {}
 
 beforeEach(() => {
   jest.clearAllMocks()
 })
 
-describe('the broker is/has {status}', () => {
+describe.each([false, true])('Given the broker is/has {status} #shards:%s', (sharded) => {
   const step = tomato.steps.Gi('the broker is/has {status}')
 
+  const containerNumber = sharded ? 1 : 0
+
   it('should be', async () => undefined)
 
   beforeEach(() => {
     context.io = generate()
+
+    if (sharded) {
+      context.sharded = true
+      context.shard = containerNumber
+    }
   })
 
   it('should start comq-rmq container', async () => {
@@ -33,7 +40,7 @@ describe('the broker is/has {status}', () => {
 
     await step.call(context, 'up')
 
-    expect(command.execute).toHaveBeenCalledWith('docker start comq-rmq')
+    expect(command.execute).toHaveBeenCalledWith('docker start comq-rmq-' + containerNumber)
   })
 
   it('should wait for healthy state', async () => {
@@ -62,12 +69,31 @@ describe('the broker is/has {status}', () => {
   it('should stop comq-rmq container', async () => {
     await step.call(context, 'down')
 
-    expect(command.execute).toHaveBeenCalledWith('docker stop comq-rmq')
+    expect(command.execute).toHaveBeenCalledWith('docker stop comq-rmq-' + containerNumber)
   })
 
-  it('should kill comq-rmq container', async () => {
+  it('should kill comq-rmq-0 container', async () => {
     await step.call(context, 'crashed')
 
-    expect(command.execute).toHaveBeenCalledWith('docker kill comq-rmq')
+    expect(command.execute).toHaveBeenCalledWith('docker kill comq-rmq-' + containerNumber)
+  })
+})
+
+describe('Given one of the brokers is/has {status}', () => {
+  const step = tomato.steps.Gi('one of the brokers is/has {status}')
+
+  it('should store picked broker', async () => {
+    await step.call(context, 'down')
+
+    expect(context.shard).toStrictEqual(expect.any(Number))
+  })
+
+  it('should start one of the containers', async () => {
+    command.execute.mockImplementationOnce(async () => undefined)
+    command.execute.mockImplementationOnce(async () => ({ output: 'healthy' }))
+
+    await step.call(context, 'up')
+
+    expect(command.execute).toHaveBeenCalledWith('docker start comq-rmq-' + context.shard)
   })
 })

package/features/steps/.test/connection.test.js

@@ -13,7 +13,7 @@ jest.mock('comq', () => mock.comq)
 
 require('../connection')
 
-/** @type {comq.features.Context} */
+/** @type {jest.MockedObject<comq.features.Context>} */
 let context
 
 beforeEach(() => {
@@ -34,6 +34,38 @@ describe('Given an active connection to the broker', () => {
   it('should connect', async () => {
     expect(context.connect).toHaveBeenCalled()
   })
+
+  it('should store promise', async () => {
+    expect(context.connecting).toBeInstanceOf(Promise)
+  })
+})
+
+describe('Given an active sharded connection', () => {
+  const step = tomato.steps.Gi('an active sharded connection')
+
+  it('should be', async () => undefined)
+
+  beforeEach(async () => {
+    await step.call(context)
+  })
+
+  it('should set sharded to true', async () => {
+    expect(context.sharded).toStrictEqual(true)
+  })
+
+  it('should connect', async () => {
+    expect(context.connect).toHaveBeenCalled()
+  })
+
+  it('should store promise', async () => {
+    expect(context.connecting).toStrictEqual(expect.any(Promise))
+  })
+})
+
+describe('Given the connection to both shards is established', () => {
+  tomato.steps.Gi('the connection to both shards is established')
+
+  it('should be', async () => undefined)
 })
 
 describe('When I attempt to connect to the broker for {number} second(s)', () => {
@@ -125,6 +157,40 @@ describe('When I attempt to connect to the broker as {string} with password {str
   })
 })
 
+describe('When I attempt to establish sharded connection', () => {
+  const step = tomato.steps.Wh('I attempt to establish sharded connection')
+
+  it('should be', async () => undefined)
+
+  it('should set context.sharded to true', async () => {
+    await step.call(context)
+
+    expect(context.sharded).toStrictEqual(true)
+  })
+
+  it('should connect', async () => {
+    await step.call(context)
+
+    expect(context.connect).toHaveBeenCalled()
+  })
+})
+
+describe('When I attempt to establish a sharded connection as {string} with password {string}', () => {
+  const step = tomato.steps.Wh('I attempt to establish a sharded connection as {string} with password {string}')
+
+  it('should be', async () => undefined)
+
+  const user = generate()
+  const password = generate()
+
+  it('should connect with credentials', async () => {
+    await step.call(context, user, password)
+
+    expect(context.sharded).toStrictEqual(true)
+    expect(context.connect).toHaveBeenCalledWith(user, password)
+  })
+})
+
 describe.each([['', true], [' not', false]])('Then the connection is%s established',
   (not, connected) => {
     const step = tomato.steps.Th(`the connection is${not} established`)
@@ -185,3 +251,41 @@ describe('Then an exception is thrown: {string}', () => {
     expect(() => step.call(context)).toThrow(AssertionError)
   })
 })
+
+describe('Given the connection has started sealing', () => {
+  const step = tomato.steps.Gi('the connection has started sealing')
+
+  it('should be', async () => undefined)
+
+  it('should seal the connection', async () => {
+    context.io = /** @type {jest.MockedObject<comq.IO>} */ {
+      seal: jest.fn(async () => undefined)
+    }
+
+    await step.call(context)
+
+    expect(context.io.seal).toHaveBeenCalled()
+    expect(context.sealing).toBeInstanceOf(Promise)
+  })
+})
+
+describe('Then the connection is sealed', () => {
+  const step = tomato.steps.Th('the connection is sealed')
+
+  it('should be', async () => undefined)
+
+  it('should await for sealing', async () => {
+    context.sealing = promex()
+
+    let sealed = false
+
+    setImmediate(() => {
+      context.sealing.resolve()
+      sealed = true
+    })
+
+    await step.call(context)
+
+    expect(sealed).toStrictEqual(true)
+  })
+})

package/features/steps/.test/context.mock.js

@@ -1,8 +1,8 @@
 'use strict'
 
-const { generate } = require('randomstring')
+const { comq } = require('./comq.mock')
 
-const connect = jest.fn(async () => generate())
+const connect = jest.fn(() => comq.connect())
 const disconnect = jest.fn(async () => undefined)
 
 /**

package/features/steps/.test/context.test.js

@@ -20,11 +20,9 @@ beforeAll(async () => {
 
   // noinspection JSValidateTypes
   context = new Context(_)
-
-  await context.connect()
 })
 
-describe('connect', () => {
+describe.each([false, true])('connect (shards: %s)', (sharded) => {
   it('should be', async () => {
     expect(context.connect).toBeDefined()
   })
@@ -33,11 +31,18 @@ describe('connect', () => {
   let io
 
   beforeEach(async () => {
+    context.sharded = sharded
+
+    await context.connect()
+
     io = await comq.connect.mock.results[0].value
   })
 
   it('should connect', async () => {
-    expect(comq.connect).toHaveBeenCalledWith('amqp://developer:secret@localhost:5673')
+    const shards = ['amqp://developer:secret@localhost:5673', 'amqp://developer:secret@localhost:5674']
+
+    if (sharded) expect(comq.connect).toHaveBeenCalledWith(...shards)
+    else expect(comq.connect).toHaveBeenCalledWith(shards[0])
   })
 
   it('should connect with credentials', async () => {
@@ -46,7 +51,10 @@ describe('connect', () => {
 
     await context.connect(user, password)
 
-    expect(comq.connect).toHaveBeenCalledWith(`amqp://${user}:${password}@localhost:5673`)
+    const shards = [`amqp://${user}:${password}@localhost:5673`, `amqp://${user}:${password}@localhost:5674`]
+
+    if (sharded) expect(comq.connect).toHaveBeenCalledWith(...shards)
+    else expect(comq.connect).toHaveBeenCalledWith(shards[0])
   })
 
   it.each(['open', 'close', 'flow', 'discard'])('should store %s event',