comq 0.8.0 → 0.10.0

package/.github/dependabot.yml

@@ -4,13 +4,13 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
-    target-branch: "bumps"
+    target-branch: "dev"
     labels:
       - "Bump"
   - package-ecosystem: "npm"
     directory: "/"
     schedule:
       interval: "daily"
-    target-branch: "bumps"
+    target-branch: "dev"
     labels:
       - "Bump"

package/.github/workflows/codeql.yml

@@ -42,7 +42,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

package/.github/workflows/release.yaml

@@ -4,8 +4,8 @@ on:
   pull_request:
     types: [closed]
     branches:
-      - main
-      - dev
+      - release
+      - alpha
     paths:
       - 'source/**'
       - 'types/**'
@@ -27,16 +27,16 @@ jobs:
           registry-url: 'https://registry.npmjs.org'
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-#      - name: Run tests
-#        run: npm i && npm test
+      - name: Run tests
+        run: npm i && npm test
 
       - name: Import GPG key
         id: gpg
-        uses: crazy-max/ghaction-import-gpg@v5
+        uses: crazy-max/ghaction-import-gpg@v6
         with:
           gpg_private_key: ${{ secrets.TOABOT_GPG_PRIVATE_KEY }}
           passphrase: ${{ secrets.TOABOT_GPG_PASSPHRASE }}
@@ -45,15 +45,16 @@ jobs:
 
       - name: Bump version
         run: |
-          if [ ${{ github.base_ref }} = main ]; then
-            npx lerna@6.5.1 version --conventional-graduate --no-commit-hooks --no-changelog --yes --message "ci: %s" --sign-git-commit --sign-git-tag
-            git fetch
-            git checkout dev
-            git merge main
-            git push --all origin --no-verify
+          if [ ${{ github.base_ref }} = release ]; then
+            npx lerna@7.3.0 version --conventional-graduate --no-commit-hooks --no-changelog --yes --message "ci: %s" --sign-git-commit --sign-git-tag
           else
-            npx lerna@6.5.1 version --conventional-prerelease --preid dev --dist-tag dev --no-commit-hooks --no-changelog --yes --message "ci: %s" --git-tag-command "echo %s" --sign-git-commit
+            npx lerna@7.3.0 version --conventional-prerelease --preid alpha --dist-tag alpha --no-commit-hooks --no-changelog --yes --message "ci: %s" --git-tag-command "echo %s" --sign-git-commit
           fi
+          
+          git fetch
+          git checkout dev
+          git merge ${{ github.base_ref }}
+          git push --all origin --no-verify
         env:
           GITHUB_TOKEN: ${{ secrets.TOABOT_GITHUB_TOKEN }}
           GIT_AUTHOR_NAME: ${{ steps.gpg.outputs.name }}
@@ -65,8 +66,8 @@ jobs:
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
         run: |
-          if [ ${{ github.base_ref }} = main ]; then
+          if [ ${{ github.base_ref }} = release ]; then
             npm publish
           else
-            npm publish --tag dev
+            npm publish --tag alpha
           fi

package/examples/readme.md

@@ -1,19 +1,19 @@
 # Examples
 
-To run examples please start RabbitMQ server with `docker compose up -d` in the package root.
+To run the examples, please start RabbitMQ server with `docker compose up -d` in the package root.
 
 ## RPC
 
 Run in two terminals:
 
 ```shell
-$ node exampels/rpc/producer
+$ node examples/rpc/producer
 ```
 
 [source](rpc/producer.js)
 
 ```shell
-$ node exampels/rpc/consumer
+$ node examples/rpc/consumer
 ```
 
 [source](rpc/consumer.js)
@@ -41,3 +41,19 @@ $ node examples/events/producer
 `A` and `B` are consumer groups.
 
 > Try to run multiple instances with the same consumer group.
+
+## Reply streams
+
+Run in two terminals:
+
+```shell
+$ node examples/streams/producer
+```
+
+[source](streams/producer.js)
+
+```shell
+$ node examples/streams/consumer
+```
+
+[source](streams/consumer.js)

package/examples/streams/consumer.js

@@ -0,0 +1,36 @@
+import { connect } from 'comq'
+import { url } from '../const.js'
+
+const limit = Number(process.argv[2]) || Math.round(Math.random() * 8 + 2)
+
+let io
+let stream
+
+async function run () {
+  io = await connect(url)
+
+  console.log('Connected')
+  process.on('SIGINT', exit)
+
+  await fetch()
+  await io.close()
+
+  console.log('Disconnected')
+}
+
+async function fetch () {
+  console.log(`Fetching stream with ${limit} numbers`)
+
+  stream = await io.fetch('get_numbers', { limit })
+
+  for await (const number of stream) console.log('Received:', number)
+
+  console.log('Stream finished')
+}
+
+async function exit () {
+  console.log()
+  await io.close()
+}
+
+await run()

package/examples/streams/producer.js

@@ -0,0 +1,44 @@
+import { connect } from 'comq'
+import { url } from '../const.js'
+
+let io
+
+async function run () {
+  io = await connect(url)
+
+  console.log('Connected')
+
+  await io.reply('get_numbers', produce)
+
+  console.log('Waiting for requests...')
+
+  process.on('SIGINT', exit)
+}
+
+async function * produce ({ limit }) {
+  console.log('Request received with limit:', limit)
+
+  for (let i = 0; i < limit; i++) {
+    yield i
+
+    console.log('Yielded:', i)
+
+    await timeout()
+  }
+
+  console.log('Stream ended')
+}
+
+async function timeout () {
+  const ms = Math.round(Math.random() * 500)
+
+  await new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+async function exit () {
+  await io.close()
+
+  console.log('\nDisconnected')
+}
+
+await run()

package/features/backpressure.feature

@@ -6,7 +6,7 @@ Feature: Back pressure handling
   Scenario: Flooding a queue
     Given function replying `flood` queue:
     """
-    () => { return new Buffer.from('ok') }
+    () => { return Buffer.from('ok') }
     """
-    When I'm sending 20kB requests to the `flood` queue at 5kHz for 0.5 seconds
+    When I'm sending 100kB requests to the `flood` queue at 5kHz for 0.5 seconds
     Then back pressure was applied

package/features/events.feature

@@ -19,3 +19,8 @@ Feature: Events
     Given that events are exclusively consumed from the `numbers_added` exchange
     When an event is emitted to the `numbers_added` exchange
     Then the event is received
+
+  Scenario: Streaming requests
+    Given that events are exclusively consumed from the `numbers_added` exchange
+    When a stream of 1k events is emitted to the `numbers_added` exchange
+    Then 1k events is received

package/features/readme.md

@@ -1,4 +1,4 @@
 # Features
 
-To run features please start RabbitMQ server with `docker compose up -d` in the package root, then
-execute `npm run test:features`.
+To run features, start the RabbitMQ server by running `docker compose up -d` in the package root,
+then execute `npm run features`.

package/features/recovery.feature

@@ -98,9 +98,6 @@ Feature: Request-Reply Topology Recovery
     And `someone` receives the event
 
   Scenario: Start consuming events while broker is down
-
-  First consume will assert queue, so error will happen while calling AMQP.consume()
-
     Given that `first` is consuming events from the `numbers_added` exchange
     And `second` consuming events from the `numbers_added` exchange is expected
     When the broker has crashed

package/features/rpc.feature

@@ -5,18 +5,18 @@ Feature: Request-reply (RPC)
 
   Scenario: Sending request and getting reply
     Given function replying `add_numbers` queue:
-    """
-    ({ a, b }) => { return a + b }
-    """
+      """
+      ({ a, b }) => { return a + b }
+      """
     When the consumer sends the following request to the `add_numbers` queue:
-    """yaml
-    a: 1
-    b: 2
-    """
+      """yaml
+      a: 1
+      b: 2
+      """
     Then the consumer receives the reply:
-    """yaml
-    3
-    """
+      """yaml
+      3
+      """
 
   Scenario: Request causing an exception is discarded
     Given function replying `throw_error` queue:
@@ -26,3 +26,8 @@ Feature: Request-reply (RPC)
     When the consumer sends a request to the `throw_error` queue
     Then the message is discarded
     And the consumer does not receive the reply
+
+  Scenario: Streaming requests
+    Given a producer replying `ping` queue
+    When the consumer sends 1k requests to the `ping` queue as a stream
+    Then the consumer receives 1k replies as a stream

package/features/rpc.streams.feature

@@ -0,0 +1,81 @@
+Feature: Reply streams
+
+  Background:
+    Given an active connection to the broker
+    And heartbeat interval is set to 100ms
+    And idle timeout is set to 150ms
+
+  Scenario: Fetching a stream
+    Given function replying `get_numbers` queue:
+      """
+      function * ({ limit }) {
+        for (let i = 0; i < limit; i++) yield i
+      }
+      """
+    When the consumer fetches a stream with the following request to the `get_numbers` queue:
+      """yaml
+      limit: 5
+      """
+    Then the consumer receives the stream:
+      """yaml
+      [0, 1, 2, 3, 4]
+      """
+
+  Scenario: Fetching streams concurrently
+    Given a number generator with 10ms increasing delay replying `get_numbers` queue
+    When the consumer0 fetches a stream with request to the `get_numbers` queue
+    And the consumer0 receives the stream
+    And the consumer1 fetches a stream with request to the `get_numbers` queue
+    And the consumer1 receives the stream
+    And after 300ms
+    Then the consumer0 has received the stream:
+      """yaml
+      [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+      """
+    And the consumer1 has received the stream:
+      """yaml
+      [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+      """
+
+  Scenario: Interrupting a stream
+    Given a number generator with 20ms increasing delay replying `get_numbers` queue
+    When the consumer fetches a stream with request to the `get_numbers` queue
+    And the consumer interrupts the stream after 3 replies
+    # feedback message must be delivered
+    And after 10ms
+    Then the generator is destroyed
+
+  Scenario: Reply stream idle timeout
+    Given heartbeat interval is set to 300ms
+    And a number generator with 100ms increasing delay replying `get_numbers` queue
+    When the consumer fetches a stream with request to the `get_numbers` queue
+    Then the consumer receives the stream:
+      """yaml
+      [0, 1, 3]
+      """
+    And after 500ms
+    Then the generator is destroyed
+
+  Scenario: Reply stream heartbeat
+    # delay will exceed the idle timeout on the 3rd reply
+    Given a number generator with 90ms increasing delay replying `get_numbers` queue
+    When the consumer fetches a stream with request to the `get_numbers` queue
+    Then the consumer interrupts the stream after 5 replies
+    And after 200ms
+    Then the generator is destroyed
+
+  Scenario: Broker crashes while streaming reply
+    Given a number generator with 90ms increasing delay replying `get_numbers` queue
+    When the consumer fetches a stream with request to the `get_numbers` queue
+    Then the consumer receives the stream
+    And after 300ms
+    Then the broker has crashed
+    # idle timeout
+    Then after 150ms
+    Then the consumer has received the stream:
+      """yaml
+      [0, 1, 2]
+      """
+    Then after 200ms
+    Then the broker is up
+    And the generator is destroyed

package/features/rpc.streams.shards.feature

@@ -0,0 +1,37 @@
+Feature: Reply streams over shards
+
+  Background:
+    Given an active sharded connection
+    And heartbeat interval is set to 100ms
+    And idle timeout is set to 150ms
+
+  Scenario: Fetching a stream using sharded connection
+    Given a generator replying `get_numbers` queue:
+      """
+      function * ({ amount }) {
+        for (let i = 0; i < amount; i++) yield i
+      }
+      """
+    When the consumer fetches a stream with the following request to the `get_numbers` queue:
+      """yaml
+      amount: 5
+      """
+    Then the consumer receives the stream:
+      """yaml
+      [0, 1, 2, 3, 4]
+      """
+
+  Scenario: Broker crashes while fetching a stream
+    Given a number generator with 90ms increasing delay replying `get_numbers` queue
+    When the consumer fetches a stream with request to the `get_numbers` queue
+    And the consumer receives the stream
+    And after 300ms
+    And the broker has crashed
+    And the broker is up
+    Then the consumer interrupts the stream
+    And the consumer has received the stream containing:
+      """yaml
+      [0, 1, 2, 3, 4, 5, 6]
+      """
+    And after 1000ms
+    And the generator is destroyed

package/features/steps/.test/context.test.js

@@ -59,7 +59,7 @@ describe.each([false, true])('connect (shards: %s)', (sharded) => {
 
   it.each(['open', 'close', 'flow', 'discard'])('should store %s event',
     /**
-     * @param {comq.diagnostics.event} event
+     * @param {comq.diagnostics.Event} event
      */
     async (event) => {
       expect(io.diagnose).toHaveBeenCalledWith(event, expect.any(Function))

package/features/steps/.test/events.test.js

@@ -232,9 +232,9 @@ describe('Given I\'m publishing {quantity}B events to the {token} exchange at {q
 
     step.call(context, bytesQ, exchange, frequencyQ)
 
-    await timeout(100 + 12)
+    await timeout(100)
 
-    const callsAmountExpected = [10, 11]
+    const callsAmountExpected = [8, 9, 10, 11]
 
     expect(callsAmountExpected).toContain(context.io.emit.mock.calls.length)
     expect(context.publishing).toBeDefined()

package/features/steps/.test/rpc.test.js

@@ -221,7 +221,7 @@ describe('Given I\'m sending {quantity}B requests to the {token} queue at {quant
 
     await timeout(100 + 12)
 
-    const callsAmountExpected = [10, 11]
+    const callsAmountExpected = [9, 10, 11, 12]
 
     expect(callsAmountExpected).toContain(context.io.request.mock.calls.length)
     expect(context.sending).toBeDefined()

package/features/steps/.types/context.d.ts

@@ -1,3 +1,4 @@
+import { Readable } from 'node:stream'
 import * as _diagnostics from '../../../types/diagnostic'
 import * as _io from '../../../types/io'
 import * as _amqp from '../../../types/amqp'
@@ -15,7 +16,7 @@ declare namespace comq.features {
     eventsConsumedCount: number
     consumed?: Record<string, { payload: any, properties?: _amqp.Properties }>
     consumedCount: number
-    events?: { [K in _diagnostics.event]?: boolean }
+    events?: { [K in _diagnostics.Event]?: boolean }
     exception?: Error
     consumptionPromise?: Promise<any>
     sharded: boolean
@@ -23,6 +24,13 @@ declare namespace comq.features {
     sealing: Promise<any>
     sending: any
     publishing: any
+    stream: Readable
+    streamValues: any[]
+    streamEnded: boolean
+    streams: Record<number, Readable>
+    streamsValues: Record<number, any[]>
+    streamsEnded: Record<number, boolean>
+    generatorDestroyed: boolean
 
     connect(user?: string, password?: string): Promise<void>
     assert(user?: string, password?: string): Promise<void>

package/features/steps/common.js

@@ -0,0 +1,12 @@
+'use strict'
+
+const { When } = require('@cucumber/cucumber')
+const { timeout } = require('@toa.io/generic')
+
+When('after {number}ms',
+  /**
+   * @param {number} delay
+   */
+  async function (delay) {
+    await timeout(delay)
+  })

package/features/steps/context.js

@@ -22,6 +22,13 @@ class Context extends World {
   sharded
   shard
   sealing
+  stream
+  streamValues = []
+  streamEnded = false
+  streams = {}
+  streamsValues = {}
+  streamsEnded = {}
+  generatorDestroyed = false
 
   async connect (user, password) {
     const urls = this.#urls(user, password)
@@ -47,7 +54,7 @@ class Context extends World {
 
   /**
    * @param {string[]} urls
-   * @param {comq.connect} [method]
+   * @param {comq.Connect} [method]
    * @return {Promise<void>}
    */
   async #connect (urls, method = connect) {
@@ -87,7 +94,7 @@ const SHARDS = ['localhost:5673', 'localhost:5674']
 const USER = 'developer'
 const PASSWORD = 'secret'
 
-/** @type {comq.diagnostics.event[]} */
+/** @type {comq.diagnostics.Event[]} */
 const EVENTS = ['open', 'close', 'flow', 'discard']
 
 exports.Context = Context

package/features/steps/events.js

@@ -1,5 +1,6 @@
 'use strict'
 
+const stream = require('node:stream')
 const assert = require('node:assert')
 const { randomBytes } = require('node:crypto')
 const { timeout, quantity, match } = require('@toa.io/generic')
@@ -61,6 +62,23 @@ When('an event is emitted to the {token} exchange with properties:',
     await emit.call(this, exchange, message, properties)
   })
 
+When('a stream of {quantity} events is emitted to the {token} exchange',
+  /**
+   * @param {string} exchange
+   * @this {comq.features.Context}
+   */
+  async function (amountQ, exchange) {
+    const amount = quantity(amountQ)
+
+    function * generate () {
+      for (let i = 0; i < amount; i++) yield randomBytes(8)
+    }
+
+    const events = stream.Readable.from(generate())
+
+    await emit.call(this, exchange, events)
+  })
+
 Then('{token} receives the event',
   /**
    * @param {string} group
@@ -93,6 +111,17 @@ Then('the event is received',
     await consumed.call(this)
   })
 
+Then('{quantity} events is received',
+  /**
+   * @this {comq.features.Context}
+   */
+  async function (expectedQ) {
+    const expected = quantity(expectedQ)
+
+    await timeout(100) // let it consume
+    assert.equal(this.eventsConsumedCount, expected, 'Not all events have been consumed')
+  })
+
 Given('I\'m publishing {quantity}B events to the {token} exchange at {quantity}Hz',
   /**
    * @param {string} bytesQ