comq 0.24.1 → 0.25.1

package/.husky/commit-msg

@@ -0,0 +1,5 @@
+#!/usr/bin/env sh
+
+[ -n "$CI" ] && exit 0
+
+npx --no -- commitlint --edit "$1"

package/benchmarks/run.js

@@ -1,7 +1,7 @@
 // What one request costs the process that answers it. See readme.md.
 //
 // node benchmarks/run.js [--rates 1000,5000] [--concurrency 1,8] [--messages 20000]
-// [--seconds 6] [--rounds 2]
+// [--seconds 6] [--rounds 2] [--streams 8000]
 
 import { fork } from 'node:child_process'
 import { parseArgs } from 'node:util'
@@ -17,7 +17,8 @@ const { values } = parseArgs({
     concurrency: { type: 'string', default: '1,8,64,256' },
     messages: { type: 'string', default: '20000' },
     seconds: { type: 'string', default: '6' },
-    rounds: { type: 'string', default: '2' }
+    rounds: { type: 'string', default: '2' },
+    streams: { type: 'string', default: '' }
   }
 })
 
@@ -27,6 +28,7 @@ const concurrencies = numbers(values.concurrency)
 const messages = Number(values.messages)
 const seconds = Number(values.seconds)
 const rounds = Number(values.rounds)
+const streams = values.streams === '' ? [] : numbers(values.streams)
 
 const durations = []
 
@@ -45,14 +47,24 @@ const io = await connect(url)
 try {
   const open = []
   const closed = []
+  const streamed = []
 
   for (let round = 1; round <= rounds; round++) {
+    if (streams.length > 0) {
+      for (const length of streams) streamed.push(await stream(length, round))
+
+      continue
+    }
+
     for (const rate of rates) open.push(await atRate(rate, round))
     for (const concurrency of concurrencies) closed.push(await inFlight(concurrency, round))
   }
 
-  report('At a fixed rate', ['rate'], open)
-  report('With requests in flight', ['in flight'], closed)
+  if (streams.length > 0) reportStreams(streamed)
+  else {
+    report('At a fixed rate', ['rate'], open)
+    report('With requests in flight', ['in flight'], closed)
+  }
 } finally {
   await io.close()
 
@@ -105,6 +117,56 @@ async function load (work) {
   }
 }
 
+/**
+ * A reply stream of `length` values, read to its end: what it cost the peer that sent it and this
+ * process that read it, per value, and how long it took.
+ */
+async function stream (length, round) {
+  await readStream(Math.min(length, 1000))
+
+  const before = await io.request('comq.benchmark.meter', null)
+  const cpu = process.cpuUsage()
+  const started = process.hrtime.bigint()
+
+  await readStream(length)
+
+  const elapsed = Number(process.hrtime.bigint() - started) / 1e6
+  const consumer = process.cpuUsage(cpu)
+  const after = await io.request('comq.benchmark.meter', null)
+  const producer = after.cpu.user + after.cpu.system - before.cpu.user - before.cpu.system
+
+  return {
+    round,
+    of: length,
+    ms: round2(elapsed),
+    producer: round2(producer / length),
+    consumer: round2((consumer.user + consumer.system) / length),
+    writes: round3((after.write + after.writev - before.write - before.writev) / length)
+  }
+}
+
+async function readStream (length) {
+  const reply = await io.request('comq.benchmark.stream', { length })
+
+  let read = 0
+
+  // eslint-disable-next-line no-unused-vars
+  for await (const _ of reply) read++
+
+  if (read !== length) throw new Error(`The stream held ${read} values of ${length}`)
+}
+
+function reportStreams (rows) {
+  console.log('\n## A reply stream\n')
+  console.log('| values | round | ms | producer CPU µs | consumer CPU µs | writes |')
+  console.log('| ---: | ---: | ---: | ---: | ---: | ---: |')
+
+  for (const row of rows) {
+    console.log(`| ${row.of} | ${row.round} | ${row.ms} | ${row.producer} | ${row.consumer} | ` +
+      `${row.writes} |`)
+  }
+}
+
 async function send (rate, duration, record) {
   const started = process.hrtime.bigint()
   const pending = []

package/benchmarks/server.js

@@ -32,6 +32,15 @@ const url = process.env.COMQ_BENCHMARK_URL ?? 'amqp://developer:secret@localhost
 
 let answered = 0
 
+/** What a value of a reply stream carries: a record, as a set read whole is made of. */
+const VALUE = {
+  id: '4c4759e6f9c74da989d64511df42d6f4',
+  title: 'First pot',
+  rank: 7,
+  public: true,
+  tags: ['one', 'two', 'three']
+}
+
 const io = await connect(url)
 
 await io.reply('comq.benchmark.echo', (payload) => {
@@ -40,6 +49,11 @@ await io.reply('comq.benchmark.echo', (payload) => {
   return payload
 })
 
+// a reply stream of the values a caller asks for, each the size of an echo's payload
+await io.reply('comq.benchmark.stream', function * ({ length }) {
+  for (let i = 0; i < length; i++) yield { ...VALUE, i }
+})
+
 // read before and after a window, so what is reported is what that window cost
 await io.reply('comq.benchmark.meter', () => ({ answered, cpu: process.cpuUsage(), ...counters }))
 

package/commitlint.config.js

@@ -0,0 +1,6 @@
+'use strict'
+
+// the same convention semantic-release reads the history with (see .releaserc)
+module.exports = {
+  extends: ['@commitlint/config-conventional']
+}

package/features/rpc.streams.feature

@@ -21,6 +21,34 @@ Feature: Reply streams
       [0, 1, 2, 3, 4]
       """
 
+  Scenario: Requesting a stream of many values at once
+    Given function replying `get_numbers` queue:
+      """
+      function * ({ limit }) {
+        for (let i = 0; i < limit; i++) yield { i }
+      }
+      """
+    When the consumer requests a stream with the following request to the `get_numbers` queue:
+      """yaml
+      limit: 300
+      """
+    Then the consumer receives a stream of 300 values in order
+
+  Scenario: Requesting a stream of buffers
+    Given function replying `get_bytes` queue:
+      """
+      function * () {
+        yield Buffer.from('first')
+        yield Buffer.alloc(0)
+        yield Buffer.from('third')
+      }
+      """
+    When the consumer requests a stream with request to the `get_bytes` queue
+    Then the consumer receives the stream of buffers:
+      """yaml
+      [first, '', third]
+      """
+
   Scenario: Requesting streams concurrently
     Given a number generator with 10ms increasing delay replying `get_numbers` queue
     When the consumer0 requests a stream with request to the `get_numbers` queue

package/features/steps/retransmission.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const assert = require('node:assert')
-const { Promex } = require('promex')
+const { Promex } = require('../../source/promex')
 const { Given, When, Then, After } = require('@cucumber/cucumber')
 
 const { connect } = require('../../')

package/features/steps/rpc.js

@@ -291,7 +291,37 @@ Then('the consumer receives the stream:',
 
     for await (const reply of this.stream) replies.push(reply)
 
-    assert.equal(values.length, replies.length, `Stream length mismatch: expected ${values.length}, received ${replies.length}`)
+    assert.deepStrictEqual(replies, values)
+  })
+
+Then('the consumer receives a stream of {number} values in order',
+  /**
+   * @param {number} count
+   * @this {comq.features.Context}
+   */
+  async function (count) {
+    const replies = []
+
+    for await (const reply of this.stream) replies.push(reply)
+
+    assert.deepStrictEqual(replies, Array.from({ length: count }, (_, i) => ({ i })))
+  })
+
+Then('the consumer receives the stream of buffers:',
+  /**
+   * @param {string} yaml what each buffer holds, as text
+   * @this {comq.features.Context}
+   */
+  async function (yaml) {
+    const values = parse(yaml)
+    const replies = []
+
+    for await (const reply of this.stream) {
+      assert.ok(Buffer.isBuffer(reply), `Not a buffer: ${JSON.stringify(reply)}`)
+      replies.push(reply.toString())
+    }
+
+    assert.deepStrictEqual(replies, values)
   })
 
 Then('the consumer receives the stream',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.24.1",
+  "version": "0.25.1",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",
@@ -27,19 +27,22 @@
     "feat": "scripts/feat.js"
   },
   "devDependencies": {
+    "@commitlint/cli": "^21.2.3",
+    "@commitlint/config-conventional": "^21.2.3",
     "@cucumber/cucumber": "^13.2.1",
+    "@jest/fake-timers": "^30.5.2",
     "@semantic-release/commit-analyzer": "^13.0.1",
     "@semantic-release/git": "^11.0.1",
     "@semantic-release/github": "^12.0.2",
     "@semantic-release/npm": "^13.1.2",
     "@semantic-release/release-notes-generator": "^14.1.0",
     "@testcontainers/rabbitmq": "^12.1.0",
-    "@types/jest": "^30.0.0",
     "conventional-changelog-conventionalcommits": "^8.0.0",
+    "expect": "^30.5.2",
     "fast-glob": "^3.3.3",
     "fuzzysort": "4.0.2",
     "husky": "^9.1.7",
-    "jest": "^30.4.2",
+    "jest-mock": "^30.5.2",
     "js-yaml": "^5.4.1",
     "randomstring": "^1.3.1",
     "semantic-release": "^25.0.9",
@@ -48,28 +51,33 @@
     "testcontainers": "^12.0.4"
   },
   "dependencies": {
-    "@toa.io/amqplib": "^0.1.0",
-    "promex": "^0.1.1",
-    "reretry": "^0.1.0"
+    "@toa.io/amqplib": "^0.1.0"
   },
   "engines": {
     "node": ">=24.0.0"
   },
   "scripts": {
     "prepare": "husky",
-    "test": "standard && NODE_OPTIONS=--experimental-vm-modules jest",
+    "test": "standard && node --test --experimental-test-module-mocks --import ./test/globals.js 'test/**/*.test.js' 'source/**/*.test.js'",
     "features": "cucumber-js --fail-fast -t 'not @manual'",
     "features:fast": "cucumber-js --fail-fast -t 'not @manual and not @heavy'",
     "benchmark": "node benchmarks/run.js",
     "lint": "standard --fix --verbose | snazzy"
   },
   "standard": {
-    "env": [
-      "jest"
+    "globals": [
+      "afterAll",
+      "afterEach",
+      "beforeAll",
+      "beforeEach",
+      "describe",
+      "expect",
+      "it",
+      "jest",
+      "test"
     ]
   },
   "overrides": {
-    "test-exclude": "^8.0.0",
     "rimraf": "^6.1.3",
     "archiver-utils": {
       "glob": "^13.0.6"

package/readme.md

@@ -354,6 +354,12 @@ for await (const number of stream)
   console.log(number)
 ```
 
+Values a producer yields together travel together: what the iterator yields before the event loop
+turns is sent as one message, up to 128 values, and a value that has to be waited for is sent as
+it comes. Buffers travel in messages of their own. The stream returned by `IO.request` yields each
+value on its own either way. Values in an encoding other than `application/json` travel one to a
+message.
+
 ### Stream topology
 
 <picture>

package/source/.io/ReplyPipe.js

@@ -1,8 +1,8 @@
 'use strict'
 
 const { EventEmitter } = require('node:events')
-const { Promex } = require('promex')
-const { control, FLOW_HEADER, HEARTBEAT_INTERVAL } = require('./const')
+const { Promex } = require('../promex')
+const { control, batch, FLOW_HEADER, HEARTBEAT_INTERVAL } = require('./const')
 
 /** @typedef {(message: any, properties?: comq.amqp.options.Publish) => Promise<void>} Reply */
 
@@ -36,6 +36,9 @@ class ReplyPipe extends EventEmitter {
   /** @type {Reply} */
   #reply
 
+  /** Whether the encoding can carry a list of values. */
+  #lists
+
   /**
    * @param {comq.amqp.Message} request
    * @param {stream.Readable} stream
@@ -54,8 +57,12 @@ class ReplyPipe extends EventEmitter {
     this.#reply = reply
     this.#replyTo = replyTo
 
+    this.#lists = request.properties.contentType === LISTS
+
     this.#properties = {
       chunk: { correlationId, ...CHUNK },
+      batch: { correlationId, ...CHUNK, type: batch.values },
+      buffers: { correlationId, ...CHUNK, type: batch.buffers },
       control: { correlationId, replyTo: feedback.queue, ...CONTROL },
       ok: { correlationId, replyTo: feedback.queue, ...CONTROL, headers: FLOW }
     }
@@ -99,21 +106,90 @@ class ReplyPipe extends EventEmitter {
    */
   async #pump () {
     try {
-      for await (const chunk of this.#stream) {
-        if (this.#gate !== null) await this.#gate
-        if (this.#closed) break
+      await this.#pumpBatches()
+    } catch {
+      // the source has been destroyed, by this pipe or by whoever made it
+    }
+
+    this.#close()
+  }
+
+  /**
+   * Values go out together while they arrive together: what the source yields before the event
+   * loop turns joins the message, and the first value that has to be waited for leaves with the
+   * ones before it rather than waiting for company. A cursor yielding the documents of a batch it
+   * holds sends them as one message; a source yielding one value at a time sends each as it
+   * comes. Values and buffers travel in messages of their own, and values in an encoding that
+   * holds no list travel one to a message.
+   */
+  async #pumpBatches () {
+    const iterator = this.#stream[Symbol.asyncIterator]()
+
+    let pending = next(iterator)
+
+    while (true) {
+      const first = await pending
 
-        await this.#transmit(chunk, this.#properties.chunk)
+      if (first.done === true) return
 
-        if (this.#closed) break
+      const chunks = [first.value]
+      const buffers = Buffer.isBuffer(first.value)
+      const turn = immediate()
 
-        this.#heartbeat()
+      let done = false
+
+      pending = next(iterator)
+
+      while (chunks.length < (buffers || this.#lists ? MAX_BATCH : 1)) {
+        const arrived = await Promise.race([pending, turn])
+
+        if (arrived === TURN) break
+
+        if (arrived.done === true) {
+          done = true
+
+          break
+        }
+
+        // of the other kind, it starts the next message
+        if (Buffer.isBuffer(arrived.value) !== buffers) {
+          pending = Promise.resolve(arrived)
+
+          break
+        }
+
+        chunks.push(arrived.value)
+        pending = next(iterator)
       }
-    } catch {
-      // the source has been destroyed, by this pipe or by whoever made it
+
+      if (!(await this.#send(...this.#message(chunks, buffers)))) {
+        await iterator.return?.()
+
+        return
+      }
+
+      if (done) return
     }
+  }
 
-    this.#close()
+  #message (chunks, buffers) {
+    if (chunks.length === 1) return [chunks[0], this.#properties.chunk]
+    else if (buffers) return [prefixed(chunks), this.#properties.buffers]
+    else return [chunks, this.#properties.batch]
+  }
+
+  /** Answers whether the pipe goes on. */
+  async #send (data, properties) {
+    if (this.#gate !== null) await this.#gate
+    if (this.#closed) return false
+
+    await this.#transmit(data, properties)
+
+    if (this.#closed) return false
+
+    this.#heartbeat()
+
+    return true
   }
 
   async #transmit (data, properties) {
@@ -206,6 +282,49 @@ class ReplyPipe extends EventEmitter {
 /** @type {comq.amqp.options.Publish} */
 const CHUNK = { mandatory: true }
 
+/** the most values a message carries */
+const MAX_BATCH = 128
+
+/** the encoding a batch of values is a list in */
+const LISTS = 'application/json'
+
+const TURN = Symbol('turn')
+
+/**
+ * The next value, asked for ahead of when it is awaited. A source that fails while a message is
+ * being sent is found failed by the loop that awaits it next, or by nobody where the pipe has
+ * stopped — and a rejection nobody awaits is an unhandled one.
+ */
+function next (iterator) {
+  const pending = iterator.next()
+
+  pending.catch(noop)
+
+  return pending
+}
+
+/** Settles once the event loop has turned, with what says so. */
+function immediate () {
+  return new Promise((resolve) => setImmediate(resolve, TURN))
+}
+
+/**
+ * @param {Buffer[]} buffers
+ * @returns {Buffer}
+ */
+function prefixed (buffers) {
+  const parts = []
+
+  for (const buffer of buffers) {
+    const length = Buffer.allocUnsafe(4)
+
+    length.writeUInt32BE(buffer.length)
+    parts.push(length, buffer)
+  }
+
+  return Buffer.concat(parts)
+}
+
 /** @type {comq.amqp.options.Publish} */
 const CONTROL = { type: 'control', mandatory: true }
 

package/source/.io/ReplyStream.js

@@ -1,8 +1,8 @@
 'use strict'
 
 const { Readable } = require('node:stream')
-const { Promex } = require('promex')
-const { IDLE_INTERVAL, FLOW_HEADER, control } = require('./const')
+const { Promex } = require('../promex')
+const { IDLE_INTERVAL, FLOW_HEADER, batch, control } = require('./const')
 const { Interrupted } = require('../interrupted')
 
 class ReplyStream extends Readable {
@@ -133,10 +133,29 @@ class ReplyStream extends Readable {
 
     if (properties.type === 'control')
       this._control(payload, properties)
+    else if (properties.type === batch.values)
+      this._values(payload)
+    else if (properties.type === batch.buffers)
+      this._values(split(payload))
     else if (!this.push(payload))
       this._throttle()
   }
 
+  /**
+   * The values of one message, pushed whatever the consumer's room: they have arrived, and a
+   * pause asked for now holds back the messages after them.
+   *
+   * @param {unknown[]} values
+   * @private
+   */
+  _values (values) {
+    let room = true
+
+    for (const value of values) room = this.push(value) && room
+
+    if (!room) this._throttle()
+  }
+
   /**
    * Values arriving out of order are held until the gap is filled. The hold is
    * bounded by their number and, roughly, by their size: the size is that of
@@ -213,6 +232,26 @@ class ReplyStream extends Readable {
   }
 }
 
+/**
+ * The buffers of a message that carries each prefixed with its length.
+ *
+ * @param {Buffer} buffer
+ * @returns {Buffer[]}
+ */
+function split (buffer) {
+  const buffers = []
+
+  for (let offset = 0; offset < buffer.length;) {
+    const length = buffer.readUInt32BE(offset)
+    const start = offset + 4
+
+    buffers.push(buffer.subarray(start, start + length))
+    offset = start + length
+  }
+
+  return buffers
+}
+
 const MAX_BUFFER_SIZE = 1000
 
 /** @type {number} */

package/source/.io/const.js

@@ -14,5 +14,11 @@ exports.control = {
  */
 exports.FLOW_HEADER = 'x-flow'
 
+/** The type of a message carrying values: a list of them, or buffers prefixed with their length. */
+exports.batch = {
+  values: 'batch',
+  buffers: 'buffers'
+}
+
 exports.HEARTBEAT_INTERVAL = 5_000
 exports.IDLE_INTERVAL = 12_000

package/source/attributes/track.test.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const { generate } = require('randomstring')
-const { Promex } = require('promex')
+const { Promex } = require('../promex')
 
 const { track } = require('./track')
 

package/source/channel.js

@@ -1,7 +1,7 @@
 'use strict'
 
-const { Promex } = require('promex')
-const { retry } = require('reretry')
+const { Promex } = require('./promex')
+const { retry } = require('./retry')
 const { failsafe, lazy, recall } = require('./attributes')
 const { verdictOf, PARK } = require('./verdicts')
 const emitter = require('./emitter')
@@ -49,6 +49,9 @@ class Channel {
   /** @type {Set<Promex>} */
   #confirmations = new Set()
 
+  /** @type {Set<Promise<void>>} failed messages on their way to a retry or the parking queue */
+  #settling = new Set()
+
   #diagnostics = emitter.create()
 
   #closed = false
@@ -221,6 +224,10 @@ class Channel {
 
     await this.seal()
 
+    // a failed message is moved in two steps, its copy placed and then the original
+    // released; a channel closed between them leaves the broker holding both
+    await Promise.allSettled(this.#settling)
+
     // a channel that went down with its connection is the outcome this asks for
     await this.#channel?.close().catch(noop)
 
@@ -587,7 +594,13 @@ class Channel {
       } catch (exception) {
         if (exception?.message === 'Channel closed') { return } // the message is requeued by the broker
 
-        await this.#failed(queue, message, exception)
+        const settling = this.#failed(queue, message, exception)
+
+        this.#settling.add(settling)
+
+        await settling
+
+        this.#settling.delete(settling)
       }
     }
 

package/source/connection.js

@@ -2,8 +2,8 @@
 
 const { setTimeout: delay } = require('node:timers/promises')
 const amqp = require('@toa.io/amqplib')
-const { Promex } = require('promex')
-const { retry } = require('reretry')
+const { Promex } = require('./promex')
+const { retry } = require('./retry')
 
 const { failsafe } = require('./attributes')
 const presets = require('./topology')

package/source/io.js

@@ -2,7 +2,7 @@
 
 const stream = require('node:stream')
 const { setTimeout } = require('node:timers/promises')
-const { Promex } = require('promex')
+const { Promex } = require('./promex')
 const { memo, failsafe, lazy, track } = require('./attributes')
 const { verdictOf, PARK } = require('./verdicts')
 const { Unroutable } = require('./unroutable')