comq 0.24.0 → 0.25.0

package/.husky/commit-msg

@@ -0,0 +1,5 @@
+#!/usr/bin/env sh
+
+[ -n "$CI" ] && exit 0
+
+npx --no -- commitlint --edit "$1"

package/benchmarks/run.js

@@ -1,7 +1,7 @@
 // What one request costs the process that answers it. See readme.md.
 //
 // node benchmarks/run.js [--rates 1000,5000] [--concurrency 1,8] [--messages 20000]
-// [--seconds 6] [--rounds 2]
+// [--seconds 6] [--rounds 2] [--streams 8000]
 
 import { fork } from 'node:child_process'
 import { parseArgs } from 'node:util'
@@ -17,7 +17,8 @@ const { values } = parseArgs({
     concurrency: { type: 'string', default: '1,8,64,256' },
     messages: { type: 'string', default: '20000' },
     seconds: { type: 'string', default: '6' },
-    rounds: { type: 'string', default: '2' }
+    rounds: { type: 'string', default: '2' },
+    streams: { type: 'string', default: '' }
   }
 })
 
@@ -27,6 +28,7 @@ const concurrencies = numbers(values.concurrency)
 const messages = Number(values.messages)
 const seconds = Number(values.seconds)
 const rounds = Number(values.rounds)
+const streams = values.streams === '' ? [] : numbers(values.streams)
 
 const durations = []
 
@@ -45,14 +47,24 @@ const io = await connect(url)
 try {
   const open = []
   const closed = []
+  const streamed = []
 
   for (let round = 1; round <= rounds; round++) {
+    if (streams.length > 0) {
+      for (const length of streams) streamed.push(await stream(length, round))
+
+      continue
+    }
+
     for (const rate of rates) open.push(await atRate(rate, round))
     for (const concurrency of concurrencies) closed.push(await inFlight(concurrency, round))
   }
 
-  report('At a fixed rate', ['rate'], open)
-  report('With requests in flight', ['in flight'], closed)
+  if (streams.length > 0) reportStreams(streamed)
+  else {
+    report('At a fixed rate', ['rate'], open)
+    report('With requests in flight', ['in flight'], closed)
+  }
 } finally {
   await io.close()
 
@@ -105,6 +117,56 @@ async function load (work) {
   }
 }
 
+/**
+ * A reply stream of `length` values, read to its end: what it cost the peer that sent it and this
+ * process that read it, per value, and how long it took.
+ */
+async function stream (length, round) {
+  await readStream(Math.min(length, 1000))
+
+  const before = await io.request('comq.benchmark.meter', null)
+  const cpu = process.cpuUsage()
+  const started = process.hrtime.bigint()
+
+  await readStream(length)
+
+  const elapsed = Number(process.hrtime.bigint() - started) / 1e6
+  const consumer = process.cpuUsage(cpu)
+  const after = await io.request('comq.benchmark.meter', null)
+  const producer = after.cpu.user + after.cpu.system - before.cpu.user - before.cpu.system
+
+  return {
+    round,
+    of: length,
+    ms: round2(elapsed),
+    producer: round2(producer / length),
+    consumer: round2((consumer.user + consumer.system) / length),
+    writes: round3((after.write + after.writev - before.write - before.writev) / length)
+  }
+}
+
+async function readStream (length) {
+  const reply = await io.request('comq.benchmark.stream', { length })
+
+  let read = 0
+
+  // eslint-disable-next-line no-unused-vars
+  for await (const _ of reply) read++
+
+  if (read !== length) throw new Error(`The stream held ${read} values of ${length}`)
+}
+
+function reportStreams (rows) {
+  console.log('\n## A reply stream\n')
+  console.log('| values | round | ms | producer CPU µs | consumer CPU µs | writes |')
+  console.log('| ---: | ---: | ---: | ---: | ---: | ---: |')
+
+  for (const row of rows) {
+    console.log(`| ${row.of} | ${row.round} | ${row.ms} | ${row.producer} | ${row.consumer} | ` +
+      `${row.writes} |`)
+  }
+}
+
 async function send (rate, duration, record) {
   const started = process.hrtime.bigint()
   const pending = []

package/benchmarks/server.js

@@ -32,6 +32,15 @@ const url = process.env.COMQ_BENCHMARK_URL ?? 'amqp://developer:secret@localhost
 
 let answered = 0
 
+/** What a value of a reply stream carries: a record, as a set read whole is made of. */
+const VALUE = {
+  id: '4c4759e6f9c74da989d64511df42d6f4',
+  title: 'First pot',
+  rank: 7,
+  public: true,
+  tags: ['one', 'two', 'three']
+}
+
 const io = await connect(url)
 
 await io.reply('comq.benchmark.echo', (payload) => {
@@ -40,6 +49,11 @@ await io.reply('comq.benchmark.echo', (payload) => {
   return payload
 })
 
+// a reply stream of the values a caller asks for, each the size of an echo's payload
+await io.reply('comq.benchmark.stream', function * ({ length }) {
+  for (let i = 0; i < length; i++) yield { ...VALUE, i }
+})
+
 // read before and after a window, so what is reported is what that window cost
 await io.reply('comq.benchmark.meter', () => ({ answered, cpu: process.cpuUsage(), ...counters }))
 

package/commitlint.config.js

@@ -0,0 +1,6 @@
+'use strict'
+
+// the same convention semantic-release reads the history with (see .releaserc)
+module.exports = {
+  extends: ['@commitlint/config-conventional']
+}

package/features/heartbeat.feature

@@ -0,0 +1,16 @@
+@heavy
+Feature: Idle connection tolerance
+
+  A connection that says nothing but heartbeats is as healthy as a busy one, and is kept for as
+  long as the broker goes on answering them, however long that is.
+
+  Background:
+    Given 1s AMQP heartbeat
+
+  Scenario: Keeping an idle connection
+    Given an active connection to the broker
+    Then the connection is not lost for 20 seconds
+
+  Scenario: Keeping an idle sharded connection
+    Given an active sharded connection
+    Then the connection is not lost for 20 seconds

package/features/rpc.call.recovery.feature

@@ -17,14 +17,14 @@ Feature: Calls under a key across failures
     it the key. The holder finds the key taken until the broker lets that connection go, and
     claims it then.
 
-    Given watchdog interval is set to 2000ms with 60s AMQP heartbeat
+    Given 1s AMQP heartbeat
     And a network that can go silent
     And an active connection to the broker
     And a holder answering `echo` under the `a` key:
       """
       (payload) => payload
       """
-    When the network goes silent
+    When the network goes silent, with the broker still hearing heartbeats
     Then the connection is lost within 10 seconds
     And the holder finds its key taken while the broker holds the silent connection
     When the silent connection is let go

package/features/rpc.shards.retransmission.feature

@@ -0,0 +1,46 @@
+@heavy
+Feature: Re-sending the requests a lost shard held
+
+  A Request is re-sent when the shard it went through is lost, since its Reply may be lost with it.
+  One that went through a shard that stays is answered through that shard, and is never re-sent
+  because another one was lost: a Request is not assumed to be idempotent.
+
+  Each scenario first takes one broker away from the consumer, so that the Requests it sends go
+  through the other one, and gives the broker back before losing one of them again.
+
+  Background:
+    Given 1s AMQP heartbeat
+    And a network that can go silent
+    And an active sharded connection
+    And a producer replying `echo` queue
+    # the consumer's channels exist on both brokers before one of them is taken away
+    And the consumer sends a request to the `echo` queue
+    And the consumer receives the reply
+    And a producer on another connection counting the requests to the `numbered` queue, answering when told
+
+  Scenario: Requests through the broker that stays are not re-sent when the other one is lost
+    Given the consumer's connection to broker 1 is cut
+    When the consumer sends 10 numbered requests to the `numbered` queue
+    And the consumer's connection to broker 1 is restored
+    And the consumer's connection to broker 1 is cut
+    And the producer answers
+    Then every numbered request is answered within 10 seconds
+    And every numbered request has been executed once
+
+  Scenario: Replies to the requests through the broker that stays are not lost with the other one
+    Given the consumer's connection to broker 1 is cut
+    When the consumer sends 10 numbered requests to the `numbered` queue
+    And the consumer's connection to broker 1 is restored
+    And the consumer's network to broker 1 goes silent
+    And the producer answers
+    Then every numbered request is answered within 15 seconds
+    And every numbered request has been executed once
+
+  Scenario: Requests through the broker that is lost are re-sent and answered
+    Given the consumer's connection to broker 0 is cut
+    When the consumer sends 10 numbered requests to the `numbered` queue
+    And the consumer's connection to broker 0 is restored
+    And the consumer's network to broker 1 goes silent
+    And the producer answers
+    Then every numbered request is answered within 15 seconds
+    And every numbered request has been executed twice

package/features/rpc.streams.feature

@@ -21,6 +21,34 @@ Feature: Reply streams
       [0, 1, 2, 3, 4]
       """
 
+  Scenario: Requesting a stream of many values at once
+    Given function replying `get_numbers` queue:
+      """
+      function * ({ limit }) {
+        for (let i = 0; i < limit; i++) yield { i }
+      }
+      """
+    When the consumer requests a stream with the following request to the `get_numbers` queue:
+      """yaml
+      limit: 300
+      """
+    Then the consumer receives a stream of 300 values in order
+
+  Scenario: Requesting a stream of buffers
+    Given function replying `get_bytes` queue:
+      """
+      function * () {
+        yield Buffer.from('first')
+        yield Buffer.alloc(0)
+        yield Buffer.from('third')
+      }
+      """
+    When the consumer requests a stream with request to the `get_bytes` queue
+    Then the consumer receives the stream of buffers:
+      """yaml
+      [first, '', third]
+      """
+
   Scenario: Requesting streams concurrently
     Given a number generator with 10ms increasing delay replying `get_numbers` queue
     When the consumer0 requests a stream with request to the `get_numbers` queue

package/features/shards.feature

@@ -49,7 +49,7 @@ Feature: Sharded Connection
     Then the broker is up
 
   Scenario: Shard stops responding while requests are sent
-    Given watchdog interval is set to 3000ms with 1s AMQP heartbeat
+    Given 1s AMQP heartbeat
     And an active sharded connection
     And a producer replying `stalled` queue
     When the consumer sends 10 requests to the `stalled` queue as a broker freezes

package/features/silence.feature

@@ -2,10 +2,11 @@ Feature: Silent Connection Tolerance
 
   A connection that has gone silent is what a machine that woke from sleep is
   left with: the socket is open, neither the broker nor the operating system
-  reports anything, and nothing but the watchdog tells it from an idle connection.
+  reports anything, and nothing but the missing heartbeats tell it from an idle
+  connection.
 
   Background:
-    Given watchdog interval is set to 2000ms with 60s AMQP heartbeat
+    Given 1s AMQP heartbeat
     And a network that can go silent
 
   Scenario: Restoring a connection that has gone silent

package/features/steps/.types/context.d.ts

@@ -49,7 +49,11 @@ declare namespace comq.features {
     readonly address: string
 
     open(): Promise<void>
-    silence(): void
+    silence(heard?: boolean): void
+    refuse(): void
+    admit(): void
+    release(): void
+    cut(): void
     close(): Promise<void>
   }
 
@@ -57,6 +61,7 @@ declare namespace comq.features {
     client: Socket
     upstream: Socket
     silent: boolean
+    heartbeat?: NodeJS.Timeout
   }
 
 }

package/features/steps/brokers.js

@@ -139,7 +139,7 @@ const actions = {
     await docker('docker', ['kill', `comq-rmq-${n}`])
   },
   // a frozen broker keeps its connections open while answering nothing, which is
-  // what a publisher cannot tell from a working one until the watchdog fires
+  // what a publisher cannot tell from a working one until its heartbeats go missing
   frozen: async (n = 0) => {
     await docker('docker', ['pause', `comq-rmq-${n}`])
   }

package/features/steps/connection.js

@@ -5,6 +5,16 @@ const { timeout } = require('../../test/helpers')
 
 const { Given, When, Then } = require('@cucumber/cucumber')
 
+Given('{number}s AMQP heartbeat',
+  /**
+   * A connection that has received nothing for two heartbeats is taken for lost.
+   *
+   * @param {number} heartbeat
+   */
+  function (heartbeat) {
+    global.COMQ_TESTING_AMQP_HEARTBEAT = heartbeat
+  })
+
 Given('an active connection to the broker',
   /**
    * @this {comq.features.Context}
@@ -176,6 +186,19 @@ Then('the connection is lost within {number} second(s)',
     await emitted.call(this, 'close', seconds, 'connection was not lost')
   })
 
+Then('the connection is not lost for {number} seconds',
+  /**
+   * @param {number} seconds
+   * @this {comq.features.Context}
+   */
+  async function (seconds) {
+    delete this.events.close
+
+    await timeout(seconds * 1000)
+
+    assert.equal(this.events.close, undefined, 'The connection was lost')
+  })
+
 Then('publishing is {paused-event} within {number} second(s)',
   /**
    * @param {'paused' | 'resumed'} key

package/features/steps/context.js

@@ -114,8 +114,7 @@ class Context extends World {
     const url = PROTOCOL + user + ':' + password + '@' + address
     const query = []
 
-    // the watchdog measures silence, so it may only be shortened along with the
-    // interval at which a healthy broker is expected to say something
+    // a connection gone silent is told by its missing heartbeats, so a short one tells it soon
     if (global.COMQ_TESTING_AMQP_HEARTBEAT !== undefined) {
       query.push('heartbeat=' + global.COMQ_TESTING_AMQP_HEARTBEAT)
     }

package/features/steps/hooks.js

@@ -12,7 +12,6 @@ After(
    * @this {comq.features.Context}
    */
   async function () {
-    delete global.COMQ_TESTING_WATCHDOG_INTERVAL
     delete global.COMQ_TESTING_AMQP_HEARTBEAT
     delete global.COMQ_TESTING_AMQP_CHANNEL_MAX
 

package/features/steps/network.js

@@ -26,6 +26,14 @@ When('the network goes silent',
     silence.call(this)
   })
 
+When('the network goes silent, with the broker still hearing heartbeats',
+  /**
+   * @this {comq.features.Context}
+   */
+  function () {
+    silence.call(this, true)
+  })
+
 When('the network goes silent and refuses new connections',
   /**
    * @this {comq.features.Context}
@@ -66,11 +74,12 @@ When('the consumer sends a request to the {token} queue as the network goes sile
   })
 
 /**
+ * @param {boolean} [heard]
  * @this {comq.features.Context}
  */
-function silence () {
+function silence (heard) {
   // only a connection lost from now on counts as lost
   delete this.events.close
 
-  for (const network of this.networks) network.silence()
+  for (const network of this.networks) network.silence(heard)
 }

package/features/steps/networks.js

@@ -48,9 +48,19 @@ class Network {
   /**
    * Only the tunnels that are already open go silent: a connection established
    * afterwards is forwarded as usual, just as it is once a machine is awake.
+   *
+   * The broker notices a silent connection by its heartbeat as soon as the client does. One
+   * that is to be held by the broker for as long as the scenario needs is kept heard from:
+   * heartbeats are sent to the broker in the client's stead, until the network lets it go.
+   *
+   * @param {boolean} [heard] whether the broker goes on hearing heartbeats
    */
-  silence () {
-    for (const tunnel of this.#tunnels) tunnel.silent = true
+  silence (heard = false) {
+    for (const tunnel of this.#tunnels) {
+      tunnel.silent = true
+
+      if (heard) tunnel.heartbeat ??= setInterval(() => tunnel.upstream.write(HEARTBEAT), HEARTBEAT_MS)
+    }
   }
 
   /**
@@ -65,6 +75,17 @@ class Network {
     this.#refusing = false
   }
 
+  /**
+   * Takes the broker away from whoever is connected through this network: the tunnels that are
+   * open are closed at both ends, and a connection attempted from now on is refused until the
+   * network admits it.
+   */
+  cut () {
+    this.refuse()
+
+    for (const tunnel of this.#tunnels) this.#collapse(tunnel)
+  }
+
   /**
    * Closes the tunnels that went silent, at both ends, so the broker learns that the
    * connection it was still holding is gone — the moment its heartbeat would otherwise tell it.
@@ -117,10 +138,18 @@ class Network {
    * @param {comq.features.Tunnel} tunnel
    */
   #collapse (tunnel) {
+    clearInterval(tunnel.heartbeat)
+
     this.#tunnels.delete(tunnel)
     tunnel.client.destroy()
     tunnel.upstream.destroy()
   }
 }
 
+/** An AMQP heartbeat frame: type 8, channel 0, no payload, and the frame end. */
+const HEARTBEAT = Buffer.from([8, 0, 0, 0, 0, 0, 0, 0xCE])
+
+/** Often enough for the shortest heartbeat a scenario asks for. */
+const HEARTBEAT_MS = 250
+
 exports.Network = Network

package/features/steps/retransmission.js

@@ -0,0 +1,165 @@
+'use strict'
+
+const assert = require('node:assert')
+const { Promex } = require('../../source/promex')
+const { Given, When, Then, After } = require('@cucumber/cucumber')
+
+const { connect } = require('../../')
+const { getAddress, USER, PASSWORD, BROKERS_AMOUNT } = require('./brokers')
+const { timeout, until } = require('../../test/helpers')
+
+Given('a producer on another connection counting the requests to the {token} queue, answering when told',
+  /**
+   * Connected to every broker directly, so that nothing done to the consumer's network reaches it.
+   * It takes every Request as it comes, counts how many times it has been handed each of them, and
+   * holds the replies back until it is told to answer: whatever the consumer re-sends by then is
+   * counted, however quickly the first attempt would have been answered.
+   *
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  async function (queue) {
+    const urls = Array.from({ length: BROKERS_AMOUNT }, (_, n) =>
+      `amqp://${USER}:${PASSWORD}@${getAddress(n)}`)
+
+    this.producer = await connect(...urls)
+    this.executions = new Map()
+    this.answering = new Promex()
+
+    await this.producer.reply(queue, async ({ id }) => {
+      this.executions.set(id, (this.executions.get(id) ?? 0) + 1)
+
+      await this.answering
+
+      return id
+    })
+  })
+
+When('the producer answers',
+  /**
+   * @this {comq.features.Context}
+   */
+  function () {
+    this.answering.resolve()
+  })
+
+When('the consumer sends {number} numbered requests to the {token} queue',
+  /**
+   * @param {number} amount
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  function (amount, queue) {
+    this.numbered = Array.from({ length: amount }, (_, id) => {
+      const reply = this.io.request(queue, { id })
+
+      // asserted on by a later step, and an unhandled rejection would end the run before it
+      reply.catch(() => undefined)
+
+      return reply
+    })
+  })
+
+When('the consumer\'s connection to broker {number} is cut',
+  /**
+   * The broker is taken away from the consumer alone: the connection closes at both ends, and the
+   * consumer is refused until it is restored. Returns once the consumer has noticed, so that what
+   * it sends next goes to the other broker.
+   *
+   * @param {number} broker
+   * @this {comq.features.Context}
+   */
+  async function (broker) {
+    let lost = false
+
+    this.io.diagnose('close', (_error, index) => { if (index === broker) lost = true })
+    this.networks[broker].cut()
+
+    assert.equal(await until(() => lost, 10_000), true,
+      `The consumer's connection to broker ${broker} was not lost`)
+  })
+
+When('the consumer\'s connection to broker {number} is restored',
+  /**
+   * Returns once the consumer's channels to that broker have recovered, which is when the
+   * requests it may have lost with it are re-sent.
+   *
+   * @param {number} broker
+   * @this {comq.features.Context}
+   */
+  async function (broker) {
+    const recovered = new Set()
+
+    this.io.diagnose('recover', (type, index) => { if (index === broker) recovered.add(type) })
+    this.networks[broker].admit()
+
+    const restored = () => recovered.has('request') && recovered.has('reply')
+
+    assert.equal(await until(restored, 20_000), true,
+      `The consumer's channels to broker ${broker} have not recovered`)
+  })
+
+When('the consumer\'s network to broker {number} goes silent',
+  /**
+   * @param {number} broker
+   * @this {comq.features.Context}
+   */
+  function (broker) {
+    this.networks[broker].silence()
+  })
+
+Then('every numbered request is answered within {number} seconds',
+  /**
+   * @param {number} seconds
+   * @this {comq.features.Context}
+   */
+  async function (seconds) {
+    const answered = new Array(this.numbered.length).fill(false)
+
+    this.numbered.forEach((reply, id) => reply.then(() => { answered[id] = true }, () => undefined))
+
+    await until(() => answered.every(Boolean), seconds * 1000)
+
+    const unanswered = answered.flatMap((yes, id) => yes ? [] : [id])
+
+    assert.deepEqual(unanswered, [], `Requests ${unanswered.join(', ')} are not answered`)
+
+    for (const [id, reply] of this.numbered.entries()) assert.equal(await reply, id, 'Reply mismatch')
+  })
+
+Then('every numbered request has been executed {word}',
+  /**
+   * A re-sent Request is taken as soon as it arrives, so a moment is given for any copy still on
+   * its way to the producer to be counted.
+   *
+   * @param {'once' | 'twice'} times
+   * @this {comq.features.Context}
+   */
+  async function (times) {
+    const expected = TIMES[times]
+
+    await timeout(SETTLE_MS)
+
+    const executed = this.numbered.map((_, id) => this.executions.get(id) ?? 0)
+    const wrong = executed.flatMap((count, id) => count === expected ? [] : [`${id}: ${count}`])
+
+    assert.deepEqual(wrong, [], `Requests executed other than ${times} (id: times): ${wrong.join(', ')}`)
+  })
+
+After(
+  /**
+   * @this {comq.features.Context}
+   */
+  async function () {
+    this.answering?.resolve()
+    await this.producer?.close()
+
+    this.producer = undefined
+    this.executions = undefined
+    this.answering = undefined
+    this.numbered = undefined
+  })
+
+const TIMES = { once: 1, twice: 2 }
+
+const SETTLE_MS = 1000