comq 0.24.0 → 0.24.1

package/features/heartbeat.feature

@@ -0,0 +1,16 @@
+@heavy
+Feature: Idle connection tolerance
+
+  A connection that says nothing but heartbeats is as healthy as a busy one, and is kept for as
+  long as the broker goes on answering them, however long that is.
+
+  Background:
+    Given 1s AMQP heartbeat
+
+  Scenario: Keeping an idle connection
+    Given an active connection to the broker
+    Then the connection is not lost for 20 seconds
+
+  Scenario: Keeping an idle sharded connection
+    Given an active sharded connection
+    Then the connection is not lost for 20 seconds

package/features/rpc.call.recovery.feature

@@ -17,14 +17,14 @@ Feature: Calls under a key across failures
     it the key. The holder finds the key taken until the broker lets that connection go, and
     claims it then.
 
-    Given watchdog interval is set to 2000ms with 60s AMQP heartbeat
+    Given 1s AMQP heartbeat
     And a network that can go silent
     And an active connection to the broker
     And a holder answering `echo` under the `a` key:
       """
       (payload) => payload
       """
-    When the network goes silent
+    When the network goes silent, with the broker still hearing heartbeats
     Then the connection is lost within 10 seconds
     And the holder finds its key taken while the broker holds the silent connection
     When the silent connection is let go

package/features/rpc.shards.retransmission.feature

@@ -0,0 +1,46 @@
+@heavy
+Feature: Re-sending the requests a lost shard held
+
+  A Request is re-sent when the shard it went through is lost, since its Reply may be lost with it.
+  One that went through a shard that stays is answered through that shard, and is never re-sent
+  because another one was lost: a Request is not assumed to be idempotent.
+
+  Each scenario first takes one broker away from the consumer, so that the Requests it sends go
+  through the other one, and gives the broker back before losing one of them again.
+
+  Background:
+    Given 1s AMQP heartbeat
+    And a network that can go silent
+    And an active sharded connection
+    And a producer replying `echo` queue
+    # the consumer's channels exist on both brokers before one of them is taken away
+    And the consumer sends a request to the `echo` queue
+    And the consumer receives the reply
+    And a producer on another connection counting the requests to the `numbered` queue, answering when told
+
+  Scenario: Requests through the broker that stays are not re-sent when the other one is lost
+    Given the consumer's connection to broker 1 is cut
+    When the consumer sends 10 numbered requests to the `numbered` queue
+    And the consumer's connection to broker 1 is restored
+    And the consumer's connection to broker 1 is cut
+    And the producer answers
+    Then every numbered request is answered within 10 seconds
+    And every numbered request has been executed once
+
+  Scenario: Replies to the requests through the broker that stays are not lost with the other one
+    Given the consumer's connection to broker 1 is cut
+    When the consumer sends 10 numbered requests to the `numbered` queue
+    And the consumer's connection to broker 1 is restored
+    And the consumer's network to broker 1 goes silent
+    And the producer answers
+    Then every numbered request is answered within 15 seconds
+    And every numbered request has been executed once
+
+  Scenario: Requests through the broker that is lost are re-sent and answered
+    Given the consumer's connection to broker 0 is cut
+    When the consumer sends 10 numbered requests to the `numbered` queue
+    And the consumer's connection to broker 0 is restored
+    And the consumer's network to broker 1 goes silent
+    And the producer answers
+    Then every numbered request is answered within 15 seconds
+    And every numbered request has been executed twice

package/features/shards.feature

@@ -49,7 +49,7 @@ Feature: Sharded Connection
     Then the broker is up
 
   Scenario: Shard stops responding while requests are sent
-    Given watchdog interval is set to 3000ms with 1s AMQP heartbeat
+    Given 1s AMQP heartbeat
     And an active sharded connection
     And a producer replying `stalled` queue
     When the consumer sends 10 requests to the `stalled` queue as a broker freezes

package/features/silence.feature

@@ -2,10 +2,11 @@ Feature: Silent Connection Tolerance
 
   A connection that has gone silent is what a machine that woke from sleep is
   left with: the socket is open, neither the broker nor the operating system
-  reports anything, and nothing but the watchdog tells it from an idle connection.
+  reports anything, and nothing but the missing heartbeats tell it from an idle
+  connection.
 
   Background:
-    Given watchdog interval is set to 2000ms with 60s AMQP heartbeat
+    Given 1s AMQP heartbeat
     And a network that can go silent
 
   Scenario: Restoring a connection that has gone silent

package/features/steps/.types/context.d.ts

@@ -49,7 +49,11 @@ declare namespace comq.features {
     readonly address: string
 
     open(): Promise<void>
-    silence(): void
+    silence(heard?: boolean): void
+    refuse(): void
+    admit(): void
+    release(): void
+    cut(): void
     close(): Promise<void>
   }
 
@@ -57,6 +61,7 @@ declare namespace comq.features {
     client: Socket
     upstream: Socket
     silent: boolean
+    heartbeat?: NodeJS.Timeout
   }
 
 }

package/features/steps/brokers.js

@@ -139,7 +139,7 @@ const actions = {
     await docker('docker', ['kill', `comq-rmq-${n}`])
   },
   // a frozen broker keeps its connections open while answering nothing, which is
-  // what a publisher cannot tell from a working one until the watchdog fires
+  // what a publisher cannot tell from a working one until its heartbeats go missing
   frozen: async (n = 0) => {
     await docker('docker', ['pause', `comq-rmq-${n}`])
   }

package/features/steps/connection.js

@@ -5,6 +5,16 @@ const { timeout } = require('../../test/helpers')
 
 const { Given, When, Then } = require('@cucumber/cucumber')
 
+Given('{number}s AMQP heartbeat',
+  /**
+   * A connection that has received nothing for two heartbeats is taken for lost.
+   *
+   * @param {number} heartbeat
+   */
+  function (heartbeat) {
+    global.COMQ_TESTING_AMQP_HEARTBEAT = heartbeat
+  })
+
 Given('an active connection to the broker',
   /**
    * @this {comq.features.Context}
@@ -176,6 +186,19 @@ Then('the connection is lost within {number} second(s)',
     await emitted.call(this, 'close', seconds, 'connection was not lost')
   })
 
+Then('the connection is not lost for {number} seconds',
+  /**
+   * @param {number} seconds
+   * @this {comq.features.Context}
+   */
+  async function (seconds) {
+    delete this.events.close
+
+    await timeout(seconds * 1000)
+
+    assert.equal(this.events.close, undefined, 'The connection was lost')
+  })
+
 Then('publishing is {paused-event} within {number} second(s)',
   /**
    * @param {'paused' | 'resumed'} key

package/features/steps/context.js

@@ -114,8 +114,7 @@ class Context extends World {
     const url = PROTOCOL + user + ':' + password + '@' + address
     const query = []
 
-    // the watchdog measures silence, so it may only be shortened along with the
-    // interval at which a healthy broker is expected to say something
+    // a connection gone silent is told by its missing heartbeats, so a short one tells it soon
     if (global.COMQ_TESTING_AMQP_HEARTBEAT !== undefined) {
       query.push('heartbeat=' + global.COMQ_TESTING_AMQP_HEARTBEAT)
     }

package/features/steps/hooks.js

@@ -12,7 +12,6 @@ After(
    * @this {comq.features.Context}
    */
   async function () {
-    delete global.COMQ_TESTING_WATCHDOG_INTERVAL
     delete global.COMQ_TESTING_AMQP_HEARTBEAT
     delete global.COMQ_TESTING_AMQP_CHANNEL_MAX
 

package/features/steps/network.js

@@ -26,6 +26,14 @@ When('the network goes silent',
     silence.call(this)
   })
 
+When('the network goes silent, with the broker still hearing heartbeats',
+  /**
+   * @this {comq.features.Context}
+   */
+  function () {
+    silence.call(this, true)
+  })
+
 When('the network goes silent and refuses new connections',
   /**
    * @this {comq.features.Context}
@@ -66,11 +74,12 @@ When('the consumer sends a request to the {token} queue as the network goes sile
   })
 
 /**
+ * @param {boolean} [heard]
  * @this {comq.features.Context}
  */
-function silence () {
+function silence (heard) {
   // only a connection lost from now on counts as lost
   delete this.events.close
 
-  for (const network of this.networks) network.silence()
+  for (const network of this.networks) network.silence(heard)
 }

package/features/steps/networks.js

@@ -48,9 +48,19 @@ class Network {
   /**
    * Only the tunnels that are already open go silent: a connection established
    * afterwards is forwarded as usual, just as it is once a machine is awake.
+   *
+   * The broker notices a silent connection by its heartbeat as soon as the client does. One
+   * that is to be held by the broker for as long as the scenario needs is kept heard from:
+   * heartbeats are sent to the broker in the client's stead, until the network lets it go.
+   *
+   * @param {boolean} [heard] whether the broker goes on hearing heartbeats
    */
-  silence () {
-    for (const tunnel of this.#tunnels) tunnel.silent = true
+  silence (heard = false) {
+    for (const tunnel of this.#tunnels) {
+      tunnel.silent = true
+
+      if (heard) tunnel.heartbeat ??= setInterval(() => tunnel.upstream.write(HEARTBEAT), HEARTBEAT_MS)
+    }
   }
 
   /**
@@ -65,6 +75,17 @@ class Network {
     this.#refusing = false
   }
 
+  /**
+   * Takes the broker away from whoever is connected through this network: the tunnels that are
+   * open are closed at both ends, and a connection attempted from now on is refused until the
+   * network admits it.
+   */
+  cut () {
+    this.refuse()
+
+    for (const tunnel of this.#tunnels) this.#collapse(tunnel)
+  }
+
   /**
    * Closes the tunnels that went silent, at both ends, so the broker learns that the
    * connection it was still holding is gone — the moment its heartbeat would otherwise tell it.
@@ -117,10 +138,18 @@ class Network {
    * @param {comq.features.Tunnel} tunnel
    */
   #collapse (tunnel) {
+    clearInterval(tunnel.heartbeat)
+
     this.#tunnels.delete(tunnel)
     tunnel.client.destroy()
     tunnel.upstream.destroy()
   }
 }
 
+/** An AMQP heartbeat frame: type 8, channel 0, no payload, and the frame end. */
+const HEARTBEAT = Buffer.from([8, 0, 0, 0, 0, 0, 0, 0xCE])
+
+/** Often enough for the shortest heartbeat a scenario asks for. */
+const HEARTBEAT_MS = 250
+
 exports.Network = Network

package/features/steps/retransmission.js

@@ -0,0 +1,165 @@
+'use strict'
+
+const assert = require('node:assert')
+const { Promex } = require('promex')
+const { Given, When, Then, After } = require('@cucumber/cucumber')
+
+const { connect } = require('../../')
+const { getAddress, USER, PASSWORD, BROKERS_AMOUNT } = require('./brokers')
+const { timeout, until } = require('../../test/helpers')
+
+Given('a producer on another connection counting the requests to the {token} queue, answering when told',
+  /**
+   * Connected to every broker directly, so that nothing done to the consumer's network reaches it.
+   * It takes every Request as it comes, counts how many times it has been handed each of them, and
+   * holds the replies back until it is told to answer: whatever the consumer re-sends by then is
+   * counted, however quickly the first attempt would have been answered.
+   *
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  async function (queue) {
+    const urls = Array.from({ length: BROKERS_AMOUNT }, (_, n) =>
+      `amqp://${USER}:${PASSWORD}@${getAddress(n)}`)
+
+    this.producer = await connect(...urls)
+    this.executions = new Map()
+    this.answering = new Promex()
+
+    await this.producer.reply(queue, async ({ id }) => {
+      this.executions.set(id, (this.executions.get(id) ?? 0) + 1)
+
+      await this.answering
+
+      return id
+    })
+  })
+
+When('the producer answers',
+  /**
+   * @this {comq.features.Context}
+   */
+  function () {
+    this.answering.resolve()
+  })
+
+When('the consumer sends {number} numbered requests to the {token} queue',
+  /**
+   * @param {number} amount
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  function (amount, queue) {
+    this.numbered = Array.from({ length: amount }, (_, id) => {
+      const reply = this.io.request(queue, { id })
+
+      // asserted on by a later step, and an unhandled rejection would end the run before it
+      reply.catch(() => undefined)
+
+      return reply
+    })
+  })
+
+When('the consumer\'s connection to broker {number} is cut',
+  /**
+   * The broker is taken away from the consumer alone: the connection closes at both ends, and the
+   * consumer is refused until it is restored. Returns once the consumer has noticed, so that what
+   * it sends next goes to the other broker.
+   *
+   * @param {number} broker
+   * @this {comq.features.Context}
+   */
+  async function (broker) {
+    let lost = false
+
+    this.io.diagnose('close', (_error, index) => { if (index === broker) lost = true })
+    this.networks[broker].cut()
+
+    assert.equal(await until(() => lost, 10_000), true,
+      `The consumer's connection to broker ${broker} was not lost`)
+  })
+
+When('the consumer\'s connection to broker {number} is restored',
+  /**
+   * Returns once the consumer's channels to that broker have recovered, which is when the
+   * requests it may have lost with it are re-sent.
+   *
+   * @param {number} broker
+   * @this {comq.features.Context}
+   */
+  async function (broker) {
+    const recovered = new Set()
+
+    this.io.diagnose('recover', (type, index) => { if (index === broker) recovered.add(type) })
+    this.networks[broker].admit()
+
+    const restored = () => recovered.has('request') && recovered.has('reply')
+
+    assert.equal(await until(restored, 20_000), true,
+      `The consumer's channels to broker ${broker} have not recovered`)
+  })
+
+When('the consumer\'s network to broker {number} goes silent',
+  /**
+   * @param {number} broker
+   * @this {comq.features.Context}
+   */
+  function (broker) {
+    this.networks[broker].silence()
+  })
+
+Then('every numbered request is answered within {number} seconds',
+  /**
+   * @param {number} seconds
+   * @this {comq.features.Context}
+   */
+  async function (seconds) {
+    const answered = new Array(this.numbered.length).fill(false)
+
+    this.numbered.forEach((reply, id) => reply.then(() => { answered[id] = true }, () => undefined))
+
+    await until(() => answered.every(Boolean), seconds * 1000)
+
+    const unanswered = answered.flatMap((yes, id) => yes ? [] : [id])
+
+    assert.deepEqual(unanswered, [], `Requests ${unanswered.join(', ')} are not answered`)
+
+    for (const [id, reply] of this.numbered.entries()) assert.equal(await reply, id, 'Reply mismatch')
+  })
+
+Then('every numbered request has been executed {word}',
+  /**
+   * A re-sent Request is taken as soon as it arrives, so a moment is given for any copy still on
+   * its way to the producer to be counted.
+   *
+   * @param {'once' | 'twice'} times
+   * @this {comq.features.Context}
+   */
+  async function (times) {
+    const expected = TIMES[times]
+
+    await timeout(SETTLE_MS)
+
+    const executed = this.numbered.map((_, id) => this.executions.get(id) ?? 0)
+    const wrong = executed.flatMap((count, id) => count === expected ? [] : [`${id}: ${count}`])
+
+    assert.deepEqual(wrong, [], `Requests executed other than ${times} (id: times): ${wrong.join(', ')}`)
+  })
+
+After(
+  /**
+   * @this {comq.features.Context}
+   */
+  async function () {
+    this.answering?.resolve()
+    await this.producer?.close()
+
+    this.producer = undefined
+    this.executions = undefined
+    this.answering = undefined
+    this.numbered = undefined
+  })
+
+const TIMES = { once: 1, twice: 2 }
+
+const SETTLE_MS = 1000

package/features/steps/rpc.js

@@ -88,12 +88,6 @@ Given('idle timeout is set to {number}ms',
     global.COMQ_TESTING_IDLE_INTERVAL = value
   })
 
-Given('watchdog interval is set to {number}ms with {number}s AMQP heartbeat',
-  function (watchdog, heartbeat) {
-    global.COMQ_TESTING_WATCHDOG_INTERVAL = watchdog
-    global.COMQ_TESTING_AMQP_HEARTBEAT = heartbeat
-  })
-
 Given('a producer replying {token} queue',
   /**
    * @param {string} queue

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.24.0",
+  "version": "0.24.1",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",
@@ -67,5 +67,19 @@
     "env": [
       "jest"
     ]
+  },
+  "overrides": {
+    "test-exclude": "^8.0.0",
+    "rimraf": "^6.1.3",
+    "archiver-utils": {
+      "glob": "^13.0.6"
+    }
+  },
+  "allowScripts": {
+    "@parcel/watcher": false,
+    "cpu-features": false,
+    "protobufjs": false,
+    "ssh2": false,
+    "unrs-resolver": false
   }
 }

package/readme.md

@@ -457,8 +457,9 @@ Reconnection attempts will be made indefinitely, with intervals increasing up to
 Unless the URL sets one, a 15 second heartbeat is requested, so that a connection that is gone
 without a word, such as after a machine wakes from sleep, is noticed within a minute instead of
 being left to whatever the broker suggests.
-A connection that stays silent for three heartbeats is destroyed regardless of what the broker and
-the operating system have reported, since neither is guaranteed to report anything at all.
+A connection that has received nothing for two heartbeats is closed and restored regardless of what
+the broker and the operating system have reported, since neither is guaranteed to report anything
+at all. A connection that is merely idle receives the broker's heartbeats, and is kept.
 Requesting `heartbeat=0` disables both.
 If the broker rejects the connection, for example, due to access being denied, an exception will be thrown.
 Once reconnected, the topology will be recovered, and any unanswered requests and unconfirmed events will be
@@ -480,6 +481,10 @@ connection is re-established.
 
 Incoming messages are consumed from all shards.
 
+A Reply goes back through the shard its Request arrived on, while that shard is reachable. When a
+shard is lost, the Requests sent through it that are still unanswered are re-sent through the
+others, since their Replies may be lost with it; a Request sent through another shard is not.
+
 `async connect(...shards: string[]): IO`
 
 Returns an instance of `IO` once connections to the shards are established.
@@ -795,8 +800,8 @@ Subscribe to one of the diagnostic events:
   an argument.
 - `drain`: back pressure is removed from a channel. Channel type is passed.
 - `remove`: channel is removed from the [pool](#sharded-connection), having failed to publish.
-- `lost`: a shard has lost its connection, hence the requests awaiting their replies on it are
-  re-sent. Channel type is passed.
+- `lost`: a shard has lost its connection, hence the requests sent through it that are awaiting
+  their replies are re-sent. Channel type is passed.
 - `recover`: channel's topology is recovered. Channel type is passed.
 - `discard`: message is [parked](#parked-messages), having run out of attempts. Channel type,
   raw [amqp message object](https://amqp-node.github.io/amqplib/channel_api.html#channel_consume)

package/source/channel.js

@@ -70,7 +70,9 @@ class Channel {
     this.#failfast = index !== undefined
     this.#release = release
 
-    if (this.#failfast) failsafe.disable(this.send, this.publish)
+    // a shard's publish fails rather than waits for its connection to come back: the pool
+    // fails it over to another shard, and whoever published it is told where it went
+    if (this.#failfast) failsafe.disable(this.send, this.publish, this.route)
   }
 
   async create () {