comq 0.24.0 → 0.25.0

package/features/steps/rpc.js

@@ -88,12 +88,6 @@ Given('idle timeout is set to {number}ms',
     global.COMQ_TESTING_IDLE_INTERVAL = value
   })
 
-Given('watchdog interval is set to {number}ms with {number}s AMQP heartbeat',
-  function (watchdog, heartbeat) {
-    global.COMQ_TESTING_WATCHDOG_INTERVAL = watchdog
-    global.COMQ_TESTING_AMQP_HEARTBEAT = heartbeat
-  })
-
 Given('a producer replying {token} queue',
   /**
    * @param {string} queue
@@ -297,7 +291,37 @@ Then('the consumer receives the stream:',
 
     for await (const reply of this.stream) replies.push(reply)
 
-    assert.equal(values.length, replies.length, `Stream length mismatch: expected ${values.length}, received ${replies.length}`)
+    assert.deepStrictEqual(replies, values)
+  })
+
+Then('the consumer receives a stream of {number} values in order',
+  /**
+   * @param {number} count
+   * @this {comq.features.Context}
+   */
+  async function (count) {
+    const replies = []
+
+    for await (const reply of this.stream) replies.push(reply)
+
+    assert.deepStrictEqual(replies, Array.from({ length: count }, (_, i) => ({ i })))
+  })
+
+Then('the consumer receives the stream of buffers:',
+  /**
+   * @param {string} yaml what each buffer holds, as text
+   * @this {comq.features.Context}
+   */
+  async function (yaml) {
+    const values = parse(yaml)
+    const replies = []
+
+    for await (const reply of this.stream) {
+      assert.ok(Buffer.isBuffer(reply), `Not a buffer: ${JSON.stringify(reply)}`)
+      replies.push(reply.toString())
+    }
+
+    assert.deepStrictEqual(replies, values)
   })
 
 Then('the consumer receives the stream',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.24.0",
+  "version": "0.25.0",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",
@@ -27,19 +27,22 @@
     "feat": "scripts/feat.js"
   },
   "devDependencies": {
+    "@commitlint/cli": "^21.2.3",
+    "@commitlint/config-conventional": "^21.2.3",
     "@cucumber/cucumber": "^13.2.1",
+    "@jest/fake-timers": "^30.5.2",
     "@semantic-release/commit-analyzer": "^13.0.1",
     "@semantic-release/git": "^11.0.1",
     "@semantic-release/github": "^12.0.2",
     "@semantic-release/npm": "^13.1.2",
     "@semantic-release/release-notes-generator": "^14.1.0",
     "@testcontainers/rabbitmq": "^12.1.0",
-    "@types/jest": "^30.0.0",
     "conventional-changelog-conventionalcommits": "^8.0.0",
+    "expect": "^30.5.2",
     "fast-glob": "^3.3.3",
     "fuzzysort": "4.0.2",
     "husky": "^9.1.7",
-    "jest": "^30.4.2",
+    "jest-mock": "^30.5.2",
     "js-yaml": "^5.4.1",
     "randomstring": "^1.3.1",
     "semantic-release": "^25.0.9",
@@ -48,24 +51,43 @@
     "testcontainers": "^12.0.4"
   },
   "dependencies": {
-    "@toa.io/amqplib": "^0.1.0",
-    "promex": "^0.1.1",
-    "reretry": "^0.1.0"
+    "@toa.io/amqplib": "^0.1.0"
   },
   "engines": {
     "node": ">=24.0.0"
   },
   "scripts": {
     "prepare": "husky",
-    "test": "standard && NODE_OPTIONS=--experimental-vm-modules jest",
+    "test": "standard && node --test --experimental-test-module-mocks --import ./test/globals.js 'test/**/*.test.js' 'source/**/*.test.js'",
     "features": "cucumber-js --fail-fast -t 'not @manual'",
     "features:fast": "cucumber-js --fail-fast -t 'not @manual and not @heavy'",
     "benchmark": "node benchmarks/run.js",
     "lint": "standard --fix --verbose | snazzy"
   },
   "standard": {
-    "env": [
-      "jest"
+    "globals": [
+      "afterAll",
+      "afterEach",
+      "beforeAll",
+      "beforeEach",
+      "describe",
+      "expect",
+      "it",
+      "jest",
+      "test"
     ]
+  },
+  "overrides": {
+    "rimraf": "^6.1.3",
+    "archiver-utils": {
+      "glob": "^13.0.6"
+    }
+  },
+  "allowScripts": {
+    "@parcel/watcher": false,
+    "cpu-features": false,
+    "protobufjs": false,
+    "ssh2": false,
+    "unrs-resolver": false
   }
 }

package/readme.md

@@ -354,6 +354,12 @@ for await (const number of stream)
   console.log(number)
 ```
 
+Values a producer yields together travel together: what the iterator yields before the event loop
+turns is sent as one message, up to 128 values, and a value that has to be waited for is sent as
+it comes. Buffers travel in messages of their own. The stream returned by `IO.request` yields each
+value on its own either way. Values in an encoding other than `application/json` travel one to a
+message.
+
 ### Stream topology
 
 <picture>
@@ -457,8 +463,9 @@ Reconnection attempts will be made indefinitely, with intervals increasing up to
 Unless the URL sets one, a 15 second heartbeat is requested, so that a connection that is gone
 without a word, such as after a machine wakes from sleep, is noticed within a minute instead of
 being left to whatever the broker suggests.
-A connection that stays silent for three heartbeats is destroyed regardless of what the broker and
-the operating system have reported, since neither is guaranteed to report anything at all.
+A connection that has received nothing for two heartbeats is closed and restored regardless of what
+the broker and the operating system have reported, since neither is guaranteed to report anything
+at all. A connection that is merely idle receives the broker's heartbeats, and is kept.
 Requesting `heartbeat=0` disables both.
 If the broker rejects the connection, for example, due to access being denied, an exception will be thrown.
 Once reconnected, the topology will be recovered, and any unanswered requests and unconfirmed events will be
@@ -480,6 +487,10 @@ connection is re-established.
 
 Incoming messages are consumed from all shards.
 
+A Reply goes back through the shard its Request arrived on, while that shard is reachable. When a
+shard is lost, the Requests sent through it that are still unanswered are re-sent through the
+others, since their Replies may be lost with it; a Request sent through another shard is not.
+
 `async connect(...shards: string[]): IO`
 
 Returns an instance of `IO` once connections to the shards are established.
@@ -795,8 +806,8 @@ Subscribe to one of the diagnostic events:
   an argument.
 - `drain`: back pressure is removed from a channel. Channel type is passed.
 - `remove`: channel is removed from the [pool](#sharded-connection), having failed to publish.
-- `lost`: a shard has lost its connection, hence the requests awaiting their replies on it are
-  re-sent. Channel type is passed.
+- `lost`: a shard has lost its connection, hence the requests sent through it that are awaiting
+  their replies are re-sent. Channel type is passed.
 - `recover`: channel's topology is recovered. Channel type is passed.
 - `discard`: message is [parked](#parked-messages), having run out of attempts. Channel type,
   raw [amqp message object](https://amqp-node.github.io/amqplib/channel_api.html#channel_consume)

package/source/.io/ReplyPipe.js

@@ -1,8 +1,8 @@
 'use strict'
 
 const { EventEmitter } = require('node:events')
-const { Promex } = require('promex')
-const { control, FLOW_HEADER, HEARTBEAT_INTERVAL } = require('./const')
+const { Promex } = require('../promex')
+const { control, batch, FLOW_HEADER, HEARTBEAT_INTERVAL } = require('./const')
 
 /** @typedef {(message: any, properties?: comq.amqp.options.Publish) => Promise<void>} Reply */
 
@@ -36,6 +36,9 @@ class ReplyPipe extends EventEmitter {
   /** @type {Reply} */
   #reply
 
+  /** Whether the encoding can carry a list of values. */
+  #lists
+
   /**
    * @param {comq.amqp.Message} request
    * @param {stream.Readable} stream
@@ -54,8 +57,12 @@ class ReplyPipe extends EventEmitter {
     this.#reply = reply
     this.#replyTo = replyTo
 
+    this.#lists = request.properties.contentType === LISTS
+
     this.#properties = {
       chunk: { correlationId, ...CHUNK },
+      batch: { correlationId, ...CHUNK, type: batch.values },
+      buffers: { correlationId, ...CHUNK, type: batch.buffers },
       control: { correlationId, replyTo: feedback.queue, ...CONTROL },
       ok: { correlationId, replyTo: feedback.queue, ...CONTROL, headers: FLOW }
     }
@@ -99,21 +106,90 @@ class ReplyPipe extends EventEmitter {
    */
   async #pump () {
     try {
-      for await (const chunk of this.#stream) {
-        if (this.#gate !== null) await this.#gate
-        if (this.#closed) break
+      await this.#pumpBatches()
+    } catch {
+      // the source has been destroyed, by this pipe or by whoever made it
+    }
+
+    this.#close()
+  }
+
+  /**
+   * Values go out together while they arrive together: what the source yields before the event
+   * loop turns joins the message, and the first value that has to be waited for leaves with the
+   * ones before it rather than waiting for company. A cursor yielding the documents of a batch it
+   * holds sends them as one message; a source yielding one value at a time sends each as it
+   * comes. Values and buffers travel in messages of their own, and values in an encoding that
+   * holds no list travel one to a message.
+   */
+  async #pumpBatches () {
+    const iterator = this.#stream[Symbol.asyncIterator]()
+
+    let pending = next(iterator)
+
+    while (true) {
+      const first = await pending
 
-        await this.#transmit(chunk, this.#properties.chunk)
+      if (first.done === true) return
 
-        if (this.#closed) break
+      const chunks = [first.value]
+      const buffers = Buffer.isBuffer(first.value)
+      const turn = immediate()
 
-        this.#heartbeat()
+      let done = false
+
+      pending = next(iterator)
+
+      while (chunks.length < (buffers || this.#lists ? MAX_BATCH : 1)) {
+        const arrived = await Promise.race([pending, turn])
+
+        if (arrived === TURN) break
+
+        if (arrived.done === true) {
+          done = true
+
+          break
+        }
+
+        // of the other kind, it starts the next message
+        if (Buffer.isBuffer(arrived.value) !== buffers) {
+          pending = Promise.resolve(arrived)
+
+          break
+        }
+
+        chunks.push(arrived.value)
+        pending = next(iterator)
       }
-    } catch {
-      // the source has been destroyed, by this pipe or by whoever made it
+
+      if (!(await this.#send(...this.#message(chunks, buffers)))) {
+        await iterator.return?.()
+
+        return
+      }
+
+      if (done) return
     }
+  }
 
-    this.#close()
+  #message (chunks, buffers) {
+    if (chunks.length === 1) return [chunks[0], this.#properties.chunk]
+    else if (buffers) return [prefixed(chunks), this.#properties.buffers]
+    else return [chunks, this.#properties.batch]
+  }
+
+  /** Answers whether the pipe goes on. */
+  async #send (data, properties) {
+    if (this.#gate !== null) await this.#gate
+    if (this.#closed) return false
+
+    await this.#transmit(data, properties)
+
+    if (this.#closed) return false
+
+    this.#heartbeat()
+
+    return true
   }
 
   async #transmit (data, properties) {
@@ -206,6 +282,49 @@ class ReplyPipe extends EventEmitter {
 /** @type {comq.amqp.options.Publish} */
 const CHUNK = { mandatory: true }
 
+/** the most values a message carries */
+const MAX_BATCH = 128
+
+/** the encoding a batch of values is a list in */
+const LISTS = 'application/json'
+
+const TURN = Symbol('turn')
+
+/**
+ * The next value, asked for ahead of when it is awaited. A source that fails while a message is
+ * being sent is found failed by the loop that awaits it next, or by nobody where the pipe has
+ * stopped — and a rejection nobody awaits is an unhandled one.
+ */
+function next (iterator) {
+  const pending = iterator.next()
+
+  pending.catch(noop)
+
+  return pending
+}
+
+/** Settles once the event loop has turned, with what says so. */
+function immediate () {
+  return new Promise((resolve) => setImmediate(resolve, TURN))
+}
+
+/**
+ * @param {Buffer[]} buffers
+ * @returns {Buffer}
+ */
+function prefixed (buffers) {
+  const parts = []
+
+  for (const buffer of buffers) {
+    const length = Buffer.allocUnsafe(4)
+
+    length.writeUInt32BE(buffer.length)
+    parts.push(length, buffer)
+  }
+
+  return Buffer.concat(parts)
+}
+
 /** @type {comq.amqp.options.Publish} */
 const CONTROL = { type: 'control', mandatory: true }
 

package/source/.io/ReplyStream.js

@@ -1,8 +1,8 @@
 'use strict'
 
 const { Readable } = require('node:stream')
-const { Promex } = require('promex')
-const { IDLE_INTERVAL, FLOW_HEADER, control } = require('./const')
+const { Promex } = require('../promex')
+const { IDLE_INTERVAL, FLOW_HEADER, batch, control } = require('./const')
 const { Interrupted } = require('../interrupted')
 
 class ReplyStream extends Readable {
@@ -133,10 +133,29 @@ class ReplyStream extends Readable {
 
     if (properties.type === 'control')
       this._control(payload, properties)
+    else if (properties.type === batch.values)
+      this._values(payload)
+    else if (properties.type === batch.buffers)
+      this._values(split(payload))
     else if (!this.push(payload))
       this._throttle()
   }
 
+  /**
+   * The values of one message, pushed whatever the consumer's room: they have arrived, and a
+   * pause asked for now holds back the messages after them.
+   *
+   * @param {unknown[]} values
+   * @private
+   */
+  _values (values) {
+    let room = true
+
+    for (const value of values) room = this.push(value) && room
+
+    if (!room) this._throttle()
+  }
+
   /**
    * Values arriving out of order are held until the gap is filled. The hold is
    * bounded by their number and, roughly, by their size: the size is that of
@@ -213,6 +232,26 @@ class ReplyStream extends Readable {
   }
 }
 
+/**
+ * The buffers of a message that carries each prefixed with its length.
+ *
+ * @param {Buffer} buffer
+ * @returns {Buffer[]}
+ */
+function split (buffer) {
+  const buffers = []
+
+  for (let offset = 0; offset < buffer.length;) {
+    const length = buffer.readUInt32BE(offset)
+    const start = offset + 4
+
+    buffers.push(buffer.subarray(start, start + length))
+    offset = start + length
+  }
+
+  return buffers
+}
+
 const MAX_BUFFER_SIZE = 1000
 
 /** @type {number} */

package/source/.io/const.js

@@ -14,5 +14,11 @@ exports.control = {
  */
 exports.FLOW_HEADER = 'x-flow'
 
+/** The type of a message carrying values: a list of them, or buffers prefixed with their length. */
+exports.batch = {
+  values: 'batch',
+  buffers: 'buffers'
+}
+
 exports.HEARTBEAT_INTERVAL = 5_000
 exports.IDLE_INTERVAL = 12_000

package/source/attributes/track.test.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const { generate } = require('randomstring')
-const { Promex } = require('promex')
+const { Promex } = require('../promex')
 
 const { track } = require('./track')
 

package/source/channel.js

@@ -1,7 +1,7 @@
 'use strict'
 
-const { Promex } = require('promex')
-const { retry } = require('reretry')
+const { Promex } = require('./promex')
+const { retry } = require('./retry')
 const { failsafe, lazy, recall } = require('./attributes')
 const { verdictOf, PARK } = require('./verdicts')
 const emitter = require('./emitter')
@@ -70,7 +70,9 @@ class Channel {
     this.#failfast = index !== undefined
     this.#release = release
 
-    if (this.#failfast) failsafe.disable(this.send, this.publish)
+    // a shard's publish fails rather than waits for its connection to come back: the pool
+    // fails it over to another shard, and whoever published it is told where it went
+    if (this.#failfast) failsafe.disable(this.send, this.publish, this.route)
   }
 
   async create () {

package/source/connection.js

@@ -2,8 +2,8 @@
 
 const { setTimeout: delay } = require('node:timers/promises')
 const amqp = require('@toa.io/amqplib')
-const { Promex } = require('promex')
-const { retry } = require('reretry')
+const { Promex } = require('./promex')
+const { retry } = require('./retry')
 
 const { failsafe } = require('./attributes')
 const presets = require('./topology')
@@ -35,18 +35,9 @@ class Connection {
   /** @type {boolean} */
   #closed = false
 
-  /** @type {NodeJS.Timeout | null} */
-  #heartbeatTimer = null
-
   /** @type {comq.topology.Overrides} */
   #overrides
 
-  /** @type {import('node:net').Socket | null} */
-  #heartbeatSocket = null
-
-  /** @type {(() => void) | null} */
-  #heartbeatReset = null
-
   #diagnostics = emitter.create()
 
   /**
@@ -147,7 +138,6 @@ class Connection {
     connection.on('close', (error) => this.#close(connection, error))
     this.#connection = connection
 
-    this.#armWatchdog(connection)
     this.#diagnostics.emit('open')
 
     try {
@@ -177,11 +167,14 @@ class Connection {
   #close = (connection, error) => {
     if (this.#connection !== connection) return
 
-    this.#disarmWatchdog()
     this.#diagnostics.emit('close', error)
     connection.removeAllListeners()
     this.#connection = undefined
 
+    // amqplib only ends the socket of a connection it has given up on, and a peer that has gone
+    // silent never ends its side, which would leave the socket to the kernel's keepalive
+    if (error !== undefined) connection.connection?.stream?.destroy()
+
     if (error !== undefined && !this.#closed) {
       this.#reopen().catch((exception) => this.#diagnostics.emit('error', exception))
     }
@@ -224,10 +217,7 @@ class Connection {
    * @param {comq.amqp.Connection} connection
    */
   #drop (connection) {
-    if (this.#connection === connection) {
-      this.#disarmWatchdog()
-      this.#connection = undefined
-    }
+    if (this.#connection === connection) this.#connection = undefined
 
     connection.removeAllListeners()
 
@@ -294,46 +284,6 @@ class Connection {
     }
   }
 
-  /**
-   * @param {comq.amqp.Connection} connection
-   * @param {number} [timeoutMs]
-   */
-  #armWatchdog (connection, timeoutMs = tolerance(connection)) {
-    this.#disarmWatchdog()
-
-    const socket = connection.connection?.stream
-
-    if (socket === undefined || timeoutMs === undefined) return
-
-    const reset = () => {
-      clearTimeout(this.#heartbeatTimer)
-      // destroying a socket without an error tells amqplib nothing: it only
-      // listens for 'error' and 'end', so a bare destroy() leaves the connection
-      // silently dead — no 'close' event, no recovery, every pending operation
-      // hanging forever
-      this.#heartbeatTimer = setTimeout(() => socket.destroy(silence()), timeoutMs)
-      this.#heartbeatTimer.unref()
-    }
-
-    this.#heartbeatSocket = socket
-    this.#heartbeatReset = reset
-
-    reset()
-    socket.on('data', reset)
-  }
-
-  #disarmWatchdog () {
-    clearTimeout(this.#heartbeatTimer)
-    this.#heartbeatTimer = null
-
-    // otherwise a byte arriving on a replaced socket rearms the watchdog of a
-    // connection that is already gone, leaving the current one unguarded
-    this.#heartbeatSocket?.off('data', this.#heartbeatReset)
-
-    this.#heartbeatSocket = null
-    this.#heartbeatReset = null
-  }
-
   #transient (exception) {
     if (this.#running) return true
     if (TRANSIENT_CODES.has(exception.code)) return true
@@ -348,6 +298,10 @@ class Connection {
  * to suggest one that leaves a connection lost for minutes before anything
  * notices, and RabbitMQ suggests 60 by default.
  *
+ * It is amqplib that tells a connection gone silent: one that has received nothing
+ * for two heartbeats is closed with an error, whatever the broker and the operating
+ * system have reported, which is what restores it.
+ *
  * @type {number}
  */
 const HEARTBEAT_S = 15
@@ -355,15 +309,6 @@ const HEARTBEAT_S = 15
 /** @type {number} */
 const KEEPALIVE_MS = 10_000
 
-/** How many heartbeats a connection may miss before it is destroyed. */
-const MISSED_HEARTBEATS = 3
-
-/** @type {number} */
-const WATCHDOG_MIN_MS = 15_000
-
-/** The watchdog interval used when the negotiated heartbeat is unknown. */
-const WATCHDOG_MS = 60_000
-
 /** @type {number} */
 const CONNECT_MS = 30_000
 
@@ -422,28 +367,6 @@ function heartbeaten (url) {
   return url + (url.includes('?') ? '&' : '?') + 'heartbeat=' + HEARTBEAT_S
 }
 
-/**
- * The watchdog measures silence, so it cannot be shorter than the interval at
- * which a healthy broker is expected to say something, which is a heartbeat
- * frame every half a heartbeat. A connection that has agreed to no heartbeats
- * says nothing at all while it is idle, leaving nothing to measure.
- *
- * @param {comq.amqp.Connection} connection
- * @return {number | undefined}
- */
-function tolerance (connection) {
-  const override = global.COMQ_TESTING_WATCHDOG_INTERVAL
-
-  if (override !== undefined) return override
-
-  const heartbeat = connection.connection?.heartbeat
-
-  if (heartbeat === undefined) return WATCHDOG_MS
-  if (heartbeat === 0) return undefined
-
-  return Math.max(heartbeat * MISSED_HEARTBEATS * 1000, WATCHDOG_MIN_MS)
-}
-
 /**
  * @return {Error}
  */