comq 0.24.0 → 0.24.1

package/source/connection.js

@@ -35,18 +35,9 @@ class Connection {
   /** @type {boolean} */
   #closed = false
 
-  /** @type {NodeJS.Timeout | null} */
-  #heartbeatTimer = null
-
   /** @type {comq.topology.Overrides} */
   #overrides
 
-  /** @type {import('node:net').Socket | null} */
-  #heartbeatSocket = null
-
-  /** @type {(() => void) | null} */
-  #heartbeatReset = null
-
   #diagnostics = emitter.create()
 
   /**
@@ -147,7 +138,6 @@ class Connection {
     connection.on('close', (error) => this.#close(connection, error))
     this.#connection = connection
 
-    this.#armWatchdog(connection)
     this.#diagnostics.emit('open')
 
     try {
@@ -177,11 +167,14 @@ class Connection {
   #close = (connection, error) => {
     if (this.#connection !== connection) return
 
-    this.#disarmWatchdog()
     this.#diagnostics.emit('close', error)
     connection.removeAllListeners()
     this.#connection = undefined
 
+    // amqplib only ends the socket of a connection it has given up on, and a peer that has gone
+    // silent never ends its side, which would leave the socket to the kernel's keepalive
+    if (error !== undefined) connection.connection?.stream?.destroy()
+
     if (error !== undefined && !this.#closed) {
       this.#reopen().catch((exception) => this.#diagnostics.emit('error', exception))
     }
@@ -224,10 +217,7 @@ class Connection {
    * @param {comq.amqp.Connection} connection
    */
   #drop (connection) {
-    if (this.#connection === connection) {
-      this.#disarmWatchdog()
-      this.#connection = undefined
-    }
+    if (this.#connection === connection) this.#connection = undefined
 
     connection.removeAllListeners()
 
@@ -294,46 +284,6 @@ class Connection {
     }
   }
 
-  /**
-   * @param {comq.amqp.Connection} connection
-   * @param {number} [timeoutMs]
-   */
-  #armWatchdog (connection, timeoutMs = tolerance(connection)) {
-    this.#disarmWatchdog()
-
-    const socket = connection.connection?.stream
-
-    if (socket === undefined || timeoutMs === undefined) return
-
-    const reset = () => {
-      clearTimeout(this.#heartbeatTimer)
-      // destroying a socket without an error tells amqplib nothing: it only
-      // listens for 'error' and 'end', so a bare destroy() leaves the connection
-      // silently dead — no 'close' event, no recovery, every pending operation
-      // hanging forever
-      this.#heartbeatTimer = setTimeout(() => socket.destroy(silence()), timeoutMs)
-      this.#heartbeatTimer.unref()
-    }
-
-    this.#heartbeatSocket = socket
-    this.#heartbeatReset = reset
-
-    reset()
-    socket.on('data', reset)
-  }
-
-  #disarmWatchdog () {
-    clearTimeout(this.#heartbeatTimer)
-    this.#heartbeatTimer = null
-
-    // otherwise a byte arriving on a replaced socket rearms the watchdog of a
-    // connection that is already gone, leaving the current one unguarded
-    this.#heartbeatSocket?.off('data', this.#heartbeatReset)
-
-    this.#heartbeatSocket = null
-    this.#heartbeatReset = null
-  }
-
   #transient (exception) {
     if (this.#running) return true
     if (TRANSIENT_CODES.has(exception.code)) return true
@@ -348,6 +298,10 @@ class Connection {
  * to suggest one that leaves a connection lost for minutes before anything
  * notices, and RabbitMQ suggests 60 by default.
  *
+ * It is amqplib that tells a connection gone silent: one that has received nothing
+ * for two heartbeats is closed with an error, whatever the broker and the operating
+ * system have reported, which is what restores it.
+ *
  * @type {number}
  */
 const HEARTBEAT_S = 15
@@ -355,15 +309,6 @@ const HEARTBEAT_S = 15
 /** @type {number} */
 const KEEPALIVE_MS = 10_000
 
-/** How many heartbeats a connection may miss before it is destroyed. */
-const MISSED_HEARTBEATS = 3
-
-/** @type {number} */
-const WATCHDOG_MIN_MS = 15_000
-
-/** The watchdog interval used when the negotiated heartbeat is unknown. */
-const WATCHDOG_MS = 60_000
-
 /** @type {number} */
 const CONNECT_MS = 30_000
 
@@ -422,28 +367,6 @@ function heartbeaten (url) {
   return url + (url.includes('?') ? '&' : '?') + 'heartbeat=' + HEARTBEAT_S
 }
 
-/**
- * The watchdog measures silence, so it cannot be shorter than the interval at
- * which a healthy broker is expected to say something, which is a heartbeat
- * frame every half a heartbeat. A connection that has agreed to no heartbeats
- * says nothing at all while it is idle, leaving nothing to measure.
- *
- * @param {comq.amqp.Connection} connection
- * @return {number | undefined}
- */
-function tolerance (connection) {
-  const override = global.COMQ_TESTING_WATCHDOG_INTERVAL
-
-  if (override !== undefined) return override
-
-  const heartbeat = connection.connection?.heartbeat
-
-  if (heartbeat === undefined) return WATCHDOG_MS
-  if (heartbeat === 0) return undefined
-
-  return Math.max(heartbeat * MISSED_HEARTBEATS * 1000, WATCHDOG_MIN_MS)
-}
-
 /**
  * @return {Error}
  */

package/source/io.js

@@ -154,7 +154,7 @@ class IO {
         }
 
         return await this.#ask(queue, payload, terms,
-          (buffer, properties) => this.#requests.send(queue, buffer, properties))
+          (buffer, properties, via) => this.#requests.send(queue, buffer, properties, via))
       }))
 
   // the terms are taken once, so a re-sent call keeps the deadline of the first
@@ -170,7 +170,8 @@ class IO {
       async (exchange, key, payload, terms) => {
         // mandatory: a Request nobody holds the key of is returned rather than dropped
         return await this.#ask(exchange, payload, terms,
-          (buffer, properties) => this.#requests.route(exchange, key, buffer, { ...properties, mandatory: true }))
+          (buffer, properties, via) =>
+            this.#requests.route(exchange, key, buffer, { ...properties, mandatory: true }, via))
       }))
 
   consume = lazy(this, this.#createEventChannel,
@@ -357,12 +358,20 @@ class IO {
     return channel
   }
 
+  /**
+   * On a sharded connection, what is re-sent is what the shard in question holds: the index of
+   * the shard comes with each of the events, and a Request that went through another shard is
+   * answered through that one.
+   */
   #setupRetransmission () {
     if (this.#requests.sharded === true) {
       // a shard leaves the pool when it rejects a publish, and is lost when its
       // connection drops, which leaves an already sent request unanswered
       this.#requests.diagnose('remove', this.#retransmit)
       this.#requests.diagnose('lost', this.#retransmit)
+
+      // a returned call is published on another shard, and is held by that one from then on
+      this.#requests.diagnose('reroute', this.#rerouted)
     } else {
       this.#requests.diagnose('recover', this.#retransmit)
     }
@@ -462,11 +471,33 @@ class IO {
 
     if (signal !== undefined) this.#abandon(request, reply, signal)
 
-    await abortable(publish(buffer, properties), signal)
+    request.publishing = true
+
+    try {
+      await abortable(publish(buffer, properties, (index) => this.#board(request, index)), signal)
+    } finally {
+      request.publishing = false
+    }
+
+    // the shard it went through was lost while it was being published, and it has not failed
+    // over to another since
+    if (request.lost === true) this.#resend(request, reply)
 
     return reply
   }
 
+  /**
+   * Records the shard a Request is published through. It is recorded as the shard is chosen,
+   * rather than once the publish is done, so that a shard lost in between is known to hold it.
+   *
+   * @param {comq.Request} request
+   * @param {number} index
+   */
+  #board (request, index) {
+    request.shard = index
+    request.lost = false
+  }
+
   /**
    * Stops waiting for a Reply once the signal aborts. The Request is forgotten at once, so a
    * retransmission leaves it be, and a Reply arriving later finds nobody waiting for it.
@@ -494,17 +525,41 @@ class IO {
    * @param {comq.amqp.Message} message
    */
   #returned = (message) => {
+    const pending = this.#pendingOf(message)
+
+    if (pending === undefined) return
+
+    const [reply, request] = pending
+
+    request.emitter.off(request.properties.correlationId)
+    this.#pendingReplies.delete(reply)
+    reply.reject(new Unroutable(message.fields.exchange, message.fields.routingKey))
+  }
+
+  /**
+   * A Request one shard returned is published on another.
+   *
+   * @param {comq.amqp.Message} message
+   * @param {number} index
+   */
+  #rerouted = (message, index) => {
+    const pending = this.#pendingOf(message)
+
+    if (pending !== undefined) this.#board(pending[1], index)
+  }
+
+  /**
+   * @param {comq.amqp.Message} message
+   * @return {[Promex, comq.Request] | undefined}
+   */
+  #pendingOf (message) {
     const { correlationId, replyTo } = message.properties
 
     for (const [reply, request] of this.#pendingReplies) {
       if (request.properties.correlationId !== correlationId) continue
       if (request.emitter.queue !== replyTo) continue
 
-      request.emitter.off(correlationId)
-      this.#pendingReplies.delete(reply)
-      reply.reject(new Unroutable(message.fields.exchange, message.fields.routingKey))
-
-      return
+      return [reply, request]
     }
   }
 
@@ -665,24 +720,44 @@ class IO {
     // an unroutable reply must be returned by the broker rather than dropped
     properties.mandatory = true
 
-    return await this.#replies.fire(replyTo, buffer, properties)
+    // back the way the request came: see `fire` of a sharded channel
+    return await this.#replies.fire(replyTo, buffer, properties, request)
   }
 
   #recover (exception) {
     if (exception !== RETRANSMISSION) return false
   }
 
-  #retransmit = () => {
+  /**
+   * Re-sends the Requests awaiting their replies on a shard, or on the connection when no shard
+   * is given. A Request that went through another shard is not re-sent: it is answered through
+   * that shard, and it is not for comq to assume that it is idempotent.
+   *
+   * @param {number} [index]
+   */
+  #retransmit = (index) => {
     for (const [reply, request] of this.#pendingReplies) {
-      // detaching this attempt alone leaves the listeners of the reply streams
-      // that are still flowing over the other shards in place
-      request.emitter.off(request.properties.correlationId)
+      if (index !== undefined && request.shard !== index) continue
 
-      // trigger failsafe attribute
-      reply.reject(RETRANSMISSION)
+      // one still being published is re-sent once it is, unless it fails over to another shard
+      if (request.publishing === true) request.lost = true
+      else this.#resend(request, reply)
     }
   }
 
+  /**
+   * @param {comq.Request} request
+   * @param {Promex} reply
+   */
+  #resend (request, reply) {
+    // detaching this attempt alone leaves the listeners of the reply streams
+    // that are still flowing over the other shards in place
+    request.emitter.off(request.properties.correlationId)
+
+    // trigger failsafe attribute
+    reply.reject(RETRANSMISSION)
+  }
+
   /**
    * @param {any} payload
    * @param {comq.Encoding} [contentType]

package/source/shards/channel.js

@@ -57,8 +57,12 @@ class Channel {
     await Promise.any(promises)
   }
 
+  /**
+   * A message is told which shard it arrived on, so that whatever answers it can go back the same
+   * way. See `fire`.
+   */
   async consume (queue, consumer) {
-    return await this.#every((channel) => channel.consume(queue, consumer))
+    return await this.#every((channel) => channel.consume(queue, arrival(consumer, channel)))
   }
 
   async subscribe (queue, group, consumer) {
@@ -74,24 +78,51 @@ class Channel {
    * claiming it, and holds it as well once it is let go.
    */
   async held (exchange, queue, key, consumer) {
-    await Promise.any(this.#apply((channel) => channel.held(exchange, queue, key, consumer)))
+    await Promise.any(this.#apply((channel) => channel.held(exchange, queue, key, arrival(consumer, channel))))
   }
 
-  async send (queue, buffer, options) {
-    await this.#one((channel) => channel.send(queue, buffer, options))
+  /**
+   * @param {string} queue
+   * @param {Buffer} buffer
+   * @param {comq.amqp.options.Publish} [options]
+   * @param {(index: number) => void} [via] told which shard the message is published through,
+   * as soon as one is chosen and again whenever the publish fails over to another
+   */
+  async send (queue, buffer, options, via) {
+    await this.#one((channel) => channel.send(queue, buffer, options), { via })
   }
 
   async publish (exchange, buffer, options) {
     await this.#one((channel) => channel.publish(exchange, buffer, options))
   }
 
-  async route (exchange, key, buffer, options) {
-    await this.#one((channel) => channel.route(exchange, key, buffer, options))
+  /**
+   * @param {string} exchange
+   * @param {string} key
+   * @param {Buffer} buffer
+   * @param {comq.amqp.options.Publish} [options]
+   * @param {(index: number) => void} [via] as `send` has it
+   */
+  async route (exchange, key, buffer, options, via) {
+    await this.#one((channel) => channel.route(exchange, key, buffer, options), { via })
   }
 
-  async fire (queue, buffer, options) {
+  /**
+   * A reply goes back through the shard the message it answers arrived on. Whoever sent that
+   * message re-sends what it sent through a shard once the shard is lost, and nothing else, so a
+   * reply taking another way could be lost with a shard that nothing is re-sent for. It takes
+   * another way when that shard is not reachable, which leaves its message unacknowledged there,
+   * to be delivered and answered again.
+   *
+   * @param {string} queue
+   * @param {Buffer} buffer
+   * @param {comq.amqp.options.Publish} [options]
+   * @param {comq.amqp.Message} [origin] the message this one answers
+   * @return {Promise<boolean>}
+   */
+  async fire (queue, buffer, options, origin) {
     // noinspection  JSValidateTypes
-    return await this.#one((channel) => channel.fire(queue, buffer, options))
+    return await this.#one((channel) => channel.fire(queue, buffer, options), { prefer: origin?.[ARRIVAL] })
   }
 
   async close () {
@@ -230,6 +261,9 @@ class Channel {
 
     const next = rest[Math.floor(Math.random() * rest.length)]
 
+    // the message is on that shard now, and is lost with it rather than with this one
+    this.#diagnostics.emit(REROUTE, message, next.index)
+
     // a Request under a key goes through its routed exchange, which `route` declares on that
     // shard before publishing: publishing to an exchange a broker lacks closes the connection
     const retried = exchange === DEFAULT
@@ -374,9 +408,13 @@ class Channel {
   }
 
   /**
+   * Publishes through a shard chosen at random, or through the one preferred while it is
+   * reachable, and fails over to another when that one fails.
+   *
    * @param {(channel: comq.Channel) => void} fn
+   * @param {{ via?: (index: number) => void, prefer?: number }} [route]
    */
-  async #one (fn) {
+  async #one (fn, route = {}) {
     // capture before the check: recover may replace `#recovery` in between
     const waiting = this.#recovery
     const pool = this.#reachable()
@@ -384,21 +422,38 @@ class Channel {
     if (pool.length === 0) {
       await waiting
 
-      return this.#one(fn)
+      return this.#one(fn, route)
     }
 
-    const channel = pool[Math.floor(Math.random() * pool.length)]
+    const channel = pool.find((channel) => channel.index === route.prefer) ??
+      pool[Math.floor(Math.random() * pool.length)]
+
+    // told before the publish is awaited: a shard lost while it is under way has it on board
+    route.via?.(channel.index)
 
     try {
       return await fn(channel)
     } catch {
       this.#remove(channel)
 
-      return this.#one(fn)
+      return this.#one(fn, route)
     }
   }
 }
 
+/**
+ * @param {comq.channels.Consumer} consumer
+ * @param {comq.Channel} channel
+ * @return {comq.channels.Consumer}
+ */
+function arrival (consumer, channel) {
+  return (message) => {
+    message[ARRIVAL] = channel.index
+
+    return consumer(message)
+  }
+}
+
 /**
  * @param {comq.Connection[]} connections
  * @param {comq.topology.type} type
@@ -418,6 +473,12 @@ const REMOVE = 'remove'
 const RETURN_HEADER = 'x-return'
 const LOST = 'lost'
 
+/** A returned message published on another shard, which is where it can be lost from now on. */
+const REROUTE = 'reroute'
+
+/** Which shard a consumed message arrived on. */
+const ARRIVAL = Symbol('arrival')
+
 function noop () {}
 
 exports.create = create

package/test/connection.mock.js

@@ -6,13 +6,14 @@ const channel = /** @type {jest.MockedFunction<(sharded?: boolean, index?: numbe
     sharded,
     consume: jest.fn(async () => undefined),
     deliver: jest.fn(async () => undefined),
-    send: jest.fn(async () => undefined),
+    // a sharded channel tells which shard a message goes through, the first one unless told otherwise
+    send: jest.fn(async (_queue, _buffer, _options, via) => { if (sharded) via?.(0) }),
     fire: jest.fn(async () => undefined),
     subscribe: jest.fn(async () => undefined),
     bound: jest.fn(async () => undefined),
     held: jest.fn(async () => undefined),
     publish: jest.fn(async () => undefined),
-    route: jest.fn(async () => undefined),
+    route: jest.fn(async (_exchange, _key, _buffer, _options, via) => { if (sharded) via?.(0) }),
     diagnose: jest.fn(async () => undefined),
     forget: jest.fn(() => undefined),
     seal: jest.fn(async () => undefined),

package/test/connection.test.js

@@ -543,109 +543,47 @@ describe('channel exhaustion', () => {
   })
 })
 
-describe('watchdog', () => {
+describe('silence', () => {
   afterEach(() => {
     jest.useRealTimers()
   })
 
-  it('should destroy silent connection when watchdog expires', async () => {
+  // amqplib reads the socket through a callback rather than 'data' events, and tells a
+  // connection gone silent by its heartbeat
+  it('should keep an idle connection', async () => {
     jest.useFakeTimers()
 
     await connection.open()
 
     const conn = await amqplib.connect.mock.results[0].value
 
-    await jest.advanceTimersByTimeAsync(46_000)
-
-    expect(conn.connection.stream.destroy).toHaveBeenCalled()
-  })
-
-  it('should reconnect after the watchdog destroys a silent connection', async () => {
-    jest.useFakeTimers()
-
-    await connection.open()
-
-    await jest.advanceTimersByTimeAsync(46_000)
-
-    // a socket destroyed without an error is a socket amqplib never reports,
-    // which leaves the connection silently dead instead of recovering
-    expect(amqplib.connect).toHaveBeenCalledTimes(2)
-  })
-
-  it('should not let a replaced socket disarm the watchdog', async () => {
-    jest.useFakeTimers()
-
-    await connection.open()
-
-    const stale = await amqplib.connect.mock.results[0].value
-
-    stale.emit('close', new Error('lost'))
-
-    await jest.advanceTimersByTimeAsync(1)
-
-    const live = await amqplib.connect.mock.results[1].value
-
-    await jest.advanceTimersByTimeAsync(30_000)
-
-    stale.connection.stream.emit('data', Buffer.alloc(0))
-
-    await jest.advanceTimersByTimeAsync(30_000)
-
-    expect(live.connection.stream.destroy).toHaveBeenCalled()
-  })
-
-  it('should reset watchdog on socket data', async () => {
-    jest.useFakeTimers()
-
-    await connection.open()
-
-    const conn = await amqplib.connect.mock.results[0].value
-
-    await jest.advanceTimersByTimeAsync(40_000)
-
-    conn.connection.stream.emit('data', Buffer.alloc(0))
-
-    await jest.advanceTimersByTimeAsync(40_000)
+    await jest.advanceTimersByTimeAsync(600_000)
 
     expect(conn.connection.stream.destroy).not.toHaveBeenCalled()
-
-    await jest.advanceTimersByTimeAsync(10_000)
-
-    expect(conn.connection.stream.destroy).toHaveBeenCalled()
+    expect(amqplib.connect).toHaveBeenCalledTimes(1)
   })
 
-  it('should derive the watchdog from the negotiated heartbeat', async () => {
-    jest.useFakeTimers()
-
-    const connection = new Connection('amqp://localhost?heartbeat=30')
-
+  it('should destroy the socket of a connection lost with an error', async () => {
     await connection.open()
 
     const conn = await amqplib.connect.mock.results[0].value
+    const stream = conn.connection.stream
 
-    // the broker is only expected to say something every 15 seconds
-    await jest.advanceTimersByTimeAsync(60_000)
-
-    expect(conn.connection.stream.destroy).not.toHaveBeenCalled()
-
-    await jest.advanceTimersByTimeAsync(31_000)
+    conn.emit('close', new Error('Heartbeat timeout'))
 
-    expect(conn.connection.stream.destroy).toHaveBeenCalled()
+    expect(stream.destroy).toHaveBeenCalled()
   })
 
-  it('should not arm the watchdog when heartbeats are disabled', async () => {
-    jest.useFakeTimers()
-
-    const connection = new Connection('amqp://localhost?heartbeat=0')
-
+  it('should reconnect once a connection is lost with an error', async () => {
     await connection.open()
 
     const conn = await amqplib.connect.mock.results[0].value
 
-    // an idle connection that has agreed to no heartbeats is silent by design
-    await jest.advanceTimersByTimeAsync(600_000)
+    conn.emit('close', new Error('Heartbeat timeout'))
 
-    expect(conn.connection.stream.destroy).not.toHaveBeenCalled()
+    await timeout(0)
+
+    expect(amqplib.connect).toHaveBeenCalledTimes(2)
   })
 })
 

package/test/io.call.test.js

@@ -52,7 +52,7 @@ describe('call', () => {
 
   it('should publish to the exchange under the key', async () => {
     expect(requests.route).toHaveBeenCalledWith(exchange, key, expect.any(Buffer),
-      expect.objectContaining({ mandatory: true }))
+      expect.objectContaining({ mandatory: true }), expect.any(Function))
   })
 
   it('should declare no queue to publish to', async () => {
@@ -116,14 +116,16 @@ describe('back', () => {
     const contentType = 'application/json'
     const properties = { contentType, correlationId: generate(), replyTo: generate() }
 
-    await consumer({ content: encode(payload, contentType), properties })
+    const request = { content: encode(payload, contentType), properties }
+
+    await consumer(request)
 
     replies = await findChannel('reply')
 
     expect(producer).toHaveBeenCalledWith(payload)
 
     expect(replies.fire).toHaveBeenCalledWith(properties.replyTo, expect.any(Buffer),
-      expect.objectContaining({ correlationId: properties.correlationId }))
+      expect.objectContaining({ correlationId: properties.correlationId }), request)
   })
 })
 

package/test/io.reply.test.js

@@ -98,7 +98,8 @@ describe('reply', () => {
     const reply = await produce.mock.results[0].value
     const buffer = encode(reply, properties.contentType)
 
-    expect(replies.fire).toHaveBeenCalledWith(properties.replyTo, buffer, expect.anything())
+    // the request goes along, so that the reply goes back the way it came
+    expect(replies.fire).toHaveBeenCalledWith(properties.replyTo, buffer, expect.anything(), message)
   })
 
   it('should throw if producer returned undefined', async () => {
@@ -190,7 +191,7 @@ describe('encoding', () => {
       mandatory: true
     }
 
-    expect(replies.fire).toHaveBeenCalledWith(properties.replyTo, reply, props)
+    expect(replies.fire).toHaveBeenCalledWith(properties.replyTo, reply, props, message)
   })
 
   it('should set octet-stream for Buffer reply', async () => {
@@ -215,7 +216,8 @@ describe('encoding', () => {
       .toHaveBeenCalledWith(
         expect.any(String),
         buffer,
-        expect.objectContaining({ contentType: 'application/octet-stream' })
+        expect.objectContaining({ contentType: 'application/octet-stream' }),
+        message
       )
   })