comq 0.23.0 → 0.24.1

package/source/io.js

@@ -84,16 +84,21 @@ class IO {
     })
 
   /**
+   * Waits for the Reply for as long as it takes: a Request is answered eventually, and one no
+   * Producer can answer is parked rather than abandoned.
+   *
    * @param {string} queue
    * @param {any | Readable} payload
-   * @param {comq.Encoding | comq.RequestOptions} [options]
+   * @param {comq.Encoding} [encoding]
    * @returns {Promise<any | Readable>}
    */
-  request (queue, payload, options) {
-    const settled = terms(options)
+  async request (queue, payload, encoding) {
+    // a caller of the version that took a timeout here would otherwise wait on in silence
+    if (typeof encoding === 'object' && encoding !== null) {
+      throw new TypeError('A Request takes an encoding and waits for its Reply')
+    }
 
-    // the channels and the queue a first Request declares are waited for within the terms too
-    return abortable(this.#request(queue, payload, settled), settled.signal)
+    return await this.#request(queue, payload, { encoding })
   }
 
   /**
@@ -103,12 +108,13 @@ class IO {
    * @param {string} exchange
    * @param {string} key
    * @param {any} payload
-   * @param {comq.Encoding | comq.RequestOptions} [options]
+   * @param {comq.Encoding | comq.CallOptions} [options]
    * @returns {Promise<any | Readable>}
    */
   call (exchange, key, payload, options) {
     const settled = terms(options)
 
+    // the channels a first call declares are waited for within the terms too
     return abortable(this.#call(exchange, key, payload, settled), settled.signal)
   }
 
@@ -129,8 +135,7 @@ class IO {
       await this.#requests.held(exchange, exchange + '.' + key, key, consumer)
     })
 
-  // failsafe is aimed to retransmit unanswered messages; the terms are taken once, so a
-  // re-sent Request keeps the deadline of the first
+  // failsafe is aimed to retransmit unanswered messages
   #request = lazy(this, [this.#createRequestReplyChannels, this.#consumeReplies],
     failsafe(this, this.#recover,
       /**
@@ -149,9 +154,10 @@ class IO {
         }
 
         return await this.#ask(queue, payload, terms,
-          (buffer, properties) => this.#requests.send(queue, buffer, properties))
+          (buffer, properties, via) => this.#requests.send(queue, buffer, properties, via))
       }))
 
+  // the terms are taken once, so a re-sent call keeps the deadline of the first
   #call = lazy(this, [this.#createRequestReplyChannels, this.#consumeReplies],
     failsafe(this, this.#recover,
       /**
@@ -164,7 +170,8 @@ class IO {
       async (exchange, key, payload, terms) => {
         // mandatory: a Request nobody holds the key of is returned rather than dropped
         return await this.#ask(exchange, payload, terms,
-          (buffer, properties) => this.#requests.route(exchange, key, buffer, { ...properties, mandatory: true }))
+          (buffer, properties, via) =>
+            this.#requests.route(exchange, key, buffer, { ...properties, mandatory: true }, via))
       }))
 
   consume = lazy(this, this.#createEventChannel,
@@ -351,12 +358,20 @@ class IO {
     return channel
   }
 
+  /**
+   * On a sharded connection, what is re-sent is what the shard in question holds: the index of
+   * the shard comes with each of the events, and a Request that went through another shard is
+   * answered through that one.
+   */
   #setupRetransmission () {
     if (this.#requests.sharded === true) {
       // a shard leaves the pool when it rejects a publish, and is lost when its
       // connection drops, which leaves an already sent request unanswered
       this.#requests.diagnose('remove', this.#retransmit)
       this.#requests.diagnose('lost', this.#retransmit)
+
+      // a returned call is published on another shard, and is held by that one from then on
+      this.#requests.diagnose('reroute', this.#rerouted)
     } else {
       this.#requests.diagnose('recover', this.#retransmit)
     }
@@ -456,11 +471,33 @@ class IO {
 
     if (signal !== undefined) this.#abandon(request, reply, signal)
 
-    await abortable(publish(buffer, properties), signal)
+    request.publishing = true
+
+    try {
+      await abortable(publish(buffer, properties, (index) => this.#board(request, index)), signal)
+    } finally {
+      request.publishing = false
+    }
+
+    // the shard it went through was lost while it was being published, and it has not failed
+    // over to another since
+    if (request.lost === true) this.#resend(request, reply)
 
     return reply
   }
 
+  /**
+   * Records the shard a Request is published through. It is recorded as the shard is chosen,
+   * rather than once the publish is done, so that a shard lost in between is known to hold it.
+   *
+   * @param {comq.Request} request
+   * @param {number} index
+   */
+  #board (request, index) {
+    request.shard = index
+    request.lost = false
+  }
+
   /**
    * Stops waiting for a Reply once the signal aborts. The Request is forgotten at once, so a
    * retransmission leaves it be, and a Reply arriving later finds nobody waiting for it.
@@ -488,17 +525,41 @@ class IO {
    * @param {comq.amqp.Message} message
    */
   #returned = (message) => {
+    const pending = this.#pendingOf(message)
+
+    if (pending === undefined) return
+
+    const [reply, request] = pending
+
+    request.emitter.off(request.properties.correlationId)
+    this.#pendingReplies.delete(reply)
+    reply.reject(new Unroutable(message.fields.exchange, message.fields.routingKey))
+  }
+
+  /**
+   * A Request one shard returned is published on another.
+   *
+   * @param {comq.amqp.Message} message
+   * @param {number} index
+   */
+  #rerouted = (message, index) => {
+    const pending = this.#pendingOf(message)
+
+    if (pending !== undefined) this.#board(pending[1], index)
+  }
+
+  /**
+   * @param {comq.amqp.Message} message
+   * @return {[Promex, comq.Request] | undefined}
+   */
+  #pendingOf (message) {
     const { correlationId, replyTo } = message.properties
 
     for (const [reply, request] of this.#pendingReplies) {
       if (request.properties.correlationId !== correlationId) continue
       if (request.emitter.queue !== replyTo) continue
 
-      request.emitter.off(correlationId)
-      this.#pendingReplies.delete(reply)
-      reply.reject(new Unroutable(message.fields.exchange, message.fields.routingKey))
-
-      return
+      return [reply, request]
     }
   }
 
@@ -659,24 +720,44 @@ class IO {
     // an unroutable reply must be returned by the broker rather than dropped
     properties.mandatory = true
 
-    return await this.#replies.fire(replyTo, buffer, properties)
+    // back the way the request came: see `fire` of a sharded channel
+    return await this.#replies.fire(replyTo, buffer, properties, request)
   }
 
   #recover (exception) {
     if (exception !== RETRANSMISSION) return false
   }
 
-  #retransmit = () => {
+  /**
+   * Re-sends the Requests awaiting their replies on a shard, or on the connection when no shard
+   * is given. A Request that went through another shard is not re-sent: it is answered through
+   * that shard, and it is not for comq to assume that it is idempotent.
+   *
+   * @param {number} [index]
+   */
+  #retransmit = (index) => {
     for (const [reply, request] of this.#pendingReplies) {
-      // detaching this attempt alone leaves the listeners of the reply streams
-      // that are still flowing over the other shards in place
-      request.emitter.off(request.properties.correlationId)
+      if (index !== undefined && request.shard !== index) continue
 
-      // trigger failsafe attribute
-      reply.reject(RETRANSMISSION)
+      // one still being published is re-sent once it is, unless it fails over to another shard
+      if (request.publishing === true) request.lost = true
+      else this.#resend(request, reply)
     }
   }
 
+  /**
+   * @param {comq.Request} request
+   * @param {Promex} reply
+   */
+  #resend (request, reply) {
+    // detaching this attempt alone leaves the listeners of the reply streams
+    // that are still flowing over the other shards in place
+    request.emitter.off(request.properties.correlationId)
+
+    // trigger failsafe attribute
+    reply.reject(RETRANSMISSION)
+  }
+
   /**
    * @param {any} payload
    * @param {comq.Encoding} [contentType]
@@ -708,10 +789,10 @@ const REPLY = 'comq.reply'
 const CONTROL = 'control'
 
 /**
- * What a caller passed, settled once: a timeout becomes the moment the Request expires, so that
+ * What a caller passed, settled once: a timeout becomes the moment the call expires, so that
  * every attempt of it shares one deadline.
  *
- * @param {comq.Encoding | comq.RequestOptions} [options]
+ * @param {comq.Encoding | comq.CallOptions} [options]
  * @returns {comq.Terms}
  */
 function terms (options) {

package/source/shards/channel.js

@@ -57,8 +57,12 @@ class Channel {
     await Promise.any(promises)
   }
 
+  /**
+   * A message is told which shard it arrived on, so that whatever answers it can go back the same
+   * way. See `fire`.
+   */
   async consume (queue, consumer) {
-    return await this.#every((channel) => channel.consume(queue, consumer))
+    return await this.#every((channel) => channel.consume(queue, arrival(consumer, channel)))
   }
 
   async subscribe (queue, group, consumer) {
@@ -74,24 +78,51 @@ class Channel {
    * claiming it, and holds it as well once it is let go.
    */
   async held (exchange, queue, key, consumer) {
-    await Promise.any(this.#apply((channel) => channel.held(exchange, queue, key, consumer)))
+    await Promise.any(this.#apply((channel) => channel.held(exchange, queue, key, arrival(consumer, channel))))
   }
 
-  async send (queue, buffer, options) {
-    await this.#one((channel) => channel.send(queue, buffer, options))
+  /**
+   * @param {string} queue
+   * @param {Buffer} buffer
+   * @param {comq.amqp.options.Publish} [options]
+   * @param {(index: number) => void} [via] told which shard the message is published through,
+   * as soon as one is chosen and again whenever the publish fails over to another
+   */
+  async send (queue, buffer, options, via) {
+    await this.#one((channel) => channel.send(queue, buffer, options), { via })
   }
 
   async publish (exchange, buffer, options) {
     await this.#one((channel) => channel.publish(exchange, buffer, options))
   }
 
-  async route (exchange, key, buffer, options) {
-    await this.#one((channel) => channel.route(exchange, key, buffer, options))
+  /**
+   * @param {string} exchange
+   * @param {string} key
+   * @param {Buffer} buffer
+   * @param {comq.amqp.options.Publish} [options]
+   * @param {(index: number) => void} [via] as `send` has it
+   */
+  async route (exchange, key, buffer, options, via) {
+    await this.#one((channel) => channel.route(exchange, key, buffer, options), { via })
   }
 
-  async fire (queue, buffer, options) {
+  /**
+   * A reply goes back through the shard the message it answers arrived on. Whoever sent that
+   * message re-sends what it sent through a shard once the shard is lost, and nothing else, so a
+   * reply taking another way could be lost with a shard that nothing is re-sent for. It takes
+   * another way when that shard is not reachable, which leaves its message unacknowledged there,
+   * to be delivered and answered again.
+   *
+   * @param {string} queue
+   * @param {Buffer} buffer
+   * @param {comq.amqp.options.Publish} [options]
+   * @param {comq.amqp.Message} [origin] the message this one answers
+   * @return {Promise<boolean>}
+   */
+  async fire (queue, buffer, options, origin) {
     // noinspection  JSValidateTypes
-    return await this.#one((channel) => channel.fire(queue, buffer, options))
+    return await this.#one((channel) => channel.fire(queue, buffer, options), { prefer: origin?.[ARRIVAL] })
   }
 
   async close () {
@@ -230,6 +261,9 @@ class Channel {
 
     const next = rest[Math.floor(Math.random() * rest.length)]
 
+    // the message is on that shard now, and is lost with it rather than with this one
+    this.#diagnostics.emit(REROUTE, message, next.index)
+
     // a Request under a key goes through its routed exchange, which `route` declares on that
     // shard before publishing: publishing to an exchange a broker lacks closes the connection
     const retried = exchange === DEFAULT
@@ -374,9 +408,13 @@ class Channel {
   }
 
   /**
+   * Publishes through a shard chosen at random, or through the one preferred while it is
+   * reachable, and fails over to another when that one fails.
+   *
    * @param {(channel: comq.Channel) => void} fn
+   * @param {{ via?: (index: number) => void, prefer?: number }} [route]
    */
-  async #one (fn) {
+  async #one (fn, route = {}) {
     // capture before the check: recover may replace `#recovery` in between
     const waiting = this.#recovery
     const pool = this.#reachable()
@@ -384,21 +422,38 @@ class Channel {
     if (pool.length === 0) {
       await waiting
 
-      return this.#one(fn)
+      return this.#one(fn, route)
     }
 
-    const channel = pool[Math.floor(Math.random() * pool.length)]
+    const channel = pool.find((channel) => channel.index === route.prefer) ??
+      pool[Math.floor(Math.random() * pool.length)]
+
+    // told before the publish is awaited: a shard lost while it is under way has it on board
+    route.via?.(channel.index)
 
     try {
       return await fn(channel)
     } catch {
       this.#remove(channel)
 
-      return this.#one(fn)
+      return this.#one(fn, route)
     }
   }
 }
 
+/**
+ * @param {comq.channels.Consumer} consumer
+ * @param {comq.Channel} channel
+ * @return {comq.channels.Consumer}
+ */
+function arrival (consumer, channel) {
+  return (message) => {
+    message[ARRIVAL] = channel.index
+
+    return consumer(message)
+  }
+}
+
 /**
  * @param {comq.Connection[]} connections
  * @param {comq.topology.type} type
@@ -418,6 +473,12 @@ const REMOVE = 'remove'
 const RETURN_HEADER = 'x-return'
 const LOST = 'lost'
 
+/** A returned message published on another shard, which is where it can be lost from now on. */
+const REROUTE = 'reroute'
+
+/** Which shard a consumed message arrived on. */
+const ARRIVAL = Symbol('arrival')
+
 function noop () {}
 
 exports.create = create

package/test/amqplib.mock.js

@@ -58,7 +58,7 @@ class Connection extends EventEmitter {
 
 const connect = async (url) => new Connection(url)
 
-/** @type {jest.MockedObject<import('amqplib')>} */
+/** @type {jest.MockedObject<import('@toa.io/amqplib')>} */
 const amqplib = {
   connect: jest.fn(connect)
 }

package/test/connection.mock.js

@@ -6,13 +6,14 @@ const channel = /** @type {jest.MockedFunction<(sharded?: boolean, index?: numbe
     sharded,
     consume: jest.fn(async () => undefined),
     deliver: jest.fn(async () => undefined),
-    send: jest.fn(async () => undefined),
+    // a sharded channel tells which shard a message goes through, the first one unless told otherwise
+    send: jest.fn(async (_queue, _buffer, _options, via) => { if (sharded) via?.(0) }),
     fire: jest.fn(async () => undefined),
     subscribe: jest.fn(async () => undefined),
     bound: jest.fn(async () => undefined),
     held: jest.fn(async () => undefined),
     publish: jest.fn(async () => undefined),
-    route: jest.fn(async () => undefined),
+    route: jest.fn(async (_exchange, _key, _buffer, _options, via) => { if (sharded) via?.(0) }),
     diagnose: jest.fn(async () => undefined),
     forget: jest.fn(() => undefined),
     seal: jest.fn(async () => undefined),

package/test/connection.test.js

@@ -10,7 +10,7 @@ const { amqplib, connect } = require('./amqplib.mock')
 const { channel: create } = require('./connection.mock')
 const mock = { amqplib, channel: { create } }
 
-jest.mock('amqplib', () => mock.amqplib)
+jest.mock('@toa.io/amqplib', () => mock.amqplib)
 jest.mock('../source/channel', () => mock.channel)
 
 const presets = require('../source/topology')
@@ -543,109 +543,47 @@ describe('channel exhaustion', () => {
   })
 })
 
-describe('watchdog', () => {
+describe('silence', () => {
   afterEach(() => {
     jest.useRealTimers()
   })
 
-  it('should destroy silent connection when watchdog expires', async () => {
+  // amqplib reads the socket through a callback rather than 'data' events, and tells a
+  // connection gone silent by its heartbeat
+  it('should keep an idle connection', async () => {
     jest.useFakeTimers()
 
     await connection.open()
 
     const conn = await amqplib.connect.mock.results[0].value
 
-    await jest.advanceTimersByTimeAsync(46_000)
-
-    expect(conn.connection.stream.destroy).toHaveBeenCalled()
-  })
-
-  it('should reconnect after the watchdog destroys a silent connection', async () => {
-    jest.useFakeTimers()
-
-    await connection.open()
-
-    await jest.advanceTimersByTimeAsync(46_000)
-
-    // a socket destroyed without an error is a socket amqplib never reports,
-    // which leaves the connection silently dead instead of recovering
-    expect(amqplib.connect).toHaveBeenCalledTimes(2)
-  })
-
-  it('should not let a replaced socket disarm the watchdog', async () => {
-    jest.useFakeTimers()
-
-    await connection.open()
-
-    const stale = await amqplib.connect.mock.results[0].value
-
-    stale.emit('close', new Error('lost'))
-
-    await jest.advanceTimersByTimeAsync(1)
-
-    const live = await amqplib.connect.mock.results[1].value
-
-    await jest.advanceTimersByTimeAsync(30_000)
-
-    stale.connection.stream.emit('data', Buffer.alloc(0))
-
-    await jest.advanceTimersByTimeAsync(30_000)
-
-    expect(live.connection.stream.destroy).toHaveBeenCalled()
-  })
-
-  it('should reset watchdog on socket data', async () => {
-    jest.useFakeTimers()
-
-    await connection.open()
-
-    const conn = await amqplib.connect.mock.results[0].value
-
-    await jest.advanceTimersByTimeAsync(40_000)
-
-    conn.connection.stream.emit('data', Buffer.alloc(0))
-
-    await jest.advanceTimersByTimeAsync(40_000)
+    await jest.advanceTimersByTimeAsync(600_000)
 
     expect(conn.connection.stream.destroy).not.toHaveBeenCalled()
-
-    await jest.advanceTimersByTimeAsync(10_000)
-
-    expect(conn.connection.stream.destroy).toHaveBeenCalled()
+    expect(amqplib.connect).toHaveBeenCalledTimes(1)
   })
 
-  it('should derive the watchdog from the negotiated heartbeat', async () => {
-    jest.useFakeTimers()
-
-    const connection = new Connection('amqp://localhost?heartbeat=30')
-
+  it('should destroy the socket of a connection lost with an error', async () => {
     await connection.open()
 
     const conn = await amqplib.connect.mock.results[0].value
+    const stream = conn.connection.stream
 
-    // the broker is only expected to say something every 15 seconds
-    await jest.advanceTimersByTimeAsync(60_000)
-
-    expect(conn.connection.stream.destroy).not.toHaveBeenCalled()
-
-    await jest.advanceTimersByTimeAsync(31_000)
+    conn.emit('close', new Error('Heartbeat timeout'))
 
-    expect(conn.connection.stream.destroy).toHaveBeenCalled()
+    expect(stream.destroy).toHaveBeenCalled()
   })
 
-  it('should not arm the watchdog when heartbeats are disabled', async () => {
-    jest.useFakeTimers()
-
-    const connection = new Connection('amqp://localhost?heartbeat=0')
-
+  it('should reconnect once a connection is lost with an error', async () => {
     await connection.open()
 
     const conn = await amqplib.connect.mock.results[0].value
 
-    // an idle connection that has agreed to no heartbeats is silent by design
-    await jest.advanceTimersByTimeAsync(600_000)
+    conn.emit('close', new Error('Heartbeat timeout'))
 
-    expect(conn.connection.stream.destroy).not.toHaveBeenCalled()
+    await timeout(0)
+
+    expect(amqplib.connect).toHaveBeenCalledTimes(2)
   })
 })