comq 0.24.0 → 0.25.0

package/source/io.js

@@ -2,7 +2,7 @@
 
 const stream = require('node:stream')
 const { setTimeout } = require('node:timers/promises')
-const { Promex } = require('promex')
+const { Promex } = require('./promex')
 const { memo, failsafe, lazy, track } = require('./attributes')
 const { verdictOf, PARK } = require('./verdicts')
 const { Unroutable } = require('./unroutable')
@@ -154,7 +154,7 @@ class IO {
         }
 
         return await this.#ask(queue, payload, terms,
-          (buffer, properties) => this.#requests.send(queue, buffer, properties))
+          (buffer, properties, via) => this.#requests.send(queue, buffer, properties, via))
       }))
 
   // the terms are taken once, so a re-sent call keeps the deadline of the first
@@ -170,7 +170,8 @@ class IO {
       async (exchange, key, payload, terms) => {
         // mandatory: a Request nobody holds the key of is returned rather than dropped
         return await this.#ask(exchange, payload, terms,
-          (buffer, properties) => this.#requests.route(exchange, key, buffer, { ...properties, mandatory: true }))
+          (buffer, properties, via) =>
+            this.#requests.route(exchange, key, buffer, { ...properties, mandatory: true }, via))
       }))
 
   consume = lazy(this, this.#createEventChannel,
@@ -357,12 +358,20 @@ class IO {
     return channel
   }
 
+  /**
+   * On a sharded connection, what is re-sent is what the shard in question holds: the index of
+   * the shard comes with each of the events, and a Request that went through another shard is
+   * answered through that one.
+   */
   #setupRetransmission () {
     if (this.#requests.sharded === true) {
       // a shard leaves the pool when it rejects a publish, and is lost when its
       // connection drops, which leaves an already sent request unanswered
       this.#requests.diagnose('remove', this.#retransmit)
       this.#requests.diagnose('lost', this.#retransmit)
+
+      // a returned call is published on another shard, and is held by that one from then on
+      this.#requests.diagnose('reroute', this.#rerouted)
     } else {
       this.#requests.diagnose('recover', this.#retransmit)
     }
@@ -462,11 +471,33 @@ class IO {
 
     if (signal !== undefined) this.#abandon(request, reply, signal)
 
-    await abortable(publish(buffer, properties), signal)
+    request.publishing = true
+
+    try {
+      await abortable(publish(buffer, properties, (index) => this.#board(request, index)), signal)
+    } finally {
+      request.publishing = false
+    }
+
+    // the shard it went through was lost while it was being published, and it has not failed
+    // over to another since
+    if (request.lost === true) this.#resend(request, reply)
 
     return reply
   }
 
+  /**
+   * Records the shard a Request is published through. It is recorded as the shard is chosen,
+   * rather than once the publish is done, so that a shard lost in between is known to hold it.
+   *
+   * @param {comq.Request} request
+   * @param {number} index
+   */
+  #board (request, index) {
+    request.shard = index
+    request.lost = false
+  }
+
   /**
    * Stops waiting for a Reply once the signal aborts. The Request is forgotten at once, so a
    * retransmission leaves it be, and a Reply arriving later finds nobody waiting for it.
@@ -494,17 +525,41 @@ class IO {
    * @param {comq.amqp.Message} message
    */
   #returned = (message) => {
+    const pending = this.#pendingOf(message)
+
+    if (pending === undefined) return
+
+    const [reply, request] = pending
+
+    request.emitter.off(request.properties.correlationId)
+    this.#pendingReplies.delete(reply)
+    reply.reject(new Unroutable(message.fields.exchange, message.fields.routingKey))
+  }
+
+  /**
+   * A Request one shard returned is published on another.
+   *
+   * @param {comq.amqp.Message} message
+   * @param {number} index
+   */
+  #rerouted = (message, index) => {
+    const pending = this.#pendingOf(message)
+
+    if (pending !== undefined) this.#board(pending[1], index)
+  }
+
+  /**
+   * @param {comq.amqp.Message} message
+   * @return {[Promex, comq.Request] | undefined}
+   */
+  #pendingOf (message) {
     const { correlationId, replyTo } = message.properties
 
     for (const [reply, request] of this.#pendingReplies) {
       if (request.properties.correlationId !== correlationId) continue
       if (request.emitter.queue !== replyTo) continue
 
-      request.emitter.off(correlationId)
-      this.#pendingReplies.delete(reply)
-      reply.reject(new Unroutable(message.fields.exchange, message.fields.routingKey))
-
-      return
+      return [reply, request]
     }
   }
 
@@ -665,24 +720,44 @@ class IO {
     // an unroutable reply must be returned by the broker rather than dropped
     properties.mandatory = true
 
-    return await this.#replies.fire(replyTo, buffer, properties)
+    // back the way the request came: see `fire` of a sharded channel
+    return await this.#replies.fire(replyTo, buffer, properties, request)
   }
 
   #recover (exception) {
     if (exception !== RETRANSMISSION) return false
   }
 
-  #retransmit = () => {
+  /**
+   * Re-sends the Requests awaiting their replies on a shard, or on the connection when no shard
+   * is given. A Request that went through another shard is not re-sent: it is answered through
+   * that shard, and it is not for comq to assume that it is idempotent.
+   *
+   * @param {number} [index]
+   */
+  #retransmit = (index) => {
     for (const [reply, request] of this.#pendingReplies) {
-      // detaching this attempt alone leaves the listeners of the reply streams
-      // that are still flowing over the other shards in place
-      request.emitter.off(request.properties.correlationId)
+      if (index !== undefined && request.shard !== index) continue
 
-      // trigger failsafe attribute
-      reply.reject(RETRANSMISSION)
+      // one still being published is re-sent once it is, unless it fails over to another shard
+      if (request.publishing === true) request.lost = true
+      else this.#resend(request, reply)
     }
   }
 
+  /**
+   * @param {comq.Request} request
+   * @param {Promex} reply
+   */
+  #resend (request, reply) {
+    // detaching this attempt alone leaves the listeners of the reply streams
+    // that are still flowing over the other shards in place
+    request.emitter.off(request.properties.correlationId)
+
+    // trigger failsafe attribute
+    reply.reject(RETRANSMISSION)
+  }
+
   /**
    * @param {any} payload
    * @param {comq.Encoding} [contentType]

package/source/promex.js

@@ -0,0 +1,47 @@
+'use strict'
+
+/**
+ * A promise settled by whoever holds it, rather than only from within its executor.
+ * `callback` settles it the way node-style callbacks report: an error, or a value.
+ *
+ * @template T
+ * @extends {Promise<T>}
+ */
+class Promex extends Promise {
+  /** @type {(value?: T | PromiseLike<T>) => void} */
+  resolve
+
+  /** @type {(reason?: any) => void} */
+  reject
+
+  /**
+   * @param {(resolve: (value?: any) => void, reject: (reason?: any) => void) => void} [executor]
+   */
+  constructor (executor = noop) {
+    let ok
+    let oh
+
+    super((resolve, reject) => {
+      ok = resolve
+      oh = reject
+
+      return executor(resolve, reject)
+    })
+
+    this.resolve = ok
+    this.reject = oh
+  }
+
+  /**
+   * @param {any} error
+   * @param {T} [value]
+   */
+  callback = (error, value) => {
+    if (error !== null && error !== undefined) this.reject(error)
+    else this.resolve(value)
+  }
+}
+
+function noop () {}
+
+exports.Promex = Promex

package/source/promex.test.js

@@ -0,0 +1,79 @@
+'use strict'
+
+const { Promex } = require('./promex')
+
+it('should resolve', async () => {
+  const promise = new Promex()
+
+  setImmediate(() => promise.resolve('ok'))
+
+  await expect(promise).resolves.toStrictEqual('ok')
+})
+
+it('should reject', async () => {
+  const promise = new Promex()
+
+  setImmediate(() => promise.reject(new Error('oh')))
+
+  await expect(promise).rejects.toThrow('oh')
+})
+
+it('should resolve using the callback', async () => {
+  const promise = new Promex()
+
+  setImmediate(() => promise.callback(null, 'ok'))
+
+  await expect(promise).resolves.toStrictEqual('ok')
+})
+
+it('should reject using the callback', async () => {
+  const promise = new Promex()
+
+  setImmediate(() => promise.callback('oh'))
+
+  await expect(promise).rejects.toStrictEqual('oh')
+})
+
+it('should resolve using the callback given no error', async () => {
+  const promise = new Promex()
+
+  setImmediate(() => promise.callback(undefined, 'ok'))
+
+  await expect(promise).resolves.toStrictEqual('ok')
+})
+
+it('should run the executor it is given', async () => {
+  const promise = new Promex((resolve) => resolve('ok'))
+
+  await expect(promise).resolves.toStrictEqual('ok')
+})
+
+it('should stay settled by whoever holds it', async () => {
+  const promise = new Promex()
+
+  promise.resolve('first')
+  promise.resolve('second')
+
+  await expect(promise).resolves.toStrictEqual('first')
+})
+
+it('should be a promise', async () => {
+  const promise = new Promex()
+
+  expect(promise).toBeInstanceOf(Promise)
+
+  promise.resolve('ok')
+
+  await expect(Promise.all([promise])).resolves.toStrictEqual(['ok'])
+})
+
+it('should derive promises of its own kind', async () => {
+  const promise = new Promex()
+  const derived = promise.then((value) => value + '!')
+
+  expect(derived).toBeInstanceOf(Promex)
+
+  promise.resolve('ok')
+
+  await expect(derived).resolves.toStrictEqual('ok!')
+})

package/source/retry.js

@@ -0,0 +1,42 @@
+'use strict'
+
+/**
+ * Runs an operation until it stops asking for another attempt, waiting longer before
+ * each one. The operation is handed the token it returns to ask for a retry; anything
+ * else it returns is the result, and whatever it throws is thrown on.
+ *
+ * @template T
+ * @param {(again: symbol) => Promise<T | symbol>} operation
+ * @param {Partial<typeof DEFAULTS>} [options]
+ * @returns {Promise<T>}
+ */
+async function retry (operation, options) {
+  const { attempts, base, max, factor, dispersion } = { ...DEFAULTS, ...options }
+
+  for (let attempt = 0; attempt < attempts; attempt++) {
+    const result = await operation(again)
+
+    if (result !== again) return /** @type {T} */ (result)
+
+    const interval = Math.min(base * factor ** attempt, max)
+
+    // attempts that started together are spread out rather than kept in step
+    const spread = interval * dispersion * (Math.random() - 0.5)
+
+    await new Promise((resolve) => setTimeout(resolve, interval + spread))
+  }
+
+  throw new Error('Maximum attempts exceeded')
+}
+
+const again = Symbol('again')
+
+const DEFAULTS = {
+  attempts: Infinity,
+  base: 1000,
+  max: 30000,
+  factor: 1.5,
+  dispersion: 0.1
+}
+
+exports.retry = retry

package/source/retry.test.js

@@ -0,0 +1,120 @@
+'use strict'
+
+const { retry } = require('./retry')
+
+const immediate = { base: 0, max: 0, dispersion: 0 }
+
+it('should return the result', async () => {
+  const result = await retry(async () => 'ok', immediate)
+
+  expect(result).toStrictEqual('ok')
+})
+
+it('should retry while the operation asks for it', async () => {
+  let attempts = 0
+
+  const result = await retry(async (again) => {
+    attempts++
+
+    return attempts < 3 ? again : 'ok'
+  }, immediate)
+
+  expect(attempts).toStrictEqual(3)
+  expect(result).toStrictEqual('ok')
+})
+
+it('should pass the same token on every attempt', async () => {
+  const tokens = []
+
+  await retry(async (again) => {
+    tokens.push(again)
+
+    return tokens.length < 3 ? again : 'ok'
+  }, immediate)
+
+  expect(tokens).toHaveLength(3)
+  expect(new Set(tokens).size).toStrictEqual(1)
+})
+
+it('should throw what the operation throws', async () => {
+  const exception = new Error('nope')
+
+  await expect(retry(async () => { throw exception }, immediate)).rejects.toThrow(exception)
+})
+
+it('should give up after the last attempt', async () => {
+  let attempts = 0
+
+  const attempt = retry(async (again) => {
+    attempts++
+
+    return again
+  }, { ...immediate, attempts: 3 })
+
+  await expect(attempt).rejects.toThrow('Maximum attempts exceeded')
+  expect(attempts).toStrictEqual(3)
+})
+
+it('should not run the operation without an attempt to spare', async () => {
+  const operation = jest.fn(async (again) => again)
+
+  await expect(retry(operation, { ...immediate, attempts: 0 }))
+    .rejects.toThrow('Maximum attempts exceeded')
+
+  expect(operation).not.toHaveBeenCalled()
+})
+
+it('should back off, up to the maximum, before each attempt', async () => {
+  jest.useFakeTimers()
+
+  try {
+    const waits = []
+
+    let last = Date.now()
+    let attempts = 0
+
+    const attempt = retry(async (again) => {
+      waits.push(Date.now() - last)
+      last = Date.now()
+      attempts++
+
+      return attempts < 5 ? again : 'ok'
+    }, { base: 100, factor: 2, max: 300, dispersion: 0 })
+
+    await jest.advanceTimersByTimeAsync(10_000)
+
+    expect(await attempt).toStrictEqual('ok')
+
+    // the first attempt waits for nothing; then 100, 200, and 400 capped at 300
+    expect(waits).toStrictEqual([0, 100, 200, 300, 300])
+  } finally {
+    jest.useRealTimers()
+  }
+})
+
+it('should spread the wait around the interval', async () => {
+  const waits = []
+
+  const timer = global.setTimeout
+
+  global.setTimeout = (callback, ms) => {
+    waits.push(ms)
+
+    return timer(callback, 0)
+  }
+
+  try {
+    let attempts = 0
+
+    await retry(async (again) => {
+      attempts++
+
+      return attempts < 20 ? again : 'ok'
+    }, { base: 1000, factor: 1, max: 1000, dispersion: 0.1 })
+  } finally {
+    global.setTimeout = timer
+  }
+
+  expect(waits.every((ms) => ms >= 950 && ms <= 1050)).toStrictEqual(true)
+  expect(new Set(waits).size).toBeGreaterThan(1)
+})

package/source/shards/channel.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { Promex } = require('promex')
+const { Promex } = require('../promex')
 const { RETRY_PREFIX } = require('../channel')
 const events = require('../events')
 const emitter = require('../emitter')
@@ -57,8 +57,12 @@ class Channel {
     await Promise.any(promises)
   }
 
+  /**
+   * A message is told which shard it arrived on, so that whatever answers it can go back the same
+   * way. See `fire`.
+   */
   async consume (queue, consumer) {
-    return await this.#every((channel) => channel.consume(queue, consumer))
+    return await this.#every((channel) => channel.consume(queue, arrival(consumer, channel)))
   }
 
   async subscribe (queue, group, consumer) {
@@ -74,24 +78,51 @@ class Channel {
    * claiming it, and holds it as well once it is let go.
    */
   async held (exchange, queue, key, consumer) {
-    await Promise.any(this.#apply((channel) => channel.held(exchange, queue, key, consumer)))
+    await Promise.any(this.#apply((channel) => channel.held(exchange, queue, key, arrival(consumer, channel))))
   }
 
-  async send (queue, buffer, options) {
-    await this.#one((channel) => channel.send(queue, buffer, options))
+  /**
+   * @param {string} queue
+   * @param {Buffer} buffer
+   * @param {comq.amqp.options.Publish} [options]
+   * @param {(index: number) => void} [via] told which shard the message is published through,
+   * as soon as one is chosen and again whenever the publish fails over to another
+   */
+  async send (queue, buffer, options, via) {
+    await this.#one((channel) => channel.send(queue, buffer, options), { via })
   }
 
   async publish (exchange, buffer, options) {
     await this.#one((channel) => channel.publish(exchange, buffer, options))
   }
 
-  async route (exchange, key, buffer, options) {
-    await this.#one((channel) => channel.route(exchange, key, buffer, options))
+  /**
+   * @param {string} exchange
+   * @param {string} key
+   * @param {Buffer} buffer
+   * @param {comq.amqp.options.Publish} [options]
+   * @param {(index: number) => void} [via] as `send` has it
+   */
+  async route (exchange, key, buffer, options, via) {
+    await this.#one((channel) => channel.route(exchange, key, buffer, options), { via })
   }
 
-  async fire (queue, buffer, options) {
+  /**
+   * A reply goes back through the shard the message it answers arrived on. Whoever sent that
+   * message re-sends what it sent through a shard once the shard is lost, and nothing else, so a
+   * reply taking another way could be lost with a shard that nothing is re-sent for. It takes
+   * another way when that shard is not reachable, which leaves its message unacknowledged there,
+   * to be delivered and answered again.
+   *
+   * @param {string} queue
+   * @param {Buffer} buffer
+   * @param {comq.amqp.options.Publish} [options]
+   * @param {comq.amqp.Message} [origin] the message this one answers
+   * @return {Promise<boolean>}
+   */
+  async fire (queue, buffer, options, origin) {
     // noinspection  JSValidateTypes
-    return await this.#one((channel) => channel.fire(queue, buffer, options))
+    return await this.#one((channel) => channel.fire(queue, buffer, options), { prefer: origin?.[ARRIVAL] })
   }
 
   async close () {
@@ -230,6 +261,9 @@ class Channel {
 
     const next = rest[Math.floor(Math.random() * rest.length)]
 
+    // the message is on that shard now, and is lost with it rather than with this one
+    this.#diagnostics.emit(REROUTE, message, next.index)
+
     // a Request under a key goes through its routed exchange, which `route` declares on that
     // shard before publishing: publishing to an exchange a broker lacks closes the connection
     const retried = exchange === DEFAULT
@@ -374,9 +408,13 @@ class Channel {
   }
 
   /**
+   * Publishes through a shard chosen at random, or through the one preferred while it is
+   * reachable, and fails over to another when that one fails.
+   *
    * @param {(channel: comq.Channel) => void} fn
+   * @param {{ via?: (index: number) => void, prefer?: number }} [route]
    */
-  async #one (fn) {
+  async #one (fn, route = {}) {
     // capture before the check: recover may replace `#recovery` in between
     const waiting = this.#recovery
     const pool = this.#reachable()
@@ -384,21 +422,38 @@ class Channel {
     if (pool.length === 0) {
       await waiting
 
-      return this.#one(fn)
+      return this.#one(fn, route)
     }
 
-    const channel = pool[Math.floor(Math.random() * pool.length)]
+    const channel = pool.find((channel) => channel.index === route.prefer) ??
+      pool[Math.floor(Math.random() * pool.length)]
+
+    // told before the publish is awaited: a shard lost while it is under way has it on board
+    route.via?.(channel.index)
 
     try {
       return await fn(channel)
     } catch {
       this.#remove(channel)
 
-      return this.#one(fn)
+      return this.#one(fn, route)
     }
   }
 }
 
+/**
+ * @param {comq.channels.Consumer} consumer
+ * @param {comq.Channel} channel
+ * @return {comq.channels.Consumer}
+ */
+function arrival (consumer, channel) {
+  return (message) => {
+    message[ARRIVAL] = channel.index
+
+    return consumer(message)
+  }
+}
+
 /**
  * @param {comq.Connection[]} connections
  * @param {comq.topology.type} type
@@ -418,6 +473,12 @@ const REMOVE = 'remove'
 const RETURN_HEADER = 'x-return'
 const LOST = 'lost'
 
+/** A returned message published on another shard, which is where it can be lost from now on. */
+const REROUTE = 'reroute'
+
+/** Which shard a consumed message arrived on. */
+const ARRIVAL = Symbol('arrival')
+
 function noop () {}
 
 exports.create = create

package/test/.types/jest.d.ts

@@ -0,0 +1,22 @@
+import * as mock from 'jest-mock'
+
+// The test files are written against Jest's mock types, and keep being: the mocks are still
+// Jest's, taken as the standalone `jest-mock` package. This is what @types/jest declared of it.
+
+declare global {
+
+  namespace jest {
+
+    type Mock<T extends mock.FunctionLike = mock.UnknownFunction> = mock.Mock<T>
+
+    type Mocked<T extends object> = mock.Mocked<T>
+
+    type MockedClass<T extends mock.ClassLike> = mock.MockedClass<T>
+
+    type MockedFunction<T extends mock.FunctionLike = mock.UnknownFunction> = mock.MockedFunction<T>
+
+    type MockedFn<T extends mock.FunctionLike = mock.UnknownFunction> = mock.MockedFunction<T>
+
+    type MockedObject<T extends object> = mock.MockedObject<T>
+  }
+}