comq 0.17.1 → 0.18.0

package/source/channel.js

@@ -25,6 +25,14 @@ class Channel {
   /** @type {string[]} */
   #tags = []
 
+  /**
+   * How each queue was declared, so that the queue holding what it could not
+   * process is declared to live exactly as long as it does.
+   *
+   * @type {Map<string, comq.amqp.options.Queue>}
+   */
+  #queues = new Map()
+
   /** @type {Promex | null} */
   #paused = null
 
@@ -71,6 +79,9 @@ class Channel {
     // the consumers of the previous channel went down with it, their tags mean nothing here
     this.#tags = []
 
+    // a fresh channel has declared nothing
+    this.#queues.clear()
+
     await this.#channel.prefetch(this.#topology.prefetch)
 
     this.#channel.on('drain', this.#unpause)
@@ -249,6 +260,8 @@ class Channel {
 
     const { queue } = await this.#channel.assertQueue(name, options)
 
+    this.#queues.set(queue, options)
+
     return [queue]
   }
 
@@ -308,6 +321,80 @@ class Channel {
     return [exchange, queue]
   }
 
+  /**
+   * The queue a failed message waits in before it is delivered again. It has no consumer:
+   * the broker holds the message for the delay and then dead letters it, and because no
+   * routing key is configured the message keeps its own, which names the queue it came
+   * from. One queue serves every source queue that shares the delay.
+   *
+   * It is published to through a fanout exchange rather than directly. Published directly,
+   * the message's routing key would be this queue's own name, and on expiry the broker
+   * would route it back here — a cycle it resolves by dropping the message at the first
+   * expiry, silently.
+   *
+   * @returns {Promise<void>}
+   */
+  async #assertRetryQueue () {
+    if (!this.#topology.acknowledgments) return
+
+    // one queue per distinct wait, declared here rather than when a message first fails:
+    // asserting a queue from inside the failure handler is the least likely moment for it
+    // to succeed, and a failure there would spin the message without ever delaying it
+    for (const delay of new Set(this.#delays)) {
+      const name = RETRY_PREFIX + delay
+
+      await this.#channel.assertExchange(name, 'fanout', DURABLE)
+
+      await this.#assertQueue(name, {
+        ...DURABLE,
+        arguments: {
+          'x-message-ttl': delay,
+          'x-dead-letter-exchange': DEFAULT
+        }
+      })
+
+      await this.#channel.bindQueue(name, name, DEFAULT)
+    }
+  }
+
+  /**
+   * The queue a message is kept in once it has run out of attempts. It has no consumer
+   * either: what is in it is waiting for a person.
+   *
+   * It is declared to live exactly as long as the queue it serves, which for a groupless
+   * subscriber means an exclusive queue whose contents go when the connection does. That
+   * is deliberate: such a subscriber is ephemeral by construction, and a durable queue per
+   * generated name would leak one per restart, forever. An ephemeral subscriber's failures
+   * are ephemeral too.
+   *
+   * @param {string} queue the queue the message was consumed from
+   * @returns {Promise<void>}
+   */
+  async #assertParkedQueue (queue) {
+    if (!this.#topology.acknowledgments) return
+
+    const options = this.#queues.get(queue) ?? (this.#topology.durable ? DURABLE : EXCLUSIVE)
+
+    await this.#assertQueue(parkedQueueOf(queue), options)
+  }
+
+  /** The waits between attempts, as a ladder even when it is one rung. */
+  get #delays () {
+    const delay = this.#topology.delay
+
+    return Array.isArray(delay) ? delay : [delay]
+  }
+
+  /**
+   * The queue a message waits in after its nth attempt, and the exchange it is published
+   * through: they share a name.
+   *
+   * @param {number} attempt the attempt that just failed, counting from one
+   */
+  #retryQueueOf (attempt) {
+    return RETRY_PREFIX + this.#delays[attempt - 1]
+  }
+
   // endregion
 
   /**
@@ -345,69 +432,145 @@ class Channel {
   }
 
   /**
-   * @param {string} queue
-   * @param {comq.channels.Consumer} consumer
-   * @returns {Promise<string>}
+   * The retry topology is asserted once per channel and a parked queue once per queue:
+   * `lazy` keys an initializer by the arguments it takes, and both again after a recovery.
    */
-  async #consume (queue, consumer) {
-    /** @type {comq.amqp.options.Consume} */
-    const options = {}
+  #consume = lazy(this, [this.#assertRetryQueue, this.#assertParkedQueue],
+    /**
+     * @param {string} queue
+     * @param {comq.channels.Consumer} consumer
+     * @returns {Promise<string>}
+     */
+    async (queue, consumer) => {
+      /** @type {comq.amqp.options.Consume} */
+      const options = {}
 
-    if (this.#topology.acknowledgments) consumer = this.#getAcknowledgingConsumer(consumer)
-    else options.noAck = true
+      if (this.#topology.acknowledgments) consumer = this.#getAcknowledgingConsumer(queue, consumer)
+      else options.noAck = true
 
-    const response = await this.#channel.consume(queue, consumer, options)
+      const response = await this.#channel.consume(queue, consumer, options)
 
-    this.#tags.push(response.consumerTag)
+      this.#tags.push(response.consumerTag)
 
-    return response.consumerTag
-  }
+      return response.consumerTag
+    })
 
   /**
+   * @param {string} queue the queue being consumed
    * @param {comq.channels.Consumer} consumer
    * @returns {comq.channels.Consumer}
    */
-  #getAcknowledgingConsumer = (consumer) =>
+  #getAcknowledgingConsumer = (queue, consumer) =>
     async (message) => {
       try {
         await consumer(message)
 
         this.#channel.ack(message)
       } catch (exception) {
-        if (exception.message === 'Channel closed') { return } // the message is requeued by the broker
+        if (exception?.message === 'Channel closed') { return } // the message is requeued by the broker
 
-        const redeliveries = message.properties.headers[REDELIVERY_HEADER] ?? 0
-
-        if (redeliveries >= MAX_REDELIVERIES) this.#discard(message, exception)
-        else {
-          await this.#requeue(message, redeliveries)
-          throw exception
-        }
+        await this.#failed(queue, message, exception)
       }
     }
 
   /**
+   * A message its consumer could not handle is delayed and given another attempt, and
+   * kept once it has had enough of them.
+   *
+   * Nothing here may reject. amqplib dispatches a delivery through an event emitter and
+   * drops the promise it gets back, so a rejection has nobody to catch it and ends the
+   * process. What cannot be carried out is handed back to the broker instead.
+   *
+   * @param {string} queue
+   * @param {comq.amqp.Message} message
+   * @param {Error} exception
+   */
+  async #failed (queue, message, exception) {
+    try {
+      // the header is which attempt this delivery is, and the first does not carry one
+      const attempt = message.properties.headers?.[ATTEMPT_HEADER] ?? 1
+
+      // one rung per retry: a message that has climbed the ladder has nowhere left to wait
+      if (attempt > this.#delays.length) await this.#park(queue, message, exception)
+      else await this.#retry(queue, message, attempt, exception)
+    } catch {
+      // the message could not be moved: give the delivery back rather than lose it
+      this.#return(message)
+    }
+  }
+
+  /**
+   * @param {string} queue
    * @param {comq.amqp.Message} message
    * @param {number} attempt
+   * @param {Error} exception
    */
-  async #requeue (message, attempt) {
-    this.#channel.ack(message)
+  async #retry (queue, message, attempt, exception) {
+    const properties = this.#carry(message, { [ATTEMPT_HEADER]: attempt + 1 })
 
-    message.properties.headers[REDELIVERY_HEADER] = attempt + 1
+    // the copy is placed before the original is released: a message the broker holds
+    // twice can be recovered, one it no longer holds at all cannot
+    await this.#publish(this.#retryQueueOf(attempt), queue, message.content, properties)
 
-    await this.seal()
-    await this.#publish(message.fields.exchange, message.fields.routingKey, message.content, message.properties)
+    this.#channel.ack(message)
+
+    // the attempt that just failed, rather than the one it is about to get
+    this.#diagnostics.emit('retry', message, exception, attempt)
   }
 
   /**
+   * @param {string} queue
    * @param {comq.amqp.Message} message
    * @param {Error} [exception]
    */
-  #discard (message, exception) {
-    this.#channel.nack(message, false, false)
+  async #park (queue, message, exception) {
+    const properties = this.#carry(message, {
+      [PARKED_QUEUE_HEADER]: queue,
+      [PARKED_REASON_HEADER]: exception?.message,
+      [PARKED_AT_HEADER]: Date.now()
+    })
+
+    await this.#publish(DEFAULT, parkedQueueOf(queue), message.content, properties)
+
+    this.#channel.ack(message)
+
     this.#diagnostics.emit('discard', message, exception)
   }
 
+  /**
+   * The properties a copy of a failed message is published with.
+   *
+   * The exchange and routing key are recorded on the first failure and carried from
+   * then on: a message that has been through the retry queue comes back through the
+   * default exchange, so by the time it is parked its own fields describe that hop
+   * rather than where it was published.
+   *
+   * It is published persistent whatever the channel is: Requests are transient for the
+   * sake of latency, and the path a failed message takes is not the one latency is on.
+   *
+   * @param {comq.amqp.Message} message
+   * @param {object} added
+   * @returns {comq.amqp.options.Publish}
+   */
+  #carry (message, added) {
+    const headers = {
+      [ORIGIN_EXCHANGE_HEADER]: message.fields?.exchange,
+      [ORIGIN_KEY_HEADER]: message.fields?.routingKey,
+      ...message.properties.headers,
+      ...added
+    }
+
+    return { ...message.properties, headers, persistent: true, mandatory: true }
+  }
+
+  /**
+   * @param {comq.amqp.Message} message
+   */
+  #return (message) {
+    // a channel that is already gone requeues what it held anyway
+    try { this.#channel.nack(message, false, true) } catch { /* nothing left to do with it */ }
+  }
+
   #pause () {
     if (this.#paused !== null) return
 
@@ -478,8 +641,25 @@ const EXCLUSIVE = { exclusive: true }
 
 const INTERRUPTION = /** @type {Error} */ Symbol('internal interruption')
 
-const MAX_REDELIVERIES = 5
-const REDELIVERY_HEADER = 'x-attempt'
+const RETRY_PREFIX = 'comq.retry.'
+const PARKED_PREFIX = 'comq.parked.'
+
+const parkedQueueOf = (queue) => PARKED_PREFIX + queue
+
+// Everything comq writes onto a message, under its own prefix: AMQP defines no retry
+// counter, so this one is comq's invention rather than a convention, and an unprefixed
+// name would be free to collide with the application's own headers or another library's.
+//
+// What a person looking at a parked message needs, and nothing else. The broker's own
+// `x-death` is no substitute: it names the retry queue rather than where the message
+// came from, and its `count` is a second counter that agrees with this one until it
+// does not.
+const ATTEMPT_HEADER = 'x-comq-attempt'
+const ORIGIN_EXCHANGE_HEADER = 'x-comq-exchange'
+const ORIGIN_KEY_HEADER = 'x-comq-key'
+const PARKED_QUEUE_HEADER = 'x-comq-queue'
+const PARKED_REASON_HEADER = 'x-comq-reason'
+const PARKED_AT_HEADER = 'x-comq-at'
 
 function noop () {}
 

package/source/connect.js

@@ -6,45 +6,55 @@ const { SingletonConnection } = require('./singleton')
 const shards = require('./shards')
 
 /** @type {comq.Connect} */
-const connect = async (...urls) => {
-  return create(urls, connectionConstructor(Connection))
+const connect = async (...args) => {
+  return create(args, Connection)
 }
 
 /** @type {comq.Connect} */
-const assert = async (...urls) => {
-  return create(urls, connectionConstructor(SingletonConnection))
+const assert = async (...args) => {
+  return create(args, SingletonConnection)
 }
 
 /**
- * @param {string[]} urls
- * @param {(urls: string[]) => comq.Connection} constructor
+ * The urls are variadic, so the overrides are the trailing argument when there is one
+ * that is not a url.
+ *
+ * @param {(string | comq.topology.Overrides)[]} args
+ * @return {[string[], comq.topology.Overrides]}
+ */
+const split = (args) => {
+  const last = args[args.length - 1]
+
+  const urls = /** @type {string[]} */ (typeof last === 'object' ? args.slice(0, -1) : args)
+  const overrides = /** @type {comq.topology.Overrides} */ (typeof last === 'object' ? last : {})
+
+  return [urls, overrides]
+}
+
+/**
+ * @param {(string | comq.topology.Overrides)[]} args
+ * @param {new (url: string, overrides: comq.topology.Overrides) => comq.Connection} ConnectionClass
  * @return {Promise<IO>}
  */
-const create = async (urls, constructor) => {
-  const connection = constructor(urls)
+const create = async (args, ConnectionClass) => {
+  const [urls, overrides] = split(args)
+  const connection = connectionOf(urls, overrides, ConnectionClass)
 
   await connection.open()
 
   return new IO(connection)
 }
 
-const connectionConstructor = (ConnectionClass) =>
-  /**
-   * @param {string[]} urls
-   * @return {comq.Connection}
-   */
-  (urls) => {
-    if (urls.length === 1) return new ConnectionClass(urls[0])
-    else return shardedConnection(urls, ConnectionClass)
-  }
-
 /**
  * @param {string[]} urls
- * @param {function} ConnectionClass
+ * @param {comq.topology.Overrides} overrides
+ * @param {Function} ConnectionClass
  * @return {comq.Connection}
  */
-const shardedConnection = (urls, ConnectionClass) => {
-  const connections = urls.map((url) => new ConnectionClass(url))
+const connectionOf = (urls, overrides, ConnectionClass) => {
+  const connections = urls.map((url) => new ConnectionClass(url, overrides))
+
+  if (connections.length === 1) return connections[0]
 
   return new shards.Connection(connections)
 }

package/source/connection.js

@@ -38,6 +38,9 @@ class Connection {
   /** @type {NodeJS.Timeout | null} */
   #heartbeatTimer = null
 
+  /** @type {comq.topology.Overrides} */
+  #overrides
+
   /** @type {import('node:net').Socket | null} */
   #heartbeatSocket = null
 
@@ -49,8 +52,13 @@ class Connection {
   /**
    * @param {string} url
    */
-  constructor (url) {
+  /**
+   * @param {string} url
+   * @param {comq.topology.Overrides} [overrides] per channel type, merged over the presets
+   */
+  constructor (url, overrides = {}) {
     this.#url = heartbeaten(url)
+    this.#overrides = overrides
   }
 
   get connected () {
@@ -89,7 +97,8 @@ class Connection {
     async (type, index) => {
       if (this.#connection === undefined) await this.#recovery
 
-      const topology = presets[type]
+      // a copy: the presets are shared by every connection in the process
+      const topology = { ...presets[type], ...this.#overrides[type] }
 
       // a closed channel held here would hold its IO, and a shared connection is
       // in no hurry to make another channel that would have swept it out

package/source/events.js

@@ -4,4 +4,4 @@
 exports.connection = ['open', 'close', 'error', 'reconnect', 'exhausted']
 
 /** @type {comq.diagnostics.Event[]} */
-exports.channel = ['flow', 'drain', 'recover', 'discard', 'pause', 'resume', 'return', 'lost', 'remove']
+exports.channel = ['flow', 'drain', 'recover', 'discard', 'retry', 'pause', 'resume', 'return', 'lost', 'remove']

package/source/singleton.js

@@ -9,12 +9,32 @@ class SingletonConnection extends Connection {
   /** @type {Promise<void>} */
   #opened = null
 
-  constructor (url) {
-    if (instances.has(url)) return instances.get(url)
-
-    super(url)
+  /** @type {comq.topology.Overrides} */
+  #overrides
+
+  /**
+   * An instance is shared, so its topology is whatever the first caller asked for.
+   * A later caller asking for something else would get a connection retrying on a
+   * ladder it did not choose, which is expensive to debug and cheap to refuse.
+   *
+   * @param {string} url
+   * @param {comq.topology.Overrides} [overrides]
+   */
+  constructor (url, overrides = {}) {
+    const instance = instances.get(url)
+
+    if (instance !== undefined) {
+      if (!same(instance.#overrides, overrides)) {
+        throw new Error(`Connection to ${url} has already been asserted with a different topology`)
+      }
+
+      return instance
+    }
+
+    super(url, overrides)
 
     this.#url = url
+    this.#overrides = overrides
 
     instances.set(url, this)
   }
@@ -65,4 +85,27 @@ const instances = new Map()
 /** @type {Map<string, number>} */
 const counters = new Map()
 
+/**
+ * Topology overrides are a couple of small plain objects, so this is enough and
+ * does not pull in a dependency for it.
+ *
+ * @param {object} one
+ * @param {object} another
+ * @returns {boolean}
+ */
+function same (one, another) {
+  return JSON.stringify(sorted(one)) === JSON.stringify(sorted(another))
+}
+
+/**
+ * @param {any} value
+ * @returns {any} the value with every object's keys in a stable order
+ */
+function sorted (value) {
+  if (Array.isArray(value)) return value.map(sorted)
+  if (value === null || typeof value !== 'object') return value
+
+  return Object.fromEntries(Object.keys(value).sort().map((key) => [key, sorted(value[key])]))
+}
+
 exports.SingletonConnection = SingletonConnection

package/source/topology/event.json

@@ -3,5 +3,11 @@
   "confirms": true,
   "durable": true,
   "acknowledgments": true,
-  "persistent": true
+  "persistent": true,
+  "delay": [
+    1000,
+    10000,
+    30000,
+    90000
+  ]
 }

package/source/topology/request.json

@@ -3,5 +3,11 @@
   "confirms": false,
   "durable": true,
   "acknowledgments": true,
-  "persistent": false
+  "persistent": false,
+  "delay": [
+    1000,
+    3000,
+    5000,
+    10000
+  ]
 }

package/test/assert.test.js

@@ -39,7 +39,7 @@ it('should return instance of IO', async () => {
 
 it('should create singleton connection', async () => {
   expect(SingletonConnection).toHaveBeenCalled()
-  expect(SingletonConnection).toHaveBeenCalledWith(url)
+  expect(SingletonConnection).toHaveBeenCalledWith(url, {})
   expect(IO).toHaveBeenCalledWith(SingletonConnection.mock.instances[0])
 })
 
@@ -50,7 +50,7 @@ it('should create sharded connection', async () => {
 
   io = await assert(...urls)
 
-  for (const url of urls) expect(SingletonConnection).toHaveBeenCalledWith(url)
+  for (const url of urls) expect(SingletonConnection).toHaveBeenCalledWith(url, {})
 
   expect(shards.Connection).toHaveBeenCalledWith(SingletonConnection.mock.instances)
   expect(IO).toHaveBeenCalledWith(shards.Connection.mock.instances[0])

package/test/channel.fixtures.js

@@ -7,7 +7,8 @@ const preset = () => ({
   confirms: flip(),
   durable: flip(),
   acknowledgments: flip(),
-  persistent: flip()
+  persistent: flip(),
+  delay: [1000, 1000, 1000, 1000]
 })
 
 exports.preset = preset

package/test/channel.routed.test.js

@@ -93,7 +93,7 @@ describe('bound', () => {
 
     const { queue: asserted } = await chan.assertQueue.mock.results[0].value
 
-    expect(chan.bindQueue).toHaveBeenCalledTimes(1)
+    expect(bindings()).toHaveLength(1)
     expect(chan.bindQueue).toHaveBeenCalledWith(asserted, exchange, key)
   })
 
@@ -107,13 +107,17 @@ describe('bound', () => {
     await channel.bound(exchange, queue, key, consumer)
     await channel.bound(exchange, queue, key, consumer)
 
-    expect(chan.bindQueue).toHaveBeenCalledTimes(1)
+    expect(bindings()).toHaveLength(1)
   })
 
   it('should bind again for another key', async () => {
     await channel.bound(exchange, queue, key, consumer)
     await channel.bound(exchange, generate(), generate(), consumer)
 
-    expect(chan.bindQueue).toHaveBeenCalledTimes(2)
+    expect(bindings()).toHaveLength(2)
   })
+
+  /** The bindings of the caller, less the one comq makes for its own retry queue. */
+  const bindings = () =>
+    chan.bindQueue.mock.calls.filter(([, exchange]) => !exchange.startsWith('comq.'))
 })