comq 0.16.2 → 0.17.1

package/features/events.routed.feature

@@ -0,0 +1,25 @@
+Feature: Routed events
+
+  An exchange that routes by key, where an event exchange fans out: a message reaches the
+  queues bound under the key it was published with, and none of the others.
+
+  Background:
+    Given an active connection to the broker
+
+  Scenario: Receiving an event by its routing key
+    Given that `orders` is bound to the `records` exchange under the `placed` key
+    When a message is routed to the `records` exchange under the `placed` key
+    Then `orders` receives the event
+
+  Scenario: Not receiving an event bound under another key
+    Given that `orders` is bound to the `records` exchange under the `placed` key
+    And `customers` is bound to the `records` exchange under the `shipped` key
+    When a message is routed to the `records` exchange under the `placed` key
+    Then `orders` receives the event
+    And `customers` receives nothing
+
+  Scenario: Routing over a sharded connection
+    Given an active sharded connection
+    And that `orders` is bound to the `records` exchange under the `placed` key
+    When a message is routed to the `records` exchange under the `placed` key
+    Then `orders` receives the event

package/features/steps/events.routed.js

@@ -0,0 +1,55 @@
+'use strict'
+
+const assert = require('node:assert')
+const { randomBytes } = require('node:crypto')
+const { timeout } = require('../../test/helpers')
+const { Given, When, Then } = require('@cucumber/cucumber')
+
+Given('(that ){token} is bound to the {token} exchange under the {token} key',
+  /**
+   * @param {string} queue
+   * @param {string} exchange
+   * @param {string} key
+   * @this {comq.features.Context}
+   */
+  async function (queue, exchange, key) {
+    await subscribe.call(this, queue, exchange, key)
+  })
+
+When('a message is routed to the {token} exchange under the {token} key',
+  /**
+   * @param {string} exchange
+   * @param {string} key
+   * @this {comq.features.Context}
+   */
+  async function (exchange, key) {
+    const message = randomBytes(8)
+
+    await this.io.route(exchange, key, message)
+
+    this.published = message
+  })
+
+Then('{token} receives nothing',
+  /**
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  async function (queue) {
+    await timeout(100) // let it not consume
+
+    assert.equal(this.consumed[queue], undefined, `'${queue}' has received an event`)
+  })
+
+/**
+ * @param {string} queue
+ * @param {string} exchange
+ * @param {string} key
+ * @this {comq.features.Context}
+ */
+async function subscribe (queue, exchange, key) {
+  await this.io.subscribe(exchange, queue, key, async (payload, properties) => {
+    this.consumed[queue] = { payload, properties }
+    this.eventsConsumedCount++
+  })
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.16.2",
+  "version": "0.17.1",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",

package/readme.md

@@ -7,7 +7,7 @@ for distributed, eventually consistent systems running on Node.js.
 
 - [Dynamic topology](#topology)
 - [Request](#request)-[reply](#reply) (RPC)
-- Events ([pub](#emission)/[sub](#consumption))
+- Events ([pub](#emission)/[sub](#consumption)), fanned out or [routed](#routing)
 - [Tasks](#tasks)
 - [Pipelines](#pipelines)
 - [Reply streams](#reply-streams)
@@ -67,6 +67,9 @@ value as in the Request.
 
 **Event** is an AMQP message published to an exchange.
 
+**Key** is the routing key an Event is published with, and the one a queue is bound under. An
+Event published to a fanout exchange carries none.
+
 **Task** is an AMQP message sent to a queue without a `replyTo` property set.
 
 **Producer** is an application role that receives Requests and Tasks, and produces Replies and Events.
@@ -158,6 +161,39 @@ asserted.
 await io.emit('numbers_added', { a: 1, b: 2 })
 ```
 
+## Routing
+
+`async IO.route(exchange: string, key: string, payload: any, encoding?: string): void`
+
+`async IO.subscribe(exchange: string, queue: string, key: string, consumer): void`
+
+Publish and consume Events addressed by a Key, where [Emission](#emission)
+and [Consumption](#consumption) fan out.
+
+`route` asserts a
+[direct exchange](https://www.rabbitmq.com/tutorials/amqp-concepts.html#exchange-direct) (once per
+unique `exchange`) and publishes the encoded Event to it under the `key`. `subscribe` asserts the
+same exchange and a durable `queue`, binds it under the `key` (once per unique `exchange`, `queue`
+and `key`), and starts consuming. **An Event reaches the queues bound under the Key it carries,
+and no others.**
+
+The queue is named rather than derived from a Consumer group, because what identifies it here is
+the Key it is bound under rather than the exchange it belongs to. It is also durable, so what is
+published while nothing is consuming is held rather than dropped.
+
+Over a [sharded connection](#sharded-connection) they behave as the rest does: `route` publishes
+to one shard, and `subscribe` consumes the queue on every one of them.
+
+### Example
+
+```javascript
+await io.subscribe('records', 'records.orders', 'store.orders',
+  (record) => console.log(record))
+
+await io.route('records', 'store.orders', { id: 1, status: 'paid' })
+await io.route('records', 'store.customers', { id: 2 }) // not delivered to the above
+```
+
 ## Tasks
 
 `async IO.enqueue(queue: string, payload: any, encoding?: string): void`
@@ -409,6 +445,9 @@ requests and are expecting replies.
 
 - Exchanges and queues for Events, and queues for Requests
   are _durable_.
+- An exchange is asserted as _fanout_ for [Emission](#emission)
+  and [Consumption](#consumption), and as _direct_ for [Routing](#routing). One name is one or
+  the other: asserting it as both is what the broker refuses.
 - Queues for Replies are _exclusive_ and _auto deleted_.
 
 See [queue assertion options](https://amqp-node.github.io/amqplib/channel_api.html#channel_assertQueue).

package/source/channel.js

@@ -101,6 +101,24 @@ class Channel {
           if (!this.#sealed) await this.#consume(queue, callback)
         })))
 
+  /**
+   * Consumes a named queue bound to a direct exchange under a routing key, where `subscribe`
+   * binds to a fanout and takes everything published to it.
+   */
+  bound = recall(this,
+    failsafe(this, this.#recover,
+      lazy(this, [this.#assertRouted, this.#assertKeyedQueue],
+        /**
+         * @param {string} exchange
+         * @param {string} queue
+         * @param {string} key
+         * @param {comq.channels.Consumer} callback
+         * @returns {Promise<void>}
+         */
+        async (exchange, queue, key, callback) => {
+          if (!this.#sealed) await this.#consume(queue, callback)
+        })))
+
   send = failsafe(this, this.#recover,
     lazy(this, this.#assertQueue,
       /**
@@ -123,6 +141,21 @@ class Channel {
         await this.#publish(exchange, DEFAULT, buffer, options)
       }))
 
+  /**
+   * Publishes to a direct exchange under a routing key, where `publish` fans out.
+   */
+  route = failsafe(this, this.#recover,
+    lazy(this, this.#assertRouted,
+      /**
+       * @param {string} exchange
+       * @param {string} key
+       * @param {Buffer} buffer
+       * @param {comq.amqp.options.Publish} [options]
+       */
+      async (exchange, key, buffer, options) => {
+        await this.#publish(exchange, key, buffer, options)
+      }))
+
   async fire (queue, buffer, options) {
     try {
       await this.#publish(DEFAULT, queue, buffer, options)
@@ -230,6 +263,33 @@ class Channel {
     await this.#channel.assertExchange(exchange, 'fanout', options)
   }
 
+  /**
+   * A routed exchange is `direct`: a message reaches the queues bound under the key it was
+   * published with, and none of the others.
+   *
+   * @param {string} exchange
+   * @returns {Promise<void>}
+   */
+  async #assertRouted (exchange) {
+    /** @type {comq.amqp.options.Exchange} */
+    const options = { durable: this.#topology.durable }
+
+    await this.#channel.assertExchange(exchange, 'direct', options)
+  }
+
+  /**
+   * @param {string} exchange
+   * @param {string} queue
+   * @param {string} key
+   * @returns {Promise<string[]>}
+   */
+  async #assertKeyedQueue (exchange, queue, key) {
+    queue = (await this.#assertQueue(queue))[0]
+    await this.#channel.bindQueue(queue, exchange, key)
+
+    return [exchange, queue, key]
+  }
+
   /**
    *
    * @param {string} exchange

package/source/io.js

@@ -132,20 +132,46 @@ class IO {
         )
       }
 
-      /** @type {comq.amqp.options.Publish} */
-      const properties = {}
+      const [buffer, properties] = this.#properties(payload, encoding)
 
-      if (typeof encoding === 'object') { // properties passed
-        Object.assign(properties, encoding)
+      await this.#events[method](exchange, buffer, properties)
+    })
 
-        encoding = /** @type {comq.Encoding} */ properties.contentType
-      }
+  /**
+   * Publishes to a routed exchange under a key, where `emit` fans out. The exchange is
+   * `direct`, so a message reaches the queues bound under that key and no others.
+   */
+  route = lazy(this, this.#createEventChannel,
+    /**
+     * @param {string} exchange
+     * @param {string} key
+     * @param {any} payload
+     * @param {comq.Encoding | comq.amqp.options.Publish} [encoding]
+     * @returns {Promise<void>}
+     */
+    async (exchange, key, payload, encoding) => {
+      const [buffer, properties] = this.#properties(payload, encoding)
 
-      const [buffer, contentType] = this.#encode(payload, encoding)
+      await this.#events.route(exchange, key, buffer, properties)
+    })
 
-      properties.contentType = contentType
+  /**
+   * Consumes a named durable queue bound to a routed exchange under a key, where `consume`
+   * takes everything published to a fanout. The queue is named rather than derived, because
+   * what is bound to it is a key rather than an exchange.
+   */
+  subscribe = lazy(this, this.#createEventChannel,
+    /**
+     * @param {string} exchange
+     * @param {string} queue
+     * @param {string} key
+     * @param {comq.Consumer} callback
+     * @returns {Promise<void>}
+     */
+    async (exchange, queue, key, callback) => {
+      const consumer = this.#getEventConsumer(callback)
 
-      await this.#events[method](exchange, buffer, properties)
+      await this.#events.bound(exchange, queue, key, consumer)
     })
 
   process = lazy(this, this.#createEventChannel,
@@ -200,6 +226,30 @@ class IO {
     this.#events = await this.#createChannel('event')
   }
 
+  /**
+   * What a publication carries, from what its caller passed as an encoding or as properties.
+   *
+   * @param {any} payload
+   * @param {comq.Encoding | comq.amqp.options.Publish} [encoding]
+   * @returns {[Buffer, comq.amqp.options.Publish]}
+   */
+  #properties (payload, encoding) {
+    /** @type {comq.amqp.options.Publish} */
+    const properties = {}
+
+    if (typeof encoding === 'object') { // properties passed
+      Object.assign(properties, encoding)
+
+      encoding = /** @type {comq.Encoding} */ properties.contentType
+    }
+
+    const [buffer, contentType] = this.#encode(payload, encoding)
+
+    properties.contentType = contentType
+
+    return [buffer, properties]
+  }
+
   async #consumeReplies (queue) {
     const emitter = io.createReplyEmitter(queue)
     const consumer = this.#getReplyConsumer(queue, emitter)

package/source/shards/channel.js

@@ -64,6 +64,10 @@ class Channel {
     await this.#every((channel) => channel.subscribe(queue, group, consumer))
   }
 
+  async bound (exchange, queue, key, consumer) {
+    await this.#every((channel) => channel.bound(exchange, queue, key, consumer))
+  }
+
   async send (queue, buffer, options) {
     await this.#one((channel) => channel.send(queue, buffer, options))
   }
@@ -72,6 +76,10 @@ class Channel {
     await this.#one((channel) => channel.publish(exchange, buffer, options))
   }
 
+  async route (exchange, key, buffer, options) {
+    await this.#one((channel) => channel.route(exchange, key, buffer, options))
+  }
+
   async fire (queue, buffer, options) {
     // noinspection  JSValidateTypes
     return await this.#one((channel) => channel.fire(queue, buffer, options))

package/test/channel.routed.test.js

@@ -0,0 +1,119 @@
+'use strict'
+
+const { generate } = require('randomstring')
+
+const { amqplib } = require('./amqplib.mock')
+const fixtures = require('./channel.fixtures')
+const { create } = require('../source/channel')
+
+/** @type {jest.MockedObject<comq.amqp.Connection>} */
+let connection
+
+/** @type {comq.Topology} */
+let topology
+
+/** @type {comq.Channel} */
+let channel
+
+/** @type {jest.MockedObject<comq.amqp.Channel>} */
+let chan
+
+const exchange = generate()
+const queue = generate()
+const key = generate()
+
+const getCreatedChannel = () => {
+  const method = `create${topology.confirms ? 'Confirm' : ''}Channel`
+
+  return connection[method].mock.results[0].value
+}
+
+beforeEach(async () => {
+  jest.clearAllMocks()
+
+  connection = await amqplib.connect()
+  topology = fixtures.preset()
+  channel = await create(connection, topology)
+  chan = await getCreatedChannel()
+})
+
+describe('route', () => {
+  const buffer = Buffer.from(generate())
+
+  it('should assert a direct exchange', async () => {
+    await channel.route(exchange, key, buffer)
+
+    expect(chan.assertExchange).toHaveBeenCalledTimes(1)
+
+    const [name, type] = chan.assertExchange.mock.calls[0]
+
+    expect(name).toStrictEqual(exchange)
+    expect(type).toStrictEqual('direct')
+  })
+
+  it('should publish with the key as the routing key', async () => {
+    await channel.route(exchange, key, buffer)
+
+    const [name, routingKey, published] = chan.publish.mock.calls[0]
+
+    expect(name).toStrictEqual(exchange)
+    expect(routingKey).toStrictEqual(key)
+    expect(published).toStrictEqual(buffer)
+  })
+
+  it('should assert the exchange once for many keys', async () => {
+    await channel.route(exchange, key, buffer)
+    await channel.route(exchange, generate(), buffer)
+
+    expect(chan.assertExchange).toHaveBeenCalledTimes(1)
+    expect(chan.publish).toHaveBeenCalledTimes(2)
+  })
+})
+
+describe('bound', () => {
+  const consumer = /** @type {comq.channels.Consumer} */ jest.fn(() => undefined)
+
+  it('should assert a direct exchange', async () => {
+    await channel.bound(exchange, queue, key, consumer)
+
+    const [name, type] = chan.assertExchange.mock.calls[0]
+
+    expect(name).toStrictEqual(exchange)
+    expect(type).toStrictEqual('direct')
+  })
+
+  it('should assert the queue by name', async () => {
+    await channel.bound(exchange, queue, key, consumer)
+
+    expect(chan.assertQueue).toHaveBeenCalledWith(queue, expect.anything())
+  })
+
+  it('should bind the queue under the key', async () => {
+    await channel.bound(exchange, queue, key, consumer)
+
+    const { queue: asserted } = await chan.assertQueue.mock.results[0].value
+
+    expect(chan.bindQueue).toHaveBeenCalledTimes(1)
+    expect(chan.bindQueue).toHaveBeenCalledWith(asserted, exchange, key)
+  })
+
+  it('should consume the queue', async () => {
+    await channel.bound(exchange, queue, key, consumer)
+
+    expect(chan.consume).toHaveBeenCalledWith(queue, expect.any(Function), expect.anything())
+  })
+
+  it('should bind once per key', async () => {
+    await channel.bound(exchange, queue, key, consumer)
+    await channel.bound(exchange, queue, key, consumer)
+
+    expect(chan.bindQueue).toHaveBeenCalledTimes(1)
+  })
+
+  it('should bind again for another key', async () => {
+    await channel.bound(exchange, queue, key, consumer)
+    await channel.bound(exchange, generate(), generate(), consumer)
+
+    expect(chan.bindQueue).toHaveBeenCalledTimes(2)
+  })
+})

package/test/connection.mock.js

@@ -9,7 +9,9 @@ const channel = /** @type {jest.MockedFunction<(sharded?: boolean, index?: numbe
     send: jest.fn(async () => undefined),
     fire: jest.fn(async () => undefined),
     subscribe: jest.fn(async () => undefined),
+    bound: jest.fn(async () => undefined),
     publish: jest.fn(async () => undefined),
+    route: jest.fn(async () => undefined),
     diagnose: jest.fn(async () => undefined),
     forget: jest.fn(() => undefined),
     seal: jest.fn(async () => undefined),

package/test/io.route.test.js

@@ -0,0 +1,94 @@
+'use strict'
+
+const { generate } = require('randomstring')
+
+const { decode } = require('../source/decode')
+const { encode } = require('../source/encode')
+
+const mock = require('./connection.mock')
+const { IO } = require('../source/io')
+
+/** @type {comq.IO} */
+let io
+
+/** @type {jest.MockedObject<comq.Connection>} */
+let connection
+
+/** @type {jest.MockedObject<comq.Channel>} */
+let events
+
+const exchange = generate()
+const queue = generate()
+const key = generate()
+const payload = generate()
+
+beforeEach(async () => {
+  jest.clearAllMocks()
+
+  connection = mock.connection()
+  io = new IO(connection)
+})
+
+describe('route', () => {
+  beforeEach(async () => {
+    await io.route(exchange, key, payload)
+
+    events = await connection.createChannel.mock.results[0].value
+  })
+
+  it('should create an event channel', async () => {
+    expect(connection.createChannel).toHaveBeenCalledWith('event')
+  })
+
+  it('should route to the exchange under the key', async () => {
+    expect(events.route).toHaveBeenCalledTimes(1)
+
+    const [name, routingKey, buffer, properties] = events.route.mock.calls[0]
+
+    expect(name).toStrictEqual(exchange)
+    expect(routingKey).toStrictEqual(key)
+    expect(decode({ content: buffer, properties })).toStrictEqual(payload)
+  })
+
+  it('should pass properties through', async () => {
+    jest.clearAllMocks()
+
+    const properties = { headers: { 'x-region': 'eu' }, persistent: true }
+
+    await io.route(exchange, key, payload, properties)
+
+    expect(events.route.mock.calls[0][3]).toMatchObject(properties)
+  })
+})
+
+describe('subscribe', () => {
+  const consumer = jest.fn()
+
+  it('should bind the queue under the key and consume it', async () => {
+    await io.subscribe(exchange, queue, key, consumer)
+
+    events = await connection.createChannel.mock.results[0].value
+
+    expect(events.bound).toHaveBeenCalledTimes(1)
+
+    const [name, bound, routingKey] = events.bound.mock.calls[0]
+
+    expect(name).toStrictEqual(exchange)
+    expect(bound).toStrictEqual(queue)
+    expect(routingKey).toStrictEqual(key)
+  })
+
+  it('should hand the payload to the consumer', async () => {
+    await io.subscribe(exchange, queue, key, consumer)
+
+    events = await connection.createChannel.mock.results[0].value
+
+    const deliver = events.bound.mock.calls[0][3]
+    const properties = { contentType: 'application/json' }
+    const message = { content: encode(payload, properties.contentType), properties }
+
+    await deliver(message)
+
+    expect(consumer).toHaveBeenCalledWith(payload, properties)
+  })
+})

package/test/shards/channel.test.js

@@ -718,3 +718,36 @@ async function getCreatedChannels () {
 
   return await Promise.all(promises)
 }
+
+describe('routed', () => {
+  const exchange = generate()
+  const queue = generate()
+  const key = generate()
+  const buffer = randomBytes(8)
+  const consumer = jest.fn()
+
+  /** @type {jest.MockedObject<comq.Channel>[]} */
+  let channels
+
+  beforeEach(async () => {
+    channel = await create(connections, type)
+    channels = await getCreatedChannels()
+  })
+
+  it('should bind using all channels', async () => {
+    await channel.bound(exchange, queue, key, consumer)
+
+    for (const chan of channels) {
+      expect(chan.bound).toHaveBeenCalledWith(exchange, queue, key, consumer)
+    }
+  })
+
+  it('should route to a single channel', async () => {
+    await channel.route(exchange, key, buffer)
+
+    const used = channels.filter((chan) => chan.route.mock.calls.length > 0)
+
+    expect(used).toHaveLength(1)
+    expect(used[0].route).toHaveBeenCalledWith(exchange, key, buffer, undefined)
+  })
+})

package/types/channel.d.ts

@@ -19,10 +19,16 @@ declare namespace comq {
 
     subscribe (exchange: string, queue: string, consumer: channels.Consumer): Promise<void>
 
+    /** consumes `queue`, bound to a routed `exchange` under `key` */
+    bound (exchange: string, queue: string, key: string, consumer: channels.Consumer): Promise<void>
+
     send (queue: string, buffer: Buffer, options?: Options.Publish): Promise<void>
 
     publish (exchange: string, buffer: Buffer, options?: Options.Publish): Promise<void>
 
+    /** publishes to a routed exchange under `key`, where `publish` fans out */
+    route (exchange: string, key: string, buffer: Buffer, options?: Options.Publish): Promise<void>
+
     fire (queue: string, buffer: Buffer, options?: Options.Publish): Promise<boolean>
 
     seal (): Promise<void>

package/types/io.d.ts

@@ -63,6 +63,20 @@ declare namespace comq {
 
     enqueue(exchange: string, stream: Readable, properties?: _amqp.Properties): Promise<void>
 
+    /**
+     * Publishes to a routed (`direct`) exchange under a key, where `emit` fans out: the
+     * message reaches the queues bound under that key and no others.
+     */
+    route(exchange: string, key: string, payload: any, encoding?: _encoding.Encoding): Promise<void>
+
+    route(exchange: string, key: string, payload: any, properties?: _amqp.Properties): Promise<void>
+
+    /**
+     * Consumes a named durable queue bound to a routed exchange under a key, where `consume`
+     * takes everything published to a fanout.
+     */
+    subscribe<T = any>(exchange: string, queue: string, key: string, consumer: Consumer<T>): Promise<void>
+
     process<T = any>(queue: string, processor: Consumer<T>): Promise<void>
 
     seal(): Promise<void>