comq 0.8.0 → 0.10.0

package/source/shards/channel.js

@@ -1,8 +1,8 @@
 'use strict'
 
-const { EventEmitter } = require('node:events')
 const { promex, sample } = require('@toa.io/generic')
 const events = require('../events')
+const emitter = require('../emitter')
 
 /**
  * @implements {comq.Channel}
@@ -30,7 +30,7 @@ class Channel {
 
   #recovery = promex()
 
-  #diagnostics = new EventEmitter()
+  #diagnostics = emitter.create()
 
   /**
    * @param {comq.Connection[]} connections
@@ -48,7 +48,7 @@ class Channel {
   }
 
   async consume (queue, consumer) {
-    await this.#any((channel) => channel.consume(queue, consumer))
+    return await this.#any((channel) => channel.consume(queue, consumer))
   }
 
   async subscribe (queue, group, consumer) {
@@ -64,17 +64,22 @@ class Channel {
   }
 
   async fire (queue, buffer, options) {
-    await this.#one((channel) => channel.fire(queue, buffer, options))
+    // noinspection  JSValidateTypes
+    return await this.#one((channel) => channel.fire(queue, buffer, options))
   }
 
   async seal () {
     await this.#all((channel) => channel.seal())
   }
 
-  async diagnose (event, listener) {
+  diagnose (event, listener) {
     this.#diagnostics.on(event, listener)
   }
 
+  forget (event, listener) {
+    this.#diagnostics.off(event, listener)
+  }
+
   /**
    * @param {comq.Connection} connection
    * @param {number} index
@@ -120,7 +125,7 @@ class Channel {
    */
   #add (channel) {
     this.#channels.add(channel)
-    this.#pool = Array.from(this.#channels)
+    this.#update()
   }
 
   /**
@@ -131,10 +136,23 @@ class Channel {
 
     this.#bench.set(channel, promex())
     this.#channels.delete(channel)
-    this.#pool = Array.from(this.#channels)
+    this.#update()
     this.#diagnostics.emit('remove', channel.index)
   }
 
+  #update () {
+    const from = this.#pool?.length
+    const to = this.#channels.size
+
+    this.#pool = Array.from(this.#channels)
+
+    if (from === undefined) return
+
+    if (from !== 0 && to === 0) { this.#diagnostics.emit('pause') }
+
+    if (from === 0 && to !== 0) { this.#diagnostics.emit('resume') }
+  }
+
   /**
    * @param {comq.Channel} channel
    */
@@ -154,12 +172,11 @@ class Channel {
 
   /**
    * @param {(channel: comq.Channel) => void} fn
-   * @return {Promise<void>}
    */
   async #any (fn) {
     const promises = this.#apply(fn)
 
-    await Promise.any(promises)
+    return await Promise.any(promises)
   }
 
   /**
@@ -188,7 +205,6 @@ class Channel {
 
   /**
    * @param {(channel: comq.Channel) => void} fn
-   * @return {Promise<void>}
    */
   async #one (fn) {
     if (this.#pool.length === 0) await this.#recovery
@@ -196,11 +212,11 @@ class Channel {
     const channel = sample(this.#pool)
 
     try {
-      await fn(channel)
+      return await fn(channel)
     } catch {
       this.#remove(channel)
 
-      await this.#one(fn)
+      return await this.#one(fn)
     }
   }
 }

package/source/shards/connection.js

@@ -1,8 +1,8 @@
 'use strict'
 
-const { EventEmitter } = require('node:events')
 const events = require('../events')
 const channel = require('./channel')
+const emitter = require('../emitter')
 
 /**
  * @implements {comq.Connection}
@@ -11,7 +11,7 @@ class Connection {
   /** @type {comq.Connection[]} */
   #connections
 
-  #diagnostics = new EventEmitter()
+  #diagnostics = emitter.create()
 
   /**
    * @param {comq.Connection[]} connections

package/test/channel.test.js

@@ -67,7 +67,9 @@ describe('consume', () => {
   it('should assert queue', async () => {
     await channel.consume(queue, consumer)
 
-    const options = topology.durable ? { durable: true } : { exclusive: true }
+    const options = topology.durable
+      ? { durable: true }
+      : { exclusive: true }
 
     expect(chan.assertQueue).toHaveBeenCalledWith(queue, options)
   })
@@ -823,17 +825,23 @@ describe('diagnostics', () => {
 
     let flowed = false
     let drained = false
+    let paused = false
+    let resumed = false
 
     channel.diagnose('flow', () => (flowed = true))
     channel.diagnose('drain', () => (drained = true))
+    channel.diagnose('pause', () => (paused = true))
+    channel.diagnose('resume', () => (resumed = true))
 
     await channel.publish(exchange, buffer)
 
     expect(flowed).toStrictEqual(true)
+    expect(paused).toStrictEqual(true)
 
     chan.emit('drain')
 
     expect(drained).toStrictEqual(true)
+    expect(resumed).toStrictEqual(true)
   })
 
   it.each(/** @type {[string, boolean][]} */ [

package/test/encode.test.js

@@ -11,7 +11,7 @@ it('should be', async () => {
 
 it('should throw if encoding is not supported', async () => {
   const value = 1
-  const encoding = /** @type {comq.encoding} */ 'wtf/' + generate()
+  const encoding = /** @type {comq.Encoding} */ 'wtf/' + generate()
 
   expect(() => encode(value, encoding)).toThrow('not supported')
 })
@@ -34,7 +34,7 @@ it('should encode as msgpack', async () => {
   expect(buffer).toStrictEqual(packed)
 })
 
-/** @type {[comq.encoding, string, Buffer][]} */
+/** @type {[comq.Encoding, string, Buffer][]} */
 const samples = [
   ['text/plain', 'some string', Buffer.from('some string')]
 ]

package/test/encodings.js

@@ -1,4 +1,4 @@
 'use strict'
 
-/** @type {comq.encoding[]} */
+/** @type {comq.Encoding[]} */
 exports.encodings = ['application/msgpack', 'application/json']

package/test/io.diagnostics.test.js

@@ -28,9 +28,9 @@ describe.each(['request', 'reply', 'event'])('%s channel events',
    * @param {comq.topology.type} type
    */
   (type) => {
-    it.each(['flow', 'drain', 'recover', 'discard'])('should re-emit %s',
+    it.each(['flow', 'drain', 'recover', 'discard', 'pause', 'resume'])('should re-emit %s',
       /**
-       * @param {comq.diagnostics.event} event
+       * @param {comq.diagnostics.Event} event
        */
       async (event) => {
         const listener = /** @type {Function} */ jest.fn()
@@ -56,7 +56,7 @@ describe.each(['request', 'reply', 'event'])('%s channel events',
 
 it.each(['open', 'close'])('should re-emit %s from connection',
   /**
-   * @param {comq.diagnostics.event} event
+   * @param {comq.diagnostics.Event} event
    */
   async (event) => {
     const listener = /** @type {Function} */ jest.fn(() => undefined)

package/test/io.emit.test.js

@@ -1,5 +1,6 @@
 'use strict'
 
+const stream = require('node:stream')
 const { randomBytes } = require('node:crypto')
 const { generate } = require('randomstring')
 
@@ -18,6 +19,9 @@ let connection
 /** @type {jest.MockedObject<comq.Channel>} */
 let events
 
+const exchange = generate()
+const payload = generate()
+
 beforeEach(async () => {
   jest.clearAllMocks()
 
@@ -33,9 +37,6 @@ it('should be', async () => {
   expect(io.emit).toBeDefined()
 })
 
-const exchange = generate()
-const payload = generate()
-
 it('should create events channel', async () => {
   expect(connection.createChannel).toHaveBeenCalledWith('event')
   expect(events).toBeDefined()
@@ -49,6 +50,20 @@ it('should publish to an exchange', async () => {
   expect(args[0]).toStrictEqual(exchange)
 })
 
+it('should publish stream of events', async () => {
+  jest.clearAllMocks()
+
+  function * generate () {
+    for (let i = 0; i < 10; i++) yield i
+  }
+
+  const input = stream.Readable.from(generate())
+
+  await io.emit(exchange, input)
+
+  expect(events.publish).toHaveBeenCalledTimes(10)
+})
+
 it('should encode message as msgpack by default', async () => {
   const encoding = 'application/msgpack'
   const buf = encode(payload, encoding)

package/test/io.request.test.js

@@ -2,6 +2,7 @@
 
 // region setup
 
+const stream = require('node:stream')
 const { randomBytes } = require('node:crypto')
 const { generate } = require('randomstring')
 const { immediate } = require('@toa.io/generic')
@@ -88,25 +89,6 @@ describe('send', () => {
     expect(properties.replyTo).toMatch(rx)
   })
 
-  it('should set replyTo using specified formatter', async () => {
-    jest.clearAllMocks()
-
-    /** @type {comq.encoding} */
-    const prefix = generate()
-    const encoding = 'application/json'
-    const formatter = (replyTo) => prefix + replyTo
-
-    promise = io.request(queue, payload, encoding, formatter)
-
-    await immediate()
-
-    call = requests.send.mock.calls[0]
-
-    const properties = requests.send.mock.calls[0][2]
-
-    expect(properties.replyTo).toMatch(new RegExp(`${prefix}${queue}..[a-z0-9]+`))
-  })
-
   it('should consume replyTo', async () => {
     const properties = call[2]
     const queue = replies.consume.mock.calls[0][0]
@@ -115,7 +97,7 @@ describe('send', () => {
   })
 
   it('should encode message with msgpack by default', async () => {
-    /** @type {comq.encoding} */
+    /** @type {comq.Encoding} */
     const contentType = 'application/msgpack'
     const buffer = encode(payload, contentType)
 
@@ -124,7 +106,7 @@ describe('send', () => {
   })
 
   it('should throw if encoding is not supported', async () => {
-    const encoding = /** @type {comq.encoding} */ 'wtf/' + generate()
+    const encoding = /** @type {comq.Encoding} */ 'wtf/' + generate()
 
     await expect(io.request(queue, payload, encoding)).rejects.toThrow('is not supported')
   })
@@ -152,7 +134,7 @@ describe('send', () => {
 
     setImmediate(reply)
 
-    await io.request(queue, payload, /** @type {comq.encoding} */ encoding)
+    await io.request(queue, payload, /** @type {comq.Encoding} */ encoding)
 
     const [, buffer, properties] = requests.send.mock.calls[0]
 
@@ -160,6 +142,33 @@ describe('send', () => {
     expect(properties.contentType).toStrictEqual(encoding)
   })
 
+  it('should send stream of requests', async () => {
+    requests.send.mockClear()
+
+    function * generate () {
+      for (let i = 0; i < 10; i++) yield i
+    }
+
+    const encoding = 'application/json'
+    const replyFormatter = (queue) => queue
+
+    setTimeout(replyAll, 10) // read the stream to end
+
+    const input = stream.Readable.from(generate())
+
+    /** @type {Readable} */
+    const output = await io.request(queue, input, encoding, replyFormatter)
+
+    expect(output).toBeInstanceOf(stream.Readable)
+
+    const replies = []
+
+    for await (const reply of output) replies.push(reply)
+
+    expect(replies.length).toStrictEqual(10)
+    expect(requests.send).toHaveBeenCalledTimes(10)
+  })
+
   it('should resend unanswered Requests', async () => {
     expect(requests.diagnose).toHaveBeenCalledWith('recover', expect.any(Function))
 
@@ -216,7 +225,7 @@ describe('reply', () => {
 
   it.each(encodings)('should decode %s',
     /**
-     * @param {comq.encoding}contentType
+     * @param {comq.Encoding}contentType
      */
     async (contentType) => {
       const value = generate()
@@ -239,6 +248,19 @@ const reply = async (content = randomBytes(8), contentType = undefined) => {
   await callback(message)
 }
 
+async function replyAll () {
+  const callback = replies.consume.mock.calls[0][1]
+
+  for (let i = 0; i < requests.send.mock.calls.length; i++) {
+    const call = requests.send.mock.calls[i]
+    const correlationId = call[2].correlationId
+    const properties = { correlationId }
+    const message = /** @type {comq.amqp.Message} */ { content: randomBytes(8), properties }
+
+    await callback(message)
+  }
+}
+
 /**
  * @param {comq.topology.type} type
  * @returns {jest.MockedObject<comq.Channel>}

package/test/pipeline.test.js

@@ -0,0 +1,81 @@
+'use strict'
+
+const { EventEmitter } = require('node:events')
+const stream = require('node:stream')
+const { pipeline, transform } = require('../source/pipeline')
+const { timeout } = require('@toa.io/generic')
+
+/** @type {Channel} */
+let channel
+
+beforeEach(() => {
+  channel = new Channel()
+})
+
+it('should pipeline', async () => {
+  const input = stream.Readable.from([1, 2, 3])
+  const pipe = pipeline(input, (x) => x * x, channel)
+  const output = []
+
+  for await (const x of pipe) {
+    output.push(x)
+  }
+
+  expect(output).toStrictEqual([1, 4, 9])
+})
+
+it('should control source', async () => {
+  async function * generate () {
+    for (let i = 1; i < 4; i++) {
+      await timeout(1)
+
+      yield i
+    }
+  }
+
+  const input = stream.Readable.from(generate())
+
+  // eslint-disable-next-line no-void
+  void pipeline(input, (x) => x * x, channel)
+
+  channel.emit('pause')
+
+  await timeout(0)
+
+  expect(input.isPaused()).toBe(true)
+
+  channel.emit('resume')
+
+  await timeout(0)
+
+  expect(input.isPaused()).toBe(false)
+})
+
+it('should wait for end', async () => {
+  async function * generate () {
+    for (let i = 1; i < 4; i++) {
+      await timeout(1)
+
+      yield i
+    }
+  }
+
+  let finished = 0
+
+  const fn = jest.fn(async () => {
+    await timeout(1)
+    finished++
+  })
+  const input = stream.Readable.from(generate())
+
+  await transform(input, fn, channel)
+
+  expect(fn).toHaveBeenCalledTimes(3)
+  expect(finished).toBe(3)
+})
+
+class Channel extends EventEmitter {
+  diagnose (event, callback) {
+    this.on(event, callback)
+  }
+}

package/test/shards/channel.test.js

@@ -187,7 +187,7 @@ describe.each(/** @type {string[]} */['send', 'publish', 'fire'])('%s', (method)
 
     const last = label + i
 
-    // now channel has thrown an exception
+    // now the channel has thrown an exception
     expect(broken[method]).toHaveBeenCalledWith(last, buffer, options)
 
     const r = (b + 1) % 2
@@ -202,7 +202,7 @@ describe.each(/** @type {string[]} */['send', 'publish', 'fire'])('%s', (method)
     /** @type {jest.MockedObject<comq.Channel>} */
     let recovered
 
-    // both shards rejects and will be removed from the pool
+    // both shards reject and will be removed from the pool
     channels[0][method].mockImplementationOnce(reject)
     channels[1][method].mockImplementationOnce(reject)
 
@@ -289,7 +289,7 @@ describe('seal', () => {
 })
 
 describe('diagnose', () => {
-  const events = /** @type {comq.diagnostics.event[]} */ ['flow', 'drain', 'recover', 'discard']
+  const events = /** @type {comq.diagnostics.Event[]} */ ['flow', 'drain', 'recover', 'discard']
 
   /** @type {jest.MockedObject<comq.Channel>[]} */
   let channels
@@ -338,6 +338,42 @@ describe('diagnose', () => {
 
     expect(listener).toHaveBeenCalledWith(expect.any(Number))
   })
+
+  it('should emit `pause` and `unpause` events', async () => {
+    const queue = generate()
+    const buffer = randomBytes(8)
+    const pause = /** @type {jest.MockedFunction} */ jest.fn()
+    const resume = /** @type {jest.MockedFunction} */ jest.fn()
+
+    channel.diagnose('pause', pause)
+    channel.diagnose('resume', resume)
+
+    for (const chan of channels) {
+      chan.send.mockImplementationOnce(() => {
+        expect(pause).not.toHaveBeenCalled()
+        throw new Error()
+      })
+    }
+
+    // recover
+    setImmediate(() => {
+      for (const chan of channels) {
+        // subscriptions to the 'recover' event
+        const calls = chan.diagnose.mock.calls.filter(
+          (call) => call[0] === 'recover')
+
+        expect(calls.length).toBeGreaterThan(0)
+
+        // emit 'recover' event to return shard to the pool
+        for (const call of calls) call[1]()
+      }
+    })
+
+    await channel.send(queue, buffer)
+
+    expect(pause).toHaveBeenCalledTimes(1)
+    expect(resume).toHaveBeenCalledTimes(1)
+  })
 })
 
 it('should not throw on recovery (if the "bench" is empty)', async () => {

package/test/shards/connection.test.js

@@ -106,7 +106,7 @@ describe('createChannel', () => {
   })
 })
 
-describe.each(/** @type {comq.diagnostics.event[]} */ ['open', 'close'])('diagnose %s event',
+describe.each(/** @type {comq.diagnostics.Event[]} */ ['open', 'close'])('diagnose %s event',
   (event) => {
     it('should re-emit event', async () => {
       const index = random(connections.length)

package/types/channel.d.ts

@@ -5,7 +5,7 @@ declare namespace comq {
 
   namespace channels {
 
-    type consumer = (message: Message) => void | Promise<void>
+    type Consumer = (message: Message) => void | Promise<void>
 
   }
 
@@ -13,26 +13,28 @@ declare namespace comq {
     index?: number
     sharded?: boolean
 
-    create(): Promise<void>
+    create (): Promise<void>
 
-    consume(queue: string, consumer: channels.consumer): Promise<void>
+    consume (queue: string, consumer: channels.Consumer): Promise<string>
 
-    subscribe(exchange: string, queue: string, consumer: channels.consumer): Promise<void>
+    subscribe (exchange: string, queue: string, consumer: channels.Consumer): Promise<void>
 
-    send(queue: string, buffer: Buffer, options?: Options.Publish): Promise<void>
+    send (queue: string, buffer: Buffer, options?: Options.Publish): Promise<void>
 
-    publish(exchange: string, buffer: Buffer, options?: Options.Publish): Promise<void>
+    publish (exchange: string, buffer: Buffer, options?: Options.Publish): Promise<void>
 
-    fire(queue: string, buffer: Buffer, options?: Options.Publish): Promise<void>
+    fire (queue: string, buffer: Buffer, options?: Options.Publish): Promise<boolean>
 
-    seal(): Promise<void>
+    seal (): Promise<void>
 
-    diagnose(event: _diagnostics.event, listener: Function): void
+    diagnose (event: _diagnostics.Event, listener: Function): void
 
-    recover(connection: Connection): Promise<void>
+    forget (event: _diagnostics.Event, listener: Function): void
+
+    recover (connection: Connection): Promise<void>
   }
 
 }
 
 export type Channel = comq.Channel
-export type consumer = comq.channels.consumer
+export type Consumer = comq.channels.Consumer

package/types/connection.d.ts

@@ -14,11 +14,11 @@ declare namespace comq {
 
     createChannel(type: _topology.type, index: number): Promise<_channel.Channel>
 
-    diagnose(event: _diagnostics.event, listener: Function): void
+    diagnose(event: _diagnostics.Event, listener: Function): void
   }
 
-  type connect = (...urls: string[]) => Promise<_io.IO>
+  type Connect = (...urls: string[]) => Promise<_io.IO>
 
 }
 
-export type connect = comq.connect
+export type Connect = comq.Connect

package/types/diagnostic.d.ts

@@ -1,12 +1,13 @@
 declare namespace comq.diagnostics {
 
-  type event = 'open' | 'close' | 'flow' | 'drain' | 'remove' | 'recover' | 'discard'
+  type Event = 'open' | 'close' | 'flow' | 'drain' | 'remove' | 'recover' | 'discard' |
+    'pause' | 'resume' | 'return'
 
   interface Diagnosable {
-    diagnose(event: event, listener: Function): void
+    diagnose(event: Event, listener: Function): void
   }
 
 }
 
-export type event = comq.diagnostics.event
+export type Event = comq.diagnostics.Event
 export type Diagnosable = comq.diagnostics.Diagnosable

package/types/encoding.d.ts

@@ -1,6 +1,6 @@
 declare namespace comq {
 
-  type encoding =
+  type Encoding =
     'application/msgpack'
     | 'application/json'
     | 'application/octet-stream'
@@ -8,4 +8,4 @@ declare namespace comq {
 
 }
 
-export type encoding = comq.encoding
+export type Encoding = comq.Encoding

package/types/index.d.ts

@@ -1,6 +1,6 @@
-import { connect } from './connection'
+import { Connect } from './connection'
 
-export type { IO, consumer, producer } from './io'
+export type { IO, Consumer, Producer } from './io'
 
-export const connect: connect
-export const assert: connect
+export const connect: Connect
+export const assert: Connect