comq 0.24.0 → 0.24.1

package/test/io.request.test.js

@@ -5,6 +5,7 @@
 const stream = require('node:stream')
 const { randomBytes } = require('node:crypto')
 const { generate } = require('randomstring')
+const { Promex } = require('promex')
 const { immediate } = require('./helpers')
 const { encode } = require('../source/encode')
 
@@ -327,15 +328,276 @@ describe('send', () => {
   })
 })
 
+describe('shards', () => {
+  beforeEach(async () => {
+    jest.clearAllMocks()
+
+    connection = mock.connection(true)
+    io = new IO(connection)
+  })
+
+  /**
+   * Sends a Request through a shard.
+   *
+   * @param {number} index
+   * @param {string} target
+   * @return {Promise<void>}
+   */
+  async function ask (index, target) {
+    requests = await requestChannel()
+
+    requests.send.mockImplementationOnce(async (_queue, _buffer, _options, via) => via(index))
+
+    io.request(target, payload)
+
+    await immediate()
+  }
+
+  /**
+   * @return {Promise<jest.MockedObject<comq.Channel>>}
+   */
+  async function requestChannel () {
+    if (connection.createChannel.mock.calls.length === 0) {
+      // the channels are created with the first Request
+      io.request(generate(), payload)
+
+      await immediate()
+    }
+
+    return findChannel('request')
+  }
+
+  /**
+   * @param {jest.MockedObject<comq.Channel>} channel
+   * @return {string[]} the queues each publish went to
+   */
+  const targets = (channel) => channel.send.mock.calls.map(([queue]) => queue)
+
+  it('should re-send only the requests the lost shard holds', async () => {
+    const a = generate()
+    const b = generate()
+
+    await ask(0, a)
+    await ask(1, b)
+
+    requests.send.mockClear()
+
+    await lose(1)
+
+    expect(targets(requests)).toStrictEqual([b])
+
+    requests.send.mockClear()
+
+    // b has been re-sent through the shard that stays, and is held by it now
+    await lose(0)
+
+    expect(targets(requests)).toContain(a)
+    expect(targets(requests)).toContain(b)
+  })
+
+  it('should re-send only the requests of a shard whose reply channel recovers', async () => {
+    const a = generate()
+    const b = generate()
+
+    await ask(0, a)
+    await ask(1, b)
+
+    replies = await findChannel('reply')
+    requests.send.mockClear()
+
+    const calls = replies.diagnose.mock.calls.filter((call) => call[0] === 'recover')
+
+    for (const [, listener] of calls) listener(1)
+
+    await immediate()
+
+    expect(targets(requests)).toStrictEqual([b])
+  })
+
+  it('should re-send only the requests of a shard removed from the pool', async () => {
+    const a = generate()
+    const b = generate()
+
+    await ask(0, a)
+    await ask(1, b)
+
+    requests.send.mockClear()
+
+    const calls = requests.diagnose.mock.calls.filter((call) => call[0] === 'remove')
+
+    for (const [, listener] of calls) listener(0)
+
+    await immediate()
+
+    expect(targets(requests).includes(a)).toStrictEqual(true)
+    expect(targets(requests).includes(b)).toStrictEqual(false)
+  })
+
+  it('should re-send a request the lost shard held while it was being published', async () => {
+    requests = await requestChannel()
+
+    const published = new Promex()
+
+    requests.send.mockImplementationOnce(async (_queue, _buffer, _options, via) => {
+      via(1)
+
+      await published
+    })
+
+    const target = generate()
+
+    io.request(target, payload)
+
+    await immediate()
+    await lose(1)
+
+    // not before it is published
+    expect(targets(requests).filter((queue) => queue === target)).toHaveLength(1)
+
+    published.resolve()
+
+    await immediate()
+    await immediate()
+
+    expect(targets(requests).filter((queue) => queue === target)).toHaveLength(2)
+  })
+
+  it('should not re-send a request that failed over to another shard', async () => {
+    requests = await requestChannel()
+
+    const published = new Promex()
+
+    requests.send.mockImplementationOnce(async (_queue, _buffer, _options, via) => {
+      via(1)
+
+      await published
+
+      // the pool fails over from the lost shard
+      via(0)
+    })
+
+    const target = generate()
+
+    io.request(target, payload)
+
+    await immediate()
+    await lose(1)
+
+    published.resolve()
+
+    await immediate()
+    await immediate()
+
+    expect(targets(requests).filter((queue) => queue === target)).toHaveLength(1)
+
+    // it is held by the shard it went through
+    await lose(0)
+
+    expect(targets(requests).filter((queue) => queue === target)).toHaveLength(2)
+  })
+
+  it('should not re-send a request that went through another shard than the one lost as it was published', async () => {
+    requests = await requestChannel()
+
+    const published = new Promex()
+
+    requests.send.mockImplementationOnce(async (_queue, _buffer, _options, via) => {
+      via(0)
+
+      await published
+    })
+
+    const target = generate()
+
+    io.request(target, payload)
+
+    await immediate()
+    await lose(1)
+
+    published.resolve()
+
+    await immediate()
+    await immediate()
+
+    expect(targets(requests).filter((queue) => queue === target)).toHaveLength(1)
+  })
+
+  it('should re-send a returned call from the shard it was published on next', async () => {
+    requests = await requestChannel()
+
+    const key = generate()
+
+    requests.route.mockImplementationOnce(async (_exchange, _key, _buffer, _options, via) => via(0))
+
+    io.call(generate(), key, payload)
+
+    await immediate()
+
+    const [,,, properties] = requests.route.mock.calls[0]
+    const message = { fields: { exchange: generate(), routingKey: key }, properties }
+    const calls = requests.diagnose.mock.calls.filter((call) => call[0] === 'reroute')
+
+    expect(calls.length).toBeGreaterThan(0)
+
+    for (const [, listener] of calls) listener(message, 1)
+
+    await lose(0)
+
+    expect(requests.route).toHaveBeenCalledTimes(1)
+
+    await lose(1)
+
+    expect(requests.route).toHaveBeenCalledTimes(2)
+  })
+
+  it('should re-send a call the lost shard holds', async () => {
+    requests = await requestChannel()
+
+    requests.route.mockImplementationOnce(async (_exchange, _key, _buffer, _options, via) => via(1))
+
+    io.call(generate(), generate(), payload)
+
+    await immediate()
+    await lose(0)
+
+    expect(requests.route).toHaveBeenCalledTimes(1)
+
+    await lose(1)
+
+    expect(requests.route).toHaveBeenCalledTimes(2)
+  })
+})
+
+describe('recovery', () => {
+  it('should re-send every unanswered Request of a connection that is not sharded', async () => {
+    const other = generate()
+
+    io.request(other, payload)
+
+    await immediate()
+
+    requests.send.mockClear()
+
+    const calls = requests.diagnose.mock.calls.filter((call) => call[0] === 'recover')
+
+    for (const [, listener] of calls) listener()
+
+    await immediate()
+
+    expect(requests.send.mock.calls.map(([queue]) => queue).sort()).toStrictEqual([queue, other].sort())
+  })
+})
+
 /**
  * Fires the listeners the IO has attached to the shard loss diagnostic.
  *
+ * @param {number} [index]
  * @return {Promise<void>}
  */
-async function lose () {
+async function lose (index = 0) {
   const calls = requests.diagnose.mock.calls.filter((call) => call[0] === 'lost')
 
-  for (const [, listener] of calls) listener(0)
+  for (const [, listener] of calls) listener(index)
 
   await immediate()
   await immediate()

package/test/memory.test.js

@@ -29,14 +29,12 @@ beforeEach(() => {
   amqplib.connect.mockImplementation(async (url) => wired(await connect(url)))
 
   global.COMQ_TESTING_SHUTDOWN_TIMEOUT = 1
-  global.COMQ_TESTING_WATCHDOG_INTERVAL = 60_000
 })
 
 afterEach(() => {
   SingletonConnection.__lets_pretend_this_method_doesnt_exist()
 
   delete global.COMQ_TESTING_SHUTDOWN_TIMEOUT
-  delete global.COMQ_TESTING_WATCHDOG_INTERVAL
 })
 
 afterAll(() => {

package/test/shards/channel.test.js

@@ -230,7 +230,17 @@ describe.each(['consume', 'subscribe'])('%s', (method) => {
 
     await channel[method](...args)
 
-    for (const chan of channels) expect(chan[method]).toHaveBeenCalledWith(...args)
+    for (const chan of channels) {
+      const call = chan[method].mock.calls[0]
+      const message = /** @type {comq.amqp.Message} */ {}
+
+      expect(call.slice(0, -1)).toStrictEqual(args.slice(0, -1))
+
+      // a consumed message may be told which shard it arrived on on its way
+      await call.at(-1)(message)
+
+      expect(processor).toHaveBeenCalledWith(message)
+    }
   })
 
   it(`should not resolve ${method} until pending channels are created`, async () => {
@@ -593,6 +603,23 @@ describe('diagnose', () => {
     expect(listener).not.toHaveBeenCalled()
   })
 
+  it('should tell where a returned message is published next', async () => {
+    const listener = /** @type {jest.MockedFunction} */ jest.fn()
+    const index = random(channels.length)
+    const chan = channels[index]
+    const message = returned()
+
+    message.fields.exchange = generate()
+    channel.diagnose('reroute', listener)
+
+    emitReturn(chan, message)
+
+    const next = channels.find((one) => one.route.mock.calls.length > 0)
+
+    expect(next).not.toBe(chan)
+    expect(listener).toHaveBeenCalledWith(message, next.index)
+  })
+
   it('should report a returned retry at once', async () => {
     const listener = /** @type {jest.MockedFunction} */ jest.fn()
     const index = random(channels.length)
@@ -620,7 +647,15 @@ describe('diagnose', () => {
 
     await channel.held(exchange, queue, key, consumer)
 
-    for (const one of channels) expect(one.held).toHaveBeenCalledWith(exchange, queue, key, consumer)
+    for (const one of channels) {
+      expect(one.held).toHaveBeenCalledWith(exchange, queue, key, expect.any(Function))
+
+      const message = /** @type {comq.amqp.Message} */ {}
+
+      await one.held.mock.calls[0][3](message)
+
+      expect(consumer).toHaveBeenCalledWith(message)
+    }
   })
 
   it('should hold a queue once any shard holds it', async () => {
@@ -819,3 +854,134 @@ describe('routed', () => {
     expect(used[0].route).toHaveBeenCalledWith(exchange, key, buffer, undefined)
   })
 })
+
+describe('shards of messages', () => {
+  const queue = generate()
+  const buffer = randomBytes(8)
+
+  /** @type {jest.MockedObject<comq.Channel>[]} */
+  let channels
+
+  beforeEach(async () => {
+    channel = await create(connections, type)
+    channels = await getCreatedChannels()
+  })
+
+  it.each(['send', 'route'])('should tell `%s` which shard a message goes through', async (method) => {
+    const via = /** @type {jest.MockedFunction} */ jest.fn()
+    const args = method === 'send' ? [queue, buffer, {}] : [generate(), queue, buffer, {}]
+
+    await channel[method](...args, via)
+
+    const used = channels.find((chan) => chan[method].mock.calls.length > 0)
+
+    expect(via).toHaveBeenCalledTimes(1)
+    expect(via).toHaveBeenCalledWith(used.index)
+  })
+
+  it('should tell which shard a message goes through before it is published', async () => {
+    const via = /** @type {jest.MockedFunction} */ jest.fn()
+    const publishing = new Promex()
+
+    for (const chan of channels) chan.send.mockImplementationOnce(() => publishing)
+
+    const sending = channel.send(queue, buffer, {}, via)
+
+    await immediate()
+
+    // a shard lost from now on has it on board
+    expect(via).toHaveBeenCalledTimes(1)
+
+    publishing.resolve()
+
+    await sending
+  })
+
+  it('should tell the shard a message fails over to', async () => {
+    const via = /** @type {jest.MockedFunction} */ jest.fn()
+
+    for (const chan of channels) chan.send.mockImplementationOnce(async () => { throw new Error() })
+
+    // both shards fail once, and the one recovered first takes the message
+    setImmediate(() => emitRecover(channels[1]))
+
+    await channel.send(queue, buffer, {}, via)
+
+    const [first, second] = via.mock.calls.map(([index]) => index)
+
+    expect(via).toHaveBeenCalledTimes(3)
+    expect(second).not.toStrictEqual(first)
+    expect(via).toHaveBeenLastCalledWith(1)
+  })
+
+  it.each([0, 1])('should fire a reply through the shard its request arrived on (%i)', async (index) => {
+    const consumer = /** @type {jest.MockedFunction} */ jest.fn()
+
+    await channel.consume(queue, consumer)
+
+    const request = /** @type {comq.amqp.Message} */ {}
+
+    await channels[index].consume.mock.calls[0][1](request)
+
+    for (let i = 0; i < 20; i++) await channel.fire(queue, buffer, {}, request)
+
+    expect(channels[index].fire).toHaveBeenCalledTimes(20)
+    expect(channels[1 - index].fire).not.toHaveBeenCalled()
+  })
+
+  it('should fire a reply through the shard a held request arrived on', async () => {
+    await channel.held(generate(), generate(), generate(), jest.fn())
+
+    const request = /** @type {comq.amqp.Message} */ {}
+
+    await channels[1].held.mock.calls[0][3](request)
+
+    for (let i = 0; i < 20; i++) await channel.fire(queue, buffer, {}, request)
+
+    expect(channels[1].fire).toHaveBeenCalledTimes(20)
+    expect(channels[0].fire).not.toHaveBeenCalled()
+  })
+
+  it('should fire a reply through another shard when its own is lost', async () => {
+    await channel.consume(queue, jest.fn())
+
+    const request = /** @type {comq.amqp.Message} */ {}
+
+    await channels[1].consume.mock.calls[0][1](request)
+
+    lose(connections[1])
+
+    await channel.fire(queue, buffer, {}, request)
+
+    expect(channels[0].fire).toHaveBeenCalledTimes(1)
+    expect(channels[1].fire).not.toHaveBeenCalled()
+
+    restore(connections[1])
+  })
+
+  it('should fire a reply through another shard when its own fails', async () => {
+    await channel.consume(queue, jest.fn())
+
+    const request = /** @type {comq.amqp.Message} */ {}
+
+    await channels[1].consume.mock.calls[0][1](request)
+
+    channels[1].fire.mockImplementationOnce(async () => { throw new Error() })
+
+    await channel.fire(queue, buffer, {}, request)
+
+    expect(channels[1].fire).toHaveBeenCalledTimes(1)
+    expect(channels[0].fire).toHaveBeenCalledTimes(1)
+
+    emitRecover(channels[1])
+  })
+})
+
+/**
+ * @param {jest.MockedObject<comq.Channel>} chan
+ */
+function emitRecover (chan) {
+  const calls = chan.diagnose.mock.calls.filter((call) => call[0] === 'recover')
+
+  for (const call of calls) call[1]()
+}

package/types/channel.d.ts

@@ -25,14 +25,22 @@ declare namespace comq {
     /** consumes `queue`, exclusive to this connection and bound to a routed `exchange` under `key` */
     held (exchange: string, queue: string, key: string, consumer: channels.Consumer): Promise<void>
 
-    send (queue: string, buffer: Buffer, options?: Options.Publish): Promise<void>
+    /**
+     * Over a sharded connection, `via` is told the index of the shard the message is published
+     * through, as soon as it is chosen and again whenever the publish fails over to another.
+     */
+    send (queue: string, buffer: Buffer, options?: Options.Publish, via?: (index: number) => void): Promise<void>
 
     publish (exchange: string, buffer: Buffer, options?: Options.Publish): Promise<void>
 
     /** publishes to a routed exchange under `key`, where `publish` fans out */
-    route (exchange: string, key: string, buffer: Buffer, options?: Options.Publish): Promise<void>
+    route (exchange: string, key: string, buffer: Buffer, options?: Options.Publish, via?: (index: number) => void): Promise<void>
 
-    fire (queue: string, buffer: Buffer, options?: Options.Publish): Promise<boolean>
+    /**
+     * Over a sharded connection, the message goes back through the shard `origin`, the message it
+     * answers, arrived on, while that shard is reachable.
+     */
+    fire (queue: string, buffer: Buffer, options?: Options.Publish, origin?: Message): Promise<boolean>
 
     seal (): Promise<void>
 
@@ -40,7 +48,8 @@ declare namespace comq {
 
     readonly closed: boolean
 
-    diagnose (event: _diagnostics.Event, listener: Function): void
+    /** `reroute` is a sharded channel's own: a returned message is published on another shard */
+    diagnose (event: _diagnostics.Event | 'reroute', listener: Function): void
 
     forget (event: _diagnostics.Event, listener: Function): void
 

package/types/io.d.ts

@@ -35,6 +35,15 @@ declare namespace comq {
     emitter: ReplyEmitter
     properties: _amqp.Properties
     signal?: AbortSignal
+
+    /** The shard the Request is published through, over a sharded connection. */
+    shard?: number
+
+    /** Whether the Request is being published. */
+    publishing?: boolean
+
+    /** Whether its shard was lost while it was being published, which re-sends it once it is. */
+    lost?: boolean
   }
 
   /** How long a caller waits for the Reply to a call. */