comq 0.23.0 → 0.24.1

package/test/shards/channel.test.js

@@ -230,7 +230,17 @@ describe.each(['consume', 'subscribe'])('%s', (method) => {
 
     await channel[method](...args)
 
-    for (const chan of channels) expect(chan[method]).toHaveBeenCalledWith(...args)
+    for (const chan of channels) {
+      const call = chan[method].mock.calls[0]
+      const message = /** @type {comq.amqp.Message} */ {}
+
+      expect(call.slice(0, -1)).toStrictEqual(args.slice(0, -1))
+
+      // a consumed message may be told which shard it arrived on on its way
+      await call.at(-1)(message)
+
+      expect(processor).toHaveBeenCalledWith(message)
+    }
   })
 
   it(`should not resolve ${method} until pending channels are created`, async () => {
@@ -593,6 +603,23 @@ describe('diagnose', () => {
     expect(listener).not.toHaveBeenCalled()
   })
 
+  it('should tell where a returned message is published next', async () => {
+    const listener = /** @type {jest.MockedFunction} */ jest.fn()
+    const index = random(channels.length)
+    const chan = channels[index]
+    const message = returned()
+
+    message.fields.exchange = generate()
+    channel.diagnose('reroute', listener)
+
+    emitReturn(chan, message)
+
+    const next = channels.find((one) => one.route.mock.calls.length > 0)
+
+    expect(next).not.toBe(chan)
+    expect(listener).toHaveBeenCalledWith(message, next.index)
+  })
+
   it('should report a returned retry at once', async () => {
     const listener = /** @type {jest.MockedFunction} */ jest.fn()
     const index = random(channels.length)
@@ -620,7 +647,15 @@ describe('diagnose', () => {
 
     await channel.held(exchange, queue, key, consumer)
 
-    for (const one of channels) expect(one.held).toHaveBeenCalledWith(exchange, queue, key, consumer)
+    for (const one of channels) {
+      expect(one.held).toHaveBeenCalledWith(exchange, queue, key, expect.any(Function))
+
+      const message = /** @type {comq.amqp.Message} */ {}
+
+      await one.held.mock.calls[0][3](message)
+
+      expect(consumer).toHaveBeenCalledWith(message)
+    }
   })
 
   it('should hold a queue once any shard holds it', async () => {
@@ -819,3 +854,134 @@ describe('routed', () => {
     expect(used[0].route).toHaveBeenCalledWith(exchange, key, buffer, undefined)
   })
 })
+
+describe('shards of messages', () => {
+  const queue = generate()
+  const buffer = randomBytes(8)
+
+  /** @type {jest.MockedObject<comq.Channel>[]} */
+  let channels
+
+  beforeEach(async () => {
+    channel = await create(connections, type)
+    channels = await getCreatedChannels()
+  })
+
+  it.each(['send', 'route'])('should tell `%s` which shard a message goes through', async (method) => {
+    const via = /** @type {jest.MockedFunction} */ jest.fn()
+    const args = method === 'send' ? [queue, buffer, {}] : [generate(), queue, buffer, {}]
+
+    await channel[method](...args, via)
+
+    const used = channels.find((chan) => chan[method].mock.calls.length > 0)
+
+    expect(via).toHaveBeenCalledTimes(1)
+    expect(via).toHaveBeenCalledWith(used.index)
+  })
+
+  it('should tell which shard a message goes through before it is published', async () => {
+    const via = /** @type {jest.MockedFunction} */ jest.fn()
+    const publishing = new Promex()
+
+    for (const chan of channels) chan.send.mockImplementationOnce(() => publishing)
+
+    const sending = channel.send(queue, buffer, {}, via)
+
+    await immediate()
+
+    // a shard lost from now on has it on board
+    expect(via).toHaveBeenCalledTimes(1)
+
+    publishing.resolve()
+
+    await sending
+  })
+
+  it('should tell the shard a message fails over to', async () => {
+    const via = /** @type {jest.MockedFunction} */ jest.fn()
+
+    for (const chan of channels) chan.send.mockImplementationOnce(async () => { throw new Error() })
+
+    // both shards fail once, and the one recovered first takes the message
+    setImmediate(() => emitRecover(channels[1]))
+
+    await channel.send(queue, buffer, {}, via)
+
+    const [first, second] = via.mock.calls.map(([index]) => index)
+
+    expect(via).toHaveBeenCalledTimes(3)
+    expect(second).not.toStrictEqual(first)
+    expect(via).toHaveBeenLastCalledWith(1)
+  })
+
+  it.each([0, 1])('should fire a reply through the shard its request arrived on (%i)', async (index) => {
+    const consumer = /** @type {jest.MockedFunction} */ jest.fn()
+
+    await channel.consume(queue, consumer)
+
+    const request = /** @type {comq.amqp.Message} */ {}
+
+    await channels[index].consume.mock.calls[0][1](request)
+
+    for (let i = 0; i < 20; i++) await channel.fire(queue, buffer, {}, request)
+
+    expect(channels[index].fire).toHaveBeenCalledTimes(20)
+    expect(channels[1 - index].fire).not.toHaveBeenCalled()
+  })
+
+  it('should fire a reply through the shard a held request arrived on', async () => {
+    await channel.held(generate(), generate(), generate(), jest.fn())
+
+    const request = /** @type {comq.amqp.Message} */ {}
+
+    await channels[1].held.mock.calls[0][3](request)
+
+    for (let i = 0; i < 20; i++) await channel.fire(queue, buffer, {}, request)
+
+    expect(channels[1].fire).toHaveBeenCalledTimes(20)
+    expect(channels[0].fire).not.toHaveBeenCalled()
+  })
+
+  it('should fire a reply through another shard when its own is lost', async () => {
+    await channel.consume(queue, jest.fn())
+
+    const request = /** @type {comq.amqp.Message} */ {}
+
+    await channels[1].consume.mock.calls[0][1](request)
+
+    lose(connections[1])
+
+    await channel.fire(queue, buffer, {}, request)
+
+    expect(channels[0].fire).toHaveBeenCalledTimes(1)
+    expect(channels[1].fire).not.toHaveBeenCalled()
+
+    restore(connections[1])
+  })
+
+  it('should fire a reply through another shard when its own fails', async () => {
+    await channel.consume(queue, jest.fn())
+
+    const request = /** @type {comq.amqp.Message} */ {}
+
+    await channels[1].consume.mock.calls[0][1](request)
+
+    channels[1].fire.mockImplementationOnce(async () => { throw new Error() })
+
+    await channel.fire(queue, buffer, {}, request)
+
+    expect(channels[1].fire).toHaveBeenCalledTimes(1)
+    expect(channels[0].fire).toHaveBeenCalledTimes(1)
+
+    emitRecover(channels[1])
+  })
+})
+
+/**
+ * @param {jest.MockedObject<comq.Channel>} chan
+ */
+function emitRecover (chan) {
+  const calls = chan.diagnose.mock.calls.filter((call) => call[0] === 'recover')
+
+  for (const call of calls) call[1]()
+}

package/types/amqp.d.ts

@@ -1,4 +1,4 @@
-import * as amqplib from 'amqplib'
+import * as amqplib from '@toa.io/amqplib'
 
 declare namespace comq.amqp {
 

package/types/channel.d.ts

@@ -1,4 +1,4 @@
-import { Message, Options, Connection } from 'amqplib'
+import { Message, Options, Connection } from '@toa.io/amqplib'
 import * as _diagnostics from './diagnostic'
 
 declare namespace comq {
@@ -25,14 +25,22 @@ declare namespace comq {
     /** consumes `queue`, exclusive to this connection and bound to a routed `exchange` under `key` */
     held (exchange: string, queue: string, key: string, consumer: channels.Consumer): Promise<void>
 
-    send (queue: string, buffer: Buffer, options?: Options.Publish): Promise<void>
+    /**
+     * Over a sharded connection, `via` is told the index of the shard the message is published
+     * through, as soon as it is chosen and again whenever the publish fails over to another.
+     */
+    send (queue: string, buffer: Buffer, options?: Options.Publish, via?: (index: number) => void): Promise<void>
 
     publish (exchange: string, buffer: Buffer, options?: Options.Publish): Promise<void>
 
     /** publishes to a routed exchange under `key`, where `publish` fans out */
-    route (exchange: string, key: string, buffer: Buffer, options?: Options.Publish): Promise<void>
+    route (exchange: string, key: string, buffer: Buffer, options?: Options.Publish, via?: (index: number) => void): Promise<void>
 
-    fire (queue: string, buffer: Buffer, options?: Options.Publish): Promise<boolean>
+    /**
+     * Over a sharded connection, the message goes back through the shard `origin`, the message it
+     * answers, arrived on, while that shard is reachable.
+     */
+    fire (queue: string, buffer: Buffer, options?: Options.Publish, origin?: Message): Promise<boolean>
 
     seal (): Promise<void>
 
@@ -40,7 +48,8 @@ declare namespace comq {
 
     readonly closed: boolean
 
-    diagnose (event: _diagnostics.Event, listener: Function): void
+    /** `reroute` is a sharded channel's own: a returned message is published on another shard */
+    diagnose (event: _diagnostics.Event | 'reroute', listener: Function): void
 
     forget (event: _diagnostics.Event, listener: Function): void
 

package/types/io.d.ts

@@ -35,21 +35,30 @@ declare namespace comq {
     emitter: ReplyEmitter
     properties: _amqp.Properties
     signal?: AbortSignal
+
+    /** The shard the Request is published through, over a sharded connection. */
+    shard?: number
+
+    /** Whether the Request is being published. */
+    publishing?: boolean
+
+    /** Whether its shard was lost while it was being published, which re-sends it once it is. */
+    lost?: boolean
   }
 
-  /** How long a caller waits for a Reply. */
-  interface RequestOptions {
+  /** How long a caller waits for the Reply to a call. */
+  interface CallOptions {
     encoding?: _encoding.Encoding
 
     /**
-     * Milliseconds to wait for the Reply. A Request nobody has taken by then is dropped by the
+     * Milliseconds to wait for the Reply. A call nobody has taken by then is dropped by the
      * broker; one already taken may still be processed, and its Reply is discarded.
      */
     timeout?: number
 
     /**
-     * Ends the wait when aborted, within the timeout. The Request stays in its queue until the
-     * timeout passes, or, without one, until a Producer takes it.
+     * Ends the wait when aborted, within the timeout. The call stays in its queue until the
+     * timeout passes, or, without one, until its holder takes it.
      */
     signal?: AbortSignal
   }
@@ -64,15 +73,16 @@ declare namespace comq {
   interface IO extends _diagnostics.Diagnosable {
     reply(queue: string, produce: Producer): Promise<void>
 
-    request<Reply = any, Request = any>(queue: string, payload: Request, options?: _encoding.Encoding | RequestOptions): Promise<Reply> | Promise<Readable>
+    /** Waits for the Reply for as long as it takes. */
+    request<Reply = any, Request = any>(queue: string, payload: Request, encoding?: _encoding.Encoding): Promise<Reply> | Promise<Readable>
 
-    request(queue: string, stream: Readable, options?: _encoding.Encoding | RequestOptions): Promise<Readable>
+    request(queue: string, stream: Readable, encoding?: _encoding.Encoding): Promise<Readable>
 
     /**
      * Sends a Request to whoever holds `key` on the routed `exchange`, through `back`. One that
      * nobody holds is refused with `Unroutable`.
      */
-    call<Reply = any, Request = any>(exchange: string, key: string, payload: Request, options?: _encoding.Encoding | RequestOptions): Promise<Reply> | Promise<Readable>
+    call<Reply = any, Request = any>(exchange: string, key: string, payload: Request, options?: _encoding.Encoding | CallOptions): Promise<Reply> | Promise<Readable>
 
     /**
      * Answers the Requests `call` sends under `key`, from a queue this connection alone holds.

package/features/rpc.timeout.feature

@@ -1,19 +0,0 @@
-Feature: Request timeout
-
-  A caller may say how long it waits for a reply. A request nobody took by then is dropped by the
-  broker; one already taken is still processed, and its reply is discarded.
-
-  Background:
-    Given an active connection to the broker
-
-  Scenario: A caller stops waiting at its timeout
-    Given a producer replying `slow` queue in 1000ms
-    When the consumer sends a request to the `slow` queue with a 200ms timeout
-    Then the consumer stops waiting
-
-  Scenario: A request nobody took in time is never processed
-    When the consumer sends a request to the `later` queue with a 200ms timeout
-    Then the consumer stops waiting
-    When after 300ms
-    And a producer counting requests to the `later` queue
-    Then the producer receives nothing

package/source/batch.js

@@ -1,62 +0,0 @@
-'use strict'
-
-/**
- * Writes as one what a connection sends in one turn of the event loop.
- *
- * amqplib hands its frames to the socket one write at a time, and a request served is two of
- * them: the reply, and the acknowledgement of the request it answers. With Nagle's algorithm off,
- * which is what a socket sending without delay runs under, each is a syscall and a packet of its
- * own. Corked, they are held in the socket's buffer and leave together, in one `writev`, at the
- * end of the same turn: nothing waits for a timer, and nothing waits for a write that may never
- * come.
- *
- * This belongs in amqplib, whose `Mux` writes the frames one at a time, and where corking that
- * loop is six lines and needs no wrapping of a socket the library does not own. It is done here
- * until amqplib carries it, and goes when it does.
- *
- * @param {import('node:net').Socket} socket
- */
-function batch (socket) {
-  // whatever a connection is carried over is left as it is unless it is a stream that corks
-  if (!corkable(socket) || socket[BATCHED] === true) return
-
-  socket[BATCHED] = true
-
-  const write = socket.write.bind(socket)
-
-  const flush = () => {
-    socket[CORKED] = false
-    socket.uncork()
-  }
-
-  const cork = () => {
-    if (socket[CORKED] === true) return
-
-    socket[CORKED] = true
-    socket.cork()
-
-    process.nextTick(flush)
-  }
-
-  socket.write = (...args) => {
-    cork()
-
-    return write(...args)
-  }
-}
-
-/**
- * @param {any} socket
- * @returns {boolean}
- */
-function corkable (socket) {
-  return socket !== null && socket !== undefined &&
-    typeof socket.write === 'function' &&
-    typeof socket.cork === 'function' &&
-    typeof socket.uncork === 'function'
-}
-
-const BATCHED = Symbol('comq.batched')
-const CORKED = Symbol('comq.corked')
-
-exports.batch = batch

package/test/batch.test.js

@@ -1,94 +0,0 @@
-'use strict'
-
-const { Writable } = require('node:stream')
-
-const { batch } = require('../source/batch')
-
-/** A socket, as far as writing to it goes: what it was given, and how many times. */
-function socket () {
-  const writes = []
-
-  return Object.assign(new Writable({
-    write (chunk, _encoding, callback) {
-      writes.push([chunk.toString()])
-      callback()
-    },
-    writev (chunks, callback) {
-      writes.push(chunks.map(({ chunk }) => chunk.toString()))
-      callback()
-    }
-  }), { writes })
-}
-
-const tick = () => new Promise((resolve) => process.nextTick(resolve))
-
-it('should be', async () => {
-  expect(batch).toBeDefined()
-})
-
-it('should write what a turn wrote as one', async () => {
-  const stream = socket()
-
-  batch(stream)
-
-  stream.write('a')
-  stream.write('b')
-
-  await tick()
-
-  expect(stream.writes).toStrictEqual([['a', 'b']])
-})
-
-it('should write what a later turn wrote on its own', async () => {
-  const stream = socket()
-
-  batch(stream)
-
-  stream.write('a')
-
-  await tick()
-
-  stream.write('b')
-
-  await tick()
-
-  expect(stream.writes).toStrictEqual([['a'], ['b']])
-})
-
-it('should not hold a write past its turn', async () => {
-  const stream = socket()
-
-  batch(stream)
-
-  stream.write('a')
-
-  await tick()
-
-  expect(stream.writes).toStrictEqual([['a']])
-})
-
-it('should take a socket once', async () => {
-  const stream = socket()
-
-  batch(stream)
-
-  const wrapped = stream.write
-
-  batch(stream)
-
-  expect(stream.write).toStrictEqual(wrapped)
-})
-
-it('should leave alone what does not cork', async () => {
-  for (const value of [undefined, null, {}, { write: () => undefined }]) {
-    expect(() => batch(value)).not.toThrow()
-  }
-
-  const written = []
-  const plain = { write: (chunk) => written.push(chunk) }
-
-  batch(plain)
-  plain.write('a')
-
-  expect(written).toStrictEqual(['a'])
-})