comq 0.24.0 → 0.25.0

package/test/ReplyPipe.test.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const { Readable } = require('node:stream')
-const { Promex } = require('promex')
+const { Promex } = require('../source/promex')
 const { ReplyPipe } = require('../source/.io/ReplyPipe')
 const { createReplyEmitter } = require('../source/.io/createReplyEmitter')
 const { control, FLOW_HEADER } = require('../source/.io/const')
@@ -96,10 +96,14 @@ it('should pull the source no faster than the channel takes', async () => {
 })
 
 it('should hold the source while the consumer has asked for a pause', async () => {
-  const pipe = await ReplyPipe.create(request, Readable.from([1, 2, 3]), channel, feedback, reply)
+  // the pause is asked for while the confirmation is still being published: the pipe does not
+  // read its source until that has been published, so nothing can be in flight yet
+  const opening = ReplyPipe.create(request, Readable.from([1, 2, 3]), channel, feedback, reply)
 
   feedback.emit(request.properties.correlationId, control.pause, {})
 
+  const pipe = await opening
+
   await timeout(10)
 
   expect(messages()).toStrictEqual([control.ok])
@@ -208,3 +212,68 @@ it('should destroy the source if the confirmation cannot be sent', async () => {
 
   expect(source.destroyed).toStrictEqual(true)
 })
+
+describe('batches', () => {
+  beforeEach(() => {
+    request.properties.contentType = 'application/json'
+  })
+
+  it('should send the values its source holds as one message', async () => {
+    const pipe = await ReplyPipe.create(request, Readable.from([1, 2, 3]), channel, feedback, reply)
+
+    await closing(pipe)
+
+    expect(messages()).toStrictEqual([control.ok, [1, 2, 3], control.end])
+    expect(sent[1][1].type).toStrictEqual('batch')
+  })
+
+  it('should send each value that arrives on its own as it arrives', async () => {
+    async function * source () {
+      yield 1
+      await timeout(10)
+      yield 2
+    }
+
+    const pipe = await ReplyPipe.create(request, Readable.from(source()), channel, feedback, reply)
+
+    await closing(pipe)
+
+    expect(messages()).toStrictEqual([control.ok, 1, 2, control.end])
+    expect(sent[1][1].type).toBeUndefined()
+  })
+
+  it('should send buffers as one message, each prefixed with its length', async () => {
+    const source = Readable.from([Buffer.from('ab'), Buffer.from('cde')])
+    const pipe = await ReplyPipe.create(request, source, channel, feedback, reply)
+
+    await closing(pipe)
+
+    const expected = Buffer.from([0, 0, 0, 2, 0x61, 0x62, 0, 0, 0, 3, 0x63, 0x64, 0x65])
+
+    expect(messages()[1]).toStrictEqual(expected)
+    expect(sent[1][1].type).toStrictEqual('buffers')
+  })
+
+  it('should send values one at a time in an encoding that holds no list', async () => {
+    request.properties.contentType = 'text/plain'
+
+    const pipe = await ReplyPipe.create(request, Readable.from(['a', 'b']), channel, feedback, reply)
+
+    await closing(pipe)
+
+    expect(messages()).toStrictEqual([control.ok, 'a', 'b', control.end])
+  })
+
+  it('should keep a batch within its limit', async () => {
+    const values = Array.from({ length: 300 }, (_, i) => i)
+    const source = Readable.from(values, { highWaterMark: 1000 })
+    const pipe = await ReplyPipe.create(request, source, channel, feedback, reply)
+
+    await closing(pipe)
+
+    const batches = messages().slice(1, -1)
+
+    expect(batches.flat()).toStrictEqual(values)
+    expect(Math.max(...batches.map((batch) => [batch].flat().length))).toBeLessThanOrEqual(128)
+  })
+})

package/test/ReplyStream.test.js

@@ -204,3 +204,36 @@ describe('flow', () => {
     }
   })
 })
+
+describe('batches', () => {
+  const ok = { headers: { index: 0, [FLOW_HEADER]: true }, type: 'control' }
+  const end = (index) => [control.end, { headers: { index }, type: 'control' }]
+
+  it('should yield each value of a batch, in order', async () => {
+    stream.arrange(control.ok, ok)
+    stream.arrange([1, 2, 3], { headers: { index: 1 }, type: 'batch' })
+    stream.arrange(4, { headers: { index: 2 } })
+    stream.arrange(...end(3))
+
+    expect(await stream.toArray()).toStrictEqual([1, 2, 3, 4])
+  })
+
+  it('should yield each buffer of a batch of buffers', async () => {
+    const buffers = Buffer.from([0, 0, 0, 2, 0x61, 0x62, 0, 0, 0, 0, 0, 0, 0, 1, 0x63])
+
+    stream.arrange(control.ok, ok)
+    stream.arrange(buffers, { headers: { index: 1 }, type: 'buffers' })
+    stream.arrange(...end(2))
+
+    expect(await stream.toArray()).toStrictEqual([Buffer.from('ab'), Buffer.alloc(0), Buffer.from('c')])
+  })
+
+  it('should hold a batch that arrives out of order until the gap is filled', async () => {
+    stream.arrange(control.ok, ok)
+    stream.arrange([3, 4], { headers: { index: 2 }, type: 'batch' })
+    stream.arrange([1, 2], { headers: { index: 1 }, type: 'batch' })
+    stream.arrange(...end(3))
+
+    expect(await stream.toArray()).toStrictEqual([1, 2, 3, 4])
+  })
+})

package/test/connection.mock.js

@@ -6,13 +6,14 @@ const channel = /** @type {jest.MockedFunction<(sharded?: boolean, index?: numbe
     sharded,
     consume: jest.fn(async () => undefined),
     deliver: jest.fn(async () => undefined),
-    send: jest.fn(async () => undefined),
+    // a sharded channel tells which shard a message goes through, the first one unless told otherwise
+    send: jest.fn(async (_queue, _buffer, _options, via) => { if (sharded) via?.(0) }),
     fire: jest.fn(async () => undefined),
     subscribe: jest.fn(async () => undefined),
     bound: jest.fn(async () => undefined),
     held: jest.fn(async () => undefined),
     publish: jest.fn(async () => undefined),
-    route: jest.fn(async () => undefined),
+    route: jest.fn(async (_exchange, _key, _buffer, _options, via) => { if (sharded) via?.(0) }),
     diagnose: jest.fn(async () => undefined),
     forget: jest.fn(() => undefined),
     seal: jest.fn(async () => undefined),

package/test/connection.test.js

@@ -4,7 +4,7 @@
 
 const { generate } = require('randomstring')
 
-const { Promex } = require('promex')
+const { Promex } = require('../source/promex')
 const { timeout, random } = require('./helpers')
 const { amqplib, connect } = require('./amqplib.mock')
 const { channel: create } = require('./connection.mock')
@@ -543,109 +543,47 @@ describe('channel exhaustion', () => {
   })
 })
 
-describe('watchdog', () => {
+describe('silence', () => {
   afterEach(() => {
     jest.useRealTimers()
   })
 
-  it('should destroy silent connection when watchdog expires', async () => {
+  // amqplib reads the socket through a callback rather than 'data' events, and tells a
+  // connection gone silent by its heartbeat
+  it('should keep an idle connection', async () => {
     jest.useFakeTimers()
 
     await connection.open()
 
     const conn = await amqplib.connect.mock.results[0].value
 
-    await jest.advanceTimersByTimeAsync(46_000)
-
-    expect(conn.connection.stream.destroy).toHaveBeenCalled()
-  })
-
-  it('should reconnect after the watchdog destroys a silent connection', async () => {
-    jest.useFakeTimers()
-
-    await connection.open()
-
-    await jest.advanceTimersByTimeAsync(46_000)
-
-    // a socket destroyed without an error is a socket amqplib never reports,
-    // which leaves the connection silently dead instead of recovering
-    expect(amqplib.connect).toHaveBeenCalledTimes(2)
-  })
-
-  it('should not let a replaced socket disarm the watchdog', async () => {
-    jest.useFakeTimers()
-
-    await connection.open()
-
-    const stale = await amqplib.connect.mock.results[0].value
-
-    stale.emit('close', new Error('lost'))
-
-    await jest.advanceTimersByTimeAsync(1)
-
-    const live = await amqplib.connect.mock.results[1].value
-
-    await jest.advanceTimersByTimeAsync(30_000)
-
-    stale.connection.stream.emit('data', Buffer.alloc(0))
-
-    await jest.advanceTimersByTimeAsync(30_000)
-
-    expect(live.connection.stream.destroy).toHaveBeenCalled()
-  })
-
-  it('should reset watchdog on socket data', async () => {
-    jest.useFakeTimers()
-
-    await connection.open()
-
-    const conn = await amqplib.connect.mock.results[0].value
-
-    await jest.advanceTimersByTimeAsync(40_000)
-
-    conn.connection.stream.emit('data', Buffer.alloc(0))
-
-    await jest.advanceTimersByTimeAsync(40_000)
+    await jest.advanceTimersByTimeAsync(600_000)
 
     expect(conn.connection.stream.destroy).not.toHaveBeenCalled()
-
-    await jest.advanceTimersByTimeAsync(10_000)
-
-    expect(conn.connection.stream.destroy).toHaveBeenCalled()
+    expect(amqplib.connect).toHaveBeenCalledTimes(1)
   })
 
-  it('should derive the watchdog from the negotiated heartbeat', async () => {
-    jest.useFakeTimers()
-
-    const connection = new Connection('amqp://localhost?heartbeat=30')
-
+  it('should destroy the socket of a connection lost with an error', async () => {
     await connection.open()
 
     const conn = await amqplib.connect.mock.results[0].value
+    const stream = conn.connection.stream
 
-    // the broker is only expected to say something every 15 seconds
-    await jest.advanceTimersByTimeAsync(60_000)
-
-    expect(conn.connection.stream.destroy).not.toHaveBeenCalled()
-
-    await jest.advanceTimersByTimeAsync(31_000)
+    conn.emit('close', new Error('Heartbeat timeout'))
 
-    expect(conn.connection.stream.destroy).toHaveBeenCalled()
+    expect(stream.destroy).toHaveBeenCalled()
   })
 
-  it('should not arm the watchdog when heartbeats are disabled', async () => {
-    jest.useFakeTimers()
-
-    const connection = new Connection('amqp://localhost?heartbeat=0')
-
+  it('should reconnect once a connection is lost with an error', async () => {
     await connection.open()
 
     const conn = await amqplib.connect.mock.results[0].value
 
-    // an idle connection that has agreed to no heartbeats is silent by design
-    await jest.advanceTimersByTimeAsync(600_000)
+    conn.emit('close', new Error('Heartbeat timeout'))
 
-    expect(conn.connection.stream.destroy).not.toHaveBeenCalled()
+    await timeout(0)
+
+    expect(amqplib.connect).toHaveBeenCalledTimes(2)
   })
 })
 

package/test/globals.js

@@ -0,0 +1,143 @@
+'use strict'
+
+// Jest's globals, on top of node:test.
+//
+// The runner is Node's own; the assertions, the mocks and the fake timers are still Jest's,
+// taken as the standalone packages they are published as. Loaded with `--import`, so that
+// the test files themselves keep seeing `it`, `expect` and `jest` as globals.
+
+const { createRequire } = require('node:module')
+const { format } = require('node:util')
+const nodeTest = require('node:test')
+const { expect } = require('expect')
+const { ModuleMocker } = require('jest-mock')
+const { ModernFakeTimers } = require('@jest/fake-timers')
+
+// node:test drives its own queue with process.nextTick: faking it, as Jest does inside its
+// sandbox, stops the runner mid-file and the tests left behind are never reported as missing
+const DO_NOT_FAKE = ['nextTick']
+
+const mocker = new ModuleMocker(globalThis)
+const timers = new ModernFakeTimers({ global: globalThis, config: { rootDir: process.cwd() } })
+
+/**
+ * Where `jest.mock` was called from, so that its specifier resolves
+ * relative to the test file rather than to this one.
+ *
+ * @returns {string}
+ */
+function origin () {
+  const prepare = Error.prepareStackTrace
+
+  Error.prepareStackTrace = (_, frames) => frames
+
+  const { stack } = new Error('origin')
+
+  Error.prepareStackTrace = prepare
+
+  for (const frame of stack) {
+    const file = frame.getFileName()
+
+    if (file !== null && file !== undefined && file !== __filename && !file.startsWith('node:')) {
+      return file
+    }
+  }
+
+  throw new Error('cannot tell which file has called jest.mock')
+}
+
+/**
+ * Replaces a module, as `jest.mock` does. Without a factory the module is automocked:
+ * every function it exports becomes a mock function.
+ *
+ * @param {string} specifier
+ * @param {() => object} [factory]
+ */
+function mockModule (specifier, factory) {
+  if (nodeTest.mock.module === undefined) {
+    throw new Error('jest.mock requires node --experimental-test-module-mocks')
+  }
+
+  const path = createRequire(origin()).resolve(specifier)
+
+  const namedExports = factory === undefined
+    ? mocker.generateFromMetadata(mocker.getMetadata(require(path)))
+    : factory()
+
+  nodeTest.mock.module(path, { namedExports, cache: false })
+}
+
+const jest = {
+  fn: (implementation) => mocker.fn(implementation),
+  spyOn: (object, property, accessType) => mocker.spyOn(object, property, accessType),
+  clearAllMocks: () => mocker.clearAllMocks(),
+  resetAllMocks: () => mocker.resetAllMocks(),
+  restoreAllMocks: () => mocker.restoreAllMocks(),
+  mock: mockModule,
+  useFakeTimers: (config) => timers.useFakeTimers({ doNotFake: DO_NOT_FAKE, ...config }),
+  useRealTimers: () => timers.useRealTimers(),
+  advanceTimersByTime: (ms) => timers.advanceTimersByTime(ms),
+  advanceTimersByTimeAsync: (ms) => timers.advanceTimersByTimeAsync(ms),
+  runAllTimers: () => timers.runAllTimers(),
+  runAllTimersAsync: () => timers.runAllTimersAsync(),
+  runOnlyPendingTimers: () => timers.runOnlyPendingTimers(),
+  getTimerCount: () => timers.getTimerCount(),
+  setSystemTime: (now) => timers.setSystemTime(now)
+}
+
+/**
+ * `it.each` / `describe.each`: one test per row, the row spread into the callback,
+ * the title formatted with the row's values.
+ *
+ * @param {Function} declare
+ * @returns {(table: any[]) => (title: string, fn: Function, options?: object) => void}
+ */
+const each = (declare) => (table) => (title, fn, options) => {
+  const placeholders = (title.replace(/%%/g, '').match(/%[sdifjoOc]/g) ?? []).length
+
+  for (const row of table) {
+    const values = Array.isArray(row) ? row : [row]
+
+    declare(format(title, ...values.slice(0, placeholders)), options, () => fn(...values))
+  }
+}
+
+/**
+ * Jest's `it` and `describe` carry `.each`; node:test's do not.
+ *
+ * @param {Function} declare
+ * @returns {Function}
+ */
+function extend (declare) {
+  // Jest takes a timeout where node:test takes options
+  const call = (...args) => declare(...(typeof args[2] === 'number'
+    ? [args[0], { timeout: args[2] }, args[1]]
+    : args))
+
+  const extended = Object.assign(call, declare, { each: each(call) })
+
+  extended.only = Object.assign((...args) => declare.only(...args), { each: each(declare.only) })
+  extended.skip = Object.assign((...args) => declare.skip(...args), { each: each(declare.skip) })
+  extended.todo = (...args) => declare.todo(...args)
+
+  return extended
+}
+
+Object.assign(globalThis, {
+  jest,
+  expect,
+  it: extend(nodeTest.it),
+  test: extend(nodeTest.test),
+  describe: extend(nodeTest.describe),
+  beforeEach: nodeTest.beforeEach,
+  afterEach: nodeTest.afterEach,
+  beforeAll: nodeTest.before,
+  afterAll: nodeTest.after
+})
+
+// `expect.assertions` is only enforced by whoever runs the test
+nodeTest.afterEach(() => {
+  const errors = expect.extractExpectedAssertionsErrors()
+
+  if (errors.length > 0) throw errors[0].error
+})

package/test/io.call.test.js

@@ -52,7 +52,7 @@ describe('call', () => {
 
   it('should publish to the exchange under the key', async () => {
     expect(requests.route).toHaveBeenCalledWith(exchange, key, expect.any(Buffer),
-      expect.objectContaining({ mandatory: true }))
+      expect.objectContaining({ mandatory: true }), expect.any(Function))
   })
 
   it('should declare no queue to publish to', async () => {
@@ -116,14 +116,16 @@ describe('back', () => {
     const contentType = 'application/json'
     const properties = { contentType, correlationId: generate(), replyTo: generate() }
 
-    await consumer({ content: encode(payload, contentType), properties })
+    const request = { content: encode(payload, contentType), properties }
+
+    await consumer(request)
 
     replies = await findChannel('reply')
 
     expect(producer).toHaveBeenCalledWith(payload)
 
     expect(replies.fire).toHaveBeenCalledWith(properties.replyTo, expect.any(Buffer),
-      expect.objectContaining({ correlationId: properties.correlationId }))
+      expect.objectContaining({ correlationId: properties.correlationId }), request)
   })
 })
 

package/test/io.close.test.js

@@ -2,7 +2,7 @@
 
 const { randomBytes } = require('node:crypto')
 const { generate } = require('randomstring')
-const { Promex } = require('promex')
+const { Promex } = require('../source/promex')
 const { immediate } = require('./helpers')
 
 const mock = require('./connection.mock')

package/test/io.reply.test.js

@@ -98,7 +98,8 @@ describe('reply', () => {
     const reply = await produce.mock.results[0].value
     const buffer = encode(reply, properties.contentType)
 
-    expect(replies.fire).toHaveBeenCalledWith(properties.replyTo, buffer, expect.anything())
+    // the request goes along, so that the reply goes back the way it came
+    expect(replies.fire).toHaveBeenCalledWith(properties.replyTo, buffer, expect.anything(), message)
   })
 
   it('should throw if producer returned undefined', async () => {
@@ -190,7 +191,7 @@ describe('encoding', () => {
       mandatory: true
     }
 
-    expect(replies.fire).toHaveBeenCalledWith(properties.replyTo, reply, props)
+    expect(replies.fire).toHaveBeenCalledWith(properties.replyTo, reply, props, message)
   })
 
   it('should set octet-stream for Buffer reply', async () => {
@@ -215,7 +216,8 @@ describe('encoding', () => {
       .toHaveBeenCalledWith(
         expect.any(String),
         buffer,
-        expect.objectContaining({ contentType: 'application/octet-stream' })
+        expect.objectContaining({ contentType: 'application/octet-stream' }),
+        message
       )
   })