comq 0.24.1 → 0.25.1

package/source/promex.js

@@ -0,0 +1,47 @@
+'use strict'
+
+/**
+ * A promise settled by whoever holds it, rather than only from within its executor.
+ * `callback` settles it the way node-style callbacks report: an error, or a value.
+ *
+ * @template T
+ * @extends {Promise<T>}
+ */
+class Promex extends Promise {
+  /** @type {(value?: T | PromiseLike<T>) => void} */
+  resolve
+
+  /** @type {(reason?: any) => void} */
+  reject
+
+  /**
+   * @param {(resolve: (value?: any) => void, reject: (reason?: any) => void) => void} [executor]
+   */
+  constructor (executor = noop) {
+    let ok
+    let oh
+
+    super((resolve, reject) => {
+      ok = resolve
+      oh = reject
+
+      return executor(resolve, reject)
+    })
+
+    this.resolve = ok
+    this.reject = oh
+  }
+
+  /**
+   * @param {any} error
+   * @param {T} [value]
+   */
+  callback = (error, value) => {
+    if (error !== null && error !== undefined) this.reject(error)
+    else this.resolve(value)
+  }
+}
+
+function noop () {}
+
+exports.Promex = Promex

package/source/promex.test.js

@@ -0,0 +1,79 @@
+'use strict'
+
+const { Promex } = require('./promex')
+
+it('should resolve', async () => {
+  const promise = new Promex()
+
+  setImmediate(() => promise.resolve('ok'))
+
+  await expect(promise).resolves.toStrictEqual('ok')
+})
+
+it('should reject', async () => {
+  const promise = new Promex()
+
+  setImmediate(() => promise.reject(new Error('oh')))
+
+  await expect(promise).rejects.toThrow('oh')
+})
+
+it('should resolve using the callback', async () => {
+  const promise = new Promex()
+
+  setImmediate(() => promise.callback(null, 'ok'))
+
+  await expect(promise).resolves.toStrictEqual('ok')
+})
+
+it('should reject using the callback', async () => {
+  const promise = new Promex()
+
+  setImmediate(() => promise.callback('oh'))
+
+  await expect(promise).rejects.toStrictEqual('oh')
+})
+
+it('should resolve using the callback given no error', async () => {
+  const promise = new Promex()
+
+  setImmediate(() => promise.callback(undefined, 'ok'))
+
+  await expect(promise).resolves.toStrictEqual('ok')
+})
+
+it('should run the executor it is given', async () => {
+  const promise = new Promex((resolve) => resolve('ok'))
+
+  await expect(promise).resolves.toStrictEqual('ok')
+})
+
+it('should stay settled by whoever holds it', async () => {
+  const promise = new Promex()
+
+  promise.resolve('first')
+  promise.resolve('second')
+
+  await expect(promise).resolves.toStrictEqual('first')
+})
+
+it('should be a promise', async () => {
+  const promise = new Promex()
+
+  expect(promise).toBeInstanceOf(Promise)
+
+  promise.resolve('ok')
+
+  await expect(Promise.all([promise])).resolves.toStrictEqual(['ok'])
+})
+
+it('should derive promises of its own kind', async () => {
+  const promise = new Promex()
+  const derived = promise.then((value) => value + '!')
+
+  expect(derived).toBeInstanceOf(Promex)
+
+  promise.resolve('ok')
+
+  await expect(derived).resolves.toStrictEqual('ok!')
+})

package/source/retry.js

@@ -0,0 +1,42 @@
+'use strict'
+
+/**
+ * Runs an operation until it stops asking for another attempt, waiting longer before
+ * each one. The operation is handed the token it returns to ask for a retry; anything
+ * else it returns is the result, and whatever it throws is thrown on.
+ *
+ * @template T
+ * @param {(again: symbol) => Promise<T | symbol>} operation
+ * @param {Partial<typeof DEFAULTS>} [options]
+ * @returns {Promise<T>}
+ */
+async function retry (operation, options) {
+  const { attempts, base, max, factor, dispersion } = { ...DEFAULTS, ...options }
+
+  for (let attempt = 0; attempt < attempts; attempt++) {
+    const result = await operation(again)
+
+    if (result !== again) return /** @type {T} */ (result)
+
+    const interval = Math.min(base * factor ** attempt, max)
+
+    // attempts that started together are spread out rather than kept in step
+    const spread = interval * dispersion * (Math.random() - 0.5)
+
+    await new Promise((resolve) => setTimeout(resolve, interval + spread))
+  }
+
+  throw new Error('Maximum attempts exceeded')
+}
+
+const again = Symbol('again')
+
+const DEFAULTS = {
+  attempts: Infinity,
+  base: 1000,
+  max: 30000,
+  factor: 1.5,
+  dispersion: 0.1
+}
+
+exports.retry = retry

package/source/retry.test.js

@@ -0,0 +1,120 @@
+'use strict'
+
+const { retry } = require('./retry')
+
+const immediate = { base: 0, max: 0, dispersion: 0 }
+
+it('should return the result', async () => {
+  const result = await retry(async () => 'ok', immediate)
+
+  expect(result).toStrictEqual('ok')
+})
+
+it('should retry while the operation asks for it', async () => {
+  let attempts = 0
+
+  const result = await retry(async (again) => {
+    attempts++
+
+    return attempts < 3 ? again : 'ok'
+  }, immediate)
+
+  expect(attempts).toStrictEqual(3)
+  expect(result).toStrictEqual('ok')
+})
+
+it('should pass the same token on every attempt', async () => {
+  const tokens = []
+
+  await retry(async (again) => {
+    tokens.push(again)
+
+    return tokens.length < 3 ? again : 'ok'
+  }, immediate)
+
+  expect(tokens).toHaveLength(3)
+  expect(new Set(tokens).size).toStrictEqual(1)
+})
+
+it('should throw what the operation throws', async () => {
+  const exception = new Error('nope')
+
+  await expect(retry(async () => { throw exception }, immediate)).rejects.toThrow(exception)
+})
+
+it('should give up after the last attempt', async () => {
+  let attempts = 0
+
+  const attempt = retry(async (again) => {
+    attempts++
+
+    return again
+  }, { ...immediate, attempts: 3 })
+
+  await expect(attempt).rejects.toThrow('Maximum attempts exceeded')
+  expect(attempts).toStrictEqual(3)
+})
+
+it('should not run the operation without an attempt to spare', async () => {
+  const operation = jest.fn(async (again) => again)
+
+  await expect(retry(operation, { ...immediate, attempts: 0 }))
+    .rejects.toThrow('Maximum attempts exceeded')
+
+  expect(operation).not.toHaveBeenCalled()
+})
+
+it('should back off, up to the maximum, before each attempt', async () => {
+  jest.useFakeTimers()
+
+  try {
+    const waits = []
+
+    let last = Date.now()
+    let attempts = 0
+
+    const attempt = retry(async (again) => {
+      waits.push(Date.now() - last)
+      last = Date.now()
+      attempts++
+
+      return attempts < 5 ? again : 'ok'
+    }, { base: 100, factor: 2, max: 300, dispersion: 0 })
+
+    await jest.advanceTimersByTimeAsync(10_000)
+
+    expect(await attempt).toStrictEqual('ok')
+
+    // the first attempt waits for nothing; then 100, 200, and 400 capped at 300
+    expect(waits).toStrictEqual([0, 100, 200, 300, 300])
+  } finally {
+    jest.useRealTimers()
+  }
+})
+
+it('should spread the wait around the interval', async () => {
+  const waits = []
+
+  const timer = global.setTimeout
+
+  global.setTimeout = (callback, ms) => {
+    waits.push(ms)
+
+    return timer(callback, 0)
+  }
+
+  try {
+    let attempts = 0
+
+    await retry(async (again) => {
+      attempts++
+
+      return attempts < 20 ? again : 'ok'
+    }, { base: 1000, factor: 1, max: 1000, dispersion: 0.1 })
+  } finally {
+    global.setTimeout = timer
+  }
+
+  expect(waits.every((ms) => ms >= 950 && ms <= 1050)).toStrictEqual(true)
+  expect(new Set(waits).size).toBeGreaterThan(1)
+})

package/source/shards/channel.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { Promex } = require('promex')
+const { Promex } = require('../promex')
 const { RETRY_PREFIX } = require('../channel')
 const events = require('../events')
 const emitter = require('../emitter')

package/test/.types/jest.d.ts

@@ -0,0 +1,22 @@
+import * as mock from 'jest-mock'
+
+// The test files are written against Jest's mock types, and keep being: the mocks are still
+// Jest's, taken as the standalone `jest-mock` package. This is what @types/jest declared of it.
+
+declare global {
+
+  namespace jest {
+
+    type Mock<T extends mock.FunctionLike = mock.UnknownFunction> = mock.Mock<T>
+
+    type Mocked<T extends object> = mock.Mocked<T>
+
+    type MockedClass<T extends mock.ClassLike> = mock.MockedClass<T>
+
+    type MockedFunction<T extends mock.FunctionLike = mock.UnknownFunction> = mock.MockedFunction<T>
+
+    type MockedFn<T extends mock.FunctionLike = mock.UnknownFunction> = mock.MockedFunction<T>
+
+    type MockedObject<T extends object> = mock.MockedObject<T>
+  }
+}

package/test/ReplyPipe.test.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const { Readable } = require('node:stream')
-const { Promex } = require('promex')
+const { Promex } = require('../source/promex')
 const { ReplyPipe } = require('../source/.io/ReplyPipe')
 const { createReplyEmitter } = require('../source/.io/createReplyEmitter')
 const { control, FLOW_HEADER } = require('../source/.io/const')
@@ -96,10 +96,14 @@ it('should pull the source no faster than the channel takes', async () => {
 })
 
 it('should hold the source while the consumer has asked for a pause', async () => {
-  const pipe = await ReplyPipe.create(request, Readable.from([1, 2, 3]), channel, feedback, reply)
+  // the pause is asked for while the confirmation is still being published: the pipe does not
+  // read its source until that has been published, so nothing can be in flight yet
+  const opening = ReplyPipe.create(request, Readable.from([1, 2, 3]), channel, feedback, reply)
 
   feedback.emit(request.properties.correlationId, control.pause, {})
 
+  const pipe = await opening
+
   await timeout(10)
 
   expect(messages()).toStrictEqual([control.ok])
@@ -208,3 +212,68 @@ it('should destroy the source if the confirmation cannot be sent', async () => {
 
   expect(source.destroyed).toStrictEqual(true)
 })
+
+describe('batches', () => {
+  beforeEach(() => {
+    request.properties.contentType = 'application/json'
+  })
+
+  it('should send the values its source holds as one message', async () => {
+    const pipe = await ReplyPipe.create(request, Readable.from([1, 2, 3]), channel, feedback, reply)
+
+    await closing(pipe)
+
+    expect(messages()).toStrictEqual([control.ok, [1, 2, 3], control.end])
+    expect(sent[1][1].type).toStrictEqual('batch')
+  })
+
+  it('should send each value that arrives on its own as it arrives', async () => {
+    async function * source () {
+      yield 1
+      await timeout(10)
+      yield 2
+    }
+
+    const pipe = await ReplyPipe.create(request, Readable.from(source()), channel, feedback, reply)
+
+    await closing(pipe)
+
+    expect(messages()).toStrictEqual([control.ok, 1, 2, control.end])
+    expect(sent[1][1].type).toBeUndefined()
+  })
+
+  it('should send buffers as one message, each prefixed with its length', async () => {
+    const source = Readable.from([Buffer.from('ab'), Buffer.from('cde')])
+    const pipe = await ReplyPipe.create(request, source, channel, feedback, reply)
+
+    await closing(pipe)
+
+    const expected = Buffer.from([0, 0, 0, 2, 0x61, 0x62, 0, 0, 0, 3, 0x63, 0x64, 0x65])
+
+    expect(messages()[1]).toStrictEqual(expected)
+    expect(sent[1][1].type).toStrictEqual('buffers')
+  })
+
+  it('should send values one at a time in an encoding that holds no list', async () => {
+    request.properties.contentType = 'text/plain'
+
+    const pipe = await ReplyPipe.create(request, Readable.from(['a', 'b']), channel, feedback, reply)
+
+    await closing(pipe)
+
+    expect(messages()).toStrictEqual([control.ok, 'a', 'b', control.end])
+  })
+
+  it('should keep a batch within its limit', async () => {
+    const values = Array.from({ length: 300 }, (_, i) => i)
+    const source = Readable.from(values, { highWaterMark: 1000 })
+    const pipe = await ReplyPipe.create(request, source, channel, feedback, reply)
+
+    await closing(pipe)
+
+    const batches = messages().slice(1, -1)
+
+    expect(batches.flat()).toStrictEqual(values)
+    expect(Math.max(...batches.map((batch) => [batch].flat().length))).toBeLessThanOrEqual(128)
+  })
+})

package/test/ReplyStream.test.js

@@ -204,3 +204,36 @@ describe('flow', () => {
     }
   })
 })
+
+describe('batches', () => {
+  const ok = { headers: { index: 0, [FLOW_HEADER]: true }, type: 'control' }
+  const end = (index) => [control.end, { headers: { index }, type: 'control' }]
+
+  it('should yield each value of a batch, in order', async () => {
+    stream.arrange(control.ok, ok)
+    stream.arrange([1, 2, 3], { headers: { index: 1 }, type: 'batch' })
+    stream.arrange(4, { headers: { index: 2 } })
+    stream.arrange(...end(3))
+
+    expect(await stream.toArray()).toStrictEqual([1, 2, 3, 4])
+  })
+
+  it('should yield each buffer of a batch of buffers', async () => {
+    const buffers = Buffer.from([0, 0, 0, 2, 0x61, 0x62, 0, 0, 0, 0, 0, 0, 0, 1, 0x63])
+
+    stream.arrange(control.ok, ok)
+    stream.arrange(buffers, { headers: { index: 1 }, type: 'buffers' })
+    stream.arrange(...end(2))
+
+    expect(await stream.toArray()).toStrictEqual([Buffer.from('ab'), Buffer.alloc(0), Buffer.from('c')])
+  })
+
+  it('should hold a batch that arrives out of order until the gap is filled', async () => {
+    stream.arrange(control.ok, ok)
+    stream.arrange([3, 4], { headers: { index: 2 }, type: 'batch' })
+    stream.arrange([1, 2], { headers: { index: 1 }, type: 'batch' })
+    stream.arrange(...end(3))
+
+    expect(await stream.toArray()).toStrictEqual([1, 2, 3, 4])
+  })
+})

package/test/channel.test.js

@@ -1251,6 +1251,31 @@ describe('failed messages', () => {
       expect(chan.ack).toHaveBeenCalled()
     })
 
+    it('should release the original before the channel is given back', async () => {
+      await channel.consume(queue, consumer)
+
+      let confirm
+
+      chan.publish.mockImplementationOnce((_0, _1, _2, _3, callback) => (confirm = callback))
+
+      const delivering = deliver(delivery())
+
+      // closed while the copy is on its way: an original still unacknowledged then is
+      // requeued by the broker, and the message is held twice
+      const closing = channel.close()
+
+      await new Promise((resolve) => setImmediate(resolve))
+
+      confirm(null)
+
+      await closing
+      await delivering
+
+      expect(chan.ack).toHaveBeenCalledTimes(1)
+      expect(chan.ack.mock.invocationCallOrder[0])
+        .toBeLessThan(chan.close.mock.invocationCallOrder[0])
+    })
+
     it('should increment the attempt', async () => {
       await channel.consume(queue, consumer)
 

package/test/connection.test.js

@@ -4,7 +4,7 @@
 
 const { generate } = require('randomstring')
 
-const { Promex } = require('promex')
+const { Promex } = require('../source/promex')
 const { timeout, random } = require('./helpers')
 const { amqplib, connect } = require('./amqplib.mock')
 const { channel: create } = require('./connection.mock')

package/test/globals.js

@@ -0,0 +1,143 @@
+'use strict'
+
+// Jest's globals, on top of node:test.
+//
+// The runner is Node's own; the assertions, the mocks and the fake timers are still Jest's,
+// taken as the standalone packages they are published as. Loaded with `--import`, so that
+// the test files themselves keep seeing `it`, `expect` and `jest` as globals.
+
+const { createRequire } = require('node:module')
+const { format } = require('node:util')
+const nodeTest = require('node:test')
+const { expect } = require('expect')
+const { ModuleMocker } = require('jest-mock')
+const { ModernFakeTimers } = require('@jest/fake-timers')
+
+// node:test drives its own queue with process.nextTick: faking it, as Jest does inside its
+// sandbox, stops the runner mid-file and the tests left behind are never reported as missing
+const DO_NOT_FAKE = ['nextTick']
+
+const mocker = new ModuleMocker(globalThis)
+const timers = new ModernFakeTimers({ global: globalThis, config: { rootDir: process.cwd() } })
+
+/**
+ * Where `jest.mock` was called from, so that its specifier resolves
+ * relative to the test file rather than to this one.
+ *
+ * @returns {string}
+ */
+function origin () {
+  const prepare = Error.prepareStackTrace
+
+  Error.prepareStackTrace = (_, frames) => frames
+
+  const { stack } = new Error('origin')
+
+  Error.prepareStackTrace = prepare
+
+  for (const frame of stack) {
+    const file = frame.getFileName()
+
+    if (file !== null && file !== undefined && file !== __filename && !file.startsWith('node:')) {
+      return file
+    }
+  }
+
+  throw new Error('cannot tell which file has called jest.mock')
+}
+
+/**
+ * Replaces a module, as `jest.mock` does. Without a factory the module is automocked:
+ * every function it exports becomes a mock function.
+ *
+ * @param {string} specifier
+ * @param {() => object} [factory]
+ */
+function mockModule (specifier, factory) {
+  if (nodeTest.mock.module === undefined) {
+    throw new Error('jest.mock requires node --experimental-test-module-mocks')
+  }
+
+  const path = createRequire(origin()).resolve(specifier)
+
+  const namedExports = factory === undefined
+    ? mocker.generateFromMetadata(mocker.getMetadata(require(path)))
+    : factory()
+
+  nodeTest.mock.module(path, { namedExports, cache: false })
+}
+
+const jest = {
+  fn: (implementation) => mocker.fn(implementation),
+  spyOn: (object, property, accessType) => mocker.spyOn(object, property, accessType),
+  clearAllMocks: () => mocker.clearAllMocks(),
+  resetAllMocks: () => mocker.resetAllMocks(),
+  restoreAllMocks: () => mocker.restoreAllMocks(),
+  mock: mockModule,
+  useFakeTimers: (config) => timers.useFakeTimers({ doNotFake: DO_NOT_FAKE, ...config }),
+  useRealTimers: () => timers.useRealTimers(),
+  advanceTimersByTime: (ms) => timers.advanceTimersByTime(ms),
+  advanceTimersByTimeAsync: (ms) => timers.advanceTimersByTimeAsync(ms),
+  runAllTimers: () => timers.runAllTimers(),
+  runAllTimersAsync: () => timers.runAllTimersAsync(),
+  runOnlyPendingTimers: () => timers.runOnlyPendingTimers(),
+  getTimerCount: () => timers.getTimerCount(),
+  setSystemTime: (now) => timers.setSystemTime(now)
+}
+
+/**
+ * `it.each` / `describe.each`: one test per row, the row spread into the callback,
+ * the title formatted with the row's values.
+ *
+ * @param {Function} declare
+ * @returns {(table: any[]) => (title: string, fn: Function, options?: object) => void}
+ */
+const each = (declare) => (table) => (title, fn, options) => {
+  const placeholders = (title.replace(/%%/g, '').match(/%[sdifjoOc]/g) ?? []).length
+
+  for (const row of table) {
+    const values = Array.isArray(row) ? row : [row]
+
+    declare(format(title, ...values.slice(0, placeholders)), options, () => fn(...values))
+  }
+}
+
+/**
+ * Jest's `it` and `describe` carry `.each`; node:test's do not.
+ *
+ * @param {Function} declare
+ * @returns {Function}
+ */
+function extend (declare) {
+  // Jest takes a timeout where node:test takes options
+  const call = (...args) => declare(...(typeof args[2] === 'number'
+    ? [args[0], { timeout: args[2] }, args[1]]
+    : args))
+
+  const extended = Object.assign(call, declare, { each: each(call) })
+
+  extended.only = Object.assign((...args) => declare.only(...args), { each: each(declare.only) })
+  extended.skip = Object.assign((...args) => declare.skip(...args), { each: each(declare.skip) })
+  extended.todo = (...args) => declare.todo(...args)
+
+  return extended
+}
+
+Object.assign(globalThis, {
+  jest,
+  expect,
+  it: extend(nodeTest.it),
+  test: extend(nodeTest.test),
+  describe: extend(nodeTest.describe),
+  beforeEach: nodeTest.beforeEach,
+  afterEach: nodeTest.afterEach,
+  beforeAll: nodeTest.before,
+  afterAll: nodeTest.after
+})
+
+// `expect.assertions` is only enforced by whoever runs the test
+nodeTest.afterEach(() => {
+  const errors = expect.extractExpectedAssertionsErrors()
+
+  if (errors.length > 0) throw errors[0].error
+})

package/test/io.close.test.js

@@ -2,7 +2,7 @@
 
 const { randomBytes } = require('node:crypto')
 const { generate } = require('randomstring')
-const { Promex } = require('promex')
+const { Promex } = require('../source/promex')
 const { immediate } = require('./helpers')
 
 const mock = require('./connection.mock')

package/test/io.request.test.js

@@ -5,7 +5,7 @@
 const stream = require('node:stream')
 const { randomBytes } = require('node:crypto')
 const { generate } = require('randomstring')
-const { Promex } = require('promex')
+const { Promex } = require('../source/promex')
 const { immediate } = require('./helpers')
 const { encode } = require('../source/encode')
 

package/test/shards/channel.test.js

@@ -2,7 +2,7 @@
 
 const { randomBytes } = require('node:crypto')
 const { generate } = require('randomstring')
-const { Promex } = require('promex')
+const { Promex } = require('../../source/promex')
 const { sample, immediate, random } = require('../helpers')
 
 const mock = require('../connection.mock')

package/test/shards/connection.test.js

@@ -1,15 +1,14 @@
 'use strict'
 
 const { generate } = require('randomstring')
-const { Promex } = require('promex')
+const { Promex } = require('../../source/promex')
 const { random, immediate } = require('../helpers')
 
-const { Connection } = require('../../source/shards')
-
 const mock = require('../connection.mock')
 
 jest.mock('../../source/shards/channel')
 
+const { Connection } = require('../../source/shards')
 const { create } = require('../../source/shards/channel')
 
 it('should be', async () => {