comq 0.19.0 → 0.20.1

package/features/rpc.call.feature

@@ -0,0 +1,71 @@
+Feature: Calls under a key
+
+  A call reaches the one connection holding its key. A call under a key nobody holds is refused
+  at once, and a holder that stops withdraws its key before it finishes what it holds.
+
+  Background:
+    Given an active connection to the broker
+
+  Scenario: Answered by the holder of the key
+    Given a holder answering `sums` under the `a` key:
+      """
+      ({ a, b }) => a + b
+      """
+    When the consumer calls `sums` under the `a` key with:
+      """yaml
+      a: 1
+      b: 2
+      """
+    Then the consumer receives the reply:
+      """yaml
+      3
+      """
+
+  Scenario: Answered by the holder of that key only
+    Given a holder answering `names` under the `a` key:
+      """
+      () => 'a'
+      """
+    And a holder answering `names` under the `b` key:
+      """
+      () => 'b'
+      """
+    When the consumer calls `names` under the `b` key
+    Then the consumer receives the reply:
+      """yaml
+      b
+      """
+
+  Scenario: Refused when nobody holds the key
+    Given a holder answering `names` under the `a` key:
+      """
+      () => 'a'
+      """
+    When the consumer calls `names` under the `c` key
+    Then the call is refused as unroutable
+
+  Scenario: A sealed holder answers what it holds and refuses what follows
+    Given a holder on another connection answering `slow` under the `a` key in 500ms
+    When the consumer calls `slow` under the `a` key
+    And after 100ms
+    And the holder is sealed
+    Then the consumer receives the reply
+    When the consumer calls `slow` under the `a` key
+    Then the call is refused as unroutable
+
+  Scenario: A second holder waits for the key
+    Given a holder on another connection answering `echo` under the `a` key
+    When a second holder on another connection holds `echo` under the `a` key
+    Then the second holder finds the key taken
+    When the first holder disconnects
+    Then the second holder holds the key
+    When the consumer calls `echo` under the `a` key
+    Then the consumer receives the reply
+
+  Scenario: A caller stops waiting for a holder that crashed
+    Given a holder never answering `void` under the `a` key
+    When the consumer calls `void` under the `a` key with a 1000ms timeout
+    And after 100ms
+    And the silent holder crashes
+    Then the consumer stops waiting
+    And a call to `void` under the `a` key is refused within 10 seconds

package/features/rpc.call.recovery.feature

@@ -0,0 +1,75 @@
+@heavy
+Feature: Calls under a key across failures
+
+  A holder that loses its connection holds its key again once the connection is restored, and a
+  caller's timeout ends its wait whatever the broker is doing.
+
+  Scenario: A holder holds its key again after the broker restarts
+    Given an active connection to the broker
+    And a holder on another connection answering `echo` under the `a` key
+    When the broker has crashed
+    And the broker is up
+    Then a call to `echo` under the `a` key is answered within 60 seconds
+
+  Scenario: A holder holds its key again once the broker lets its silent connection go
+
+    The holder reconnects while the broker still holds the connection that went silent, and with
+    it the key. The holder finds the key taken until the broker lets that connection go, and
+    claims it then.
+
+    Given watchdog interval is set to 2000ms with 60s AMQP heartbeat
+    And a network that can go silent
+    And an active connection to the broker
+    And a holder answering `echo` under the `a` key:
+      """
+      (payload) => payload
+      """
+    When the network goes silent
+    Then the connection is lost within 10 seconds
+    And the holder finds its key taken while the broker holds the silent connection
+    When the silent connection is let go
+    Then a call to `echo` under the `a` key from another connection is answered within 60 seconds
+
+  Scenario: A call in flight when the broker crashes ends
+    Given an active connection to the broker
+    And a holder on another connection answering `slow` under the `a` key in 2000ms
+    When the consumer calls `slow` under the `a` key
+    And after 200ms
+    And the broker has crashed
+    And the broker is up
+    Then the call ends
+
+  Scenario: A caller stops waiting for a request at its timeout while the broker is down
+    Given an active connection to the broker
+    And a producer replying `echo` queue
+    When the broker has crashed
+    And the consumer sends a request to the `echo` queue with a 1000ms timeout
+    Then the consumer stops waiting within 5 seconds
+
+  Scenario: A caller stops waiting for a call at its timeout while the broker is down
+    Given an active connection to the broker
+    And a holder on another connection answering `echo` under the `a` key
+    When the broker has crashed
+    And the consumer calls `echo` under the `a` key with a 1000ms timeout
+    Then the consumer stops waiting within 5 seconds
+
+  Scenario: A request re-sent after the broker restarts expires at its caller's deadline
+
+    The request is lost with the broker and sent again once the connection is restored, carrying
+    the time its caller has left.
+
+    Given an active connection to the broker
+    When the consumer sends a request to the `resent` queue with a 30000ms timeout
+    And after 200ms
+    And the broker has crashed
+    And the broker is up
+    Then the consumer stops waiting within 40 seconds
+    When a producer counting requests to the `resent` queue
+    Then the producer receives nothing
+
+  Scenario: A call reaches its holder while one of the brokers is down
+    Given an active sharded connection
+    And a holder on another connection answering `echo` under the `a` key
+    When one of the brokers has crashed
+    And the consumer calls `echo` under the `a` key 20 times
+    Then every call is answered

package/features/rpc.call.shards.feature

@@ -0,0 +1,24 @@
+Feature: Calls under a key over a sharded connection
+
+  A call is published to one shard, and a shard where nobody holds the key returns it: it is
+  published on the next one, and refused once every shard has returned it.
+
+  Background:
+    Given an active sharded connection
+
+  Scenario: Answered by a holder on one broker only
+    Given a holder connected to broker 0 answering `echo` under the `a` key
+    When the consumer calls `echo` under the `a` key 20 times
+    Then every call is answered
+
+  Scenario: Refused when nobody holds the key on any broker
+    Given a holder connected to broker 0 answering `echo` under the `a` key
+    When the consumer calls `echo` under the `b` key
+    Then the call is refused as unroutable
+
+  Scenario: A key taken on one broker is held on the other, and on both once it is let go
+    Given a holder connected to broker 0 answering `whom` under the `a` key as `first`
+    When a second holder on another connection holds `whom` under the `a` key as `second`
+    Then the second holder finds the key taken
+    When the first holder disconnects
+    Then every call to `whom` under the `a` key is answered by `second` within 30 seconds

package/features/rpc.parked.feature

@@ -0,0 +1,20 @@
+Feature: Parked requests
+
+  Background:
+    Given an active connection to the broker
+
+  Scenario: A parked request can still be answered
+
+    A caller that sets no timeout is certain of an eventual answer for as long as it is
+    alive, and parking is what keeps that true when no consumer can produce one: the
+    request is kept, along with who was asking. Somebody else can answer it, and the
+    caller — still waiting, still holding an exclusive reply queue — receives it.
+
+    A caller that sets a timeout trades that for an end to the wait: once it stops
+    waiting, a parked request is addressed to a caller that has gone.
+
+    Given a producer failing every request to the `unanswerable` queue
+    When a request is sent to the `unanswerable` queue
+    Then the request is parked
+    When the parked request from the `unanswerable` queue is answered by hand
+    Then the caller receives the reply

package/features/rpc.timeout.feature

@@ -0,0 +1,19 @@
+Feature: Request timeout
+
+  A caller may say how long it waits for a reply. A request nobody took by then is dropped by the
+  broker; one already taken is still processed, and its reply is discarded.
+
+  Background:
+    Given an active connection to the broker
+
+  Scenario: A caller stops waiting at its timeout
+    Given a producer replying `slow` queue in 1000ms
+    When the consumer sends a request to the `slow` queue with a 200ms timeout
+    Then the consumer stops waiting
+
+  Scenario: A request nobody took in time is never processed
+    When the consumer sends a request to the `later` queue with a 200ms timeout
+    Then the consumer stops waiting
+    When after 300ms
+    And a producer counting requests to the `later` queue
+    Then the producer receives nothing

package/features/silence.feature

@@ -57,6 +57,7 @@ Feature: Silent Connection Tolerance
   Scenario: Reporting that there is nowhere left to publish
     Given an active sharded connection
     And a producer replying `echo` queue
-    When the network goes silent
+    When the network goes silent and refuses new connections
     Then publishing is paused within 10 seconds
-    And publishing is resumed within 10 seconds
+    When the network admits new connections
+    Then publishing is resumed within 10 seconds

package/features/steps/call.js

@@ -0,0 +1,462 @@
+'use strict'
+
+const assert = require('node:assert')
+const amqplib = require('amqplib')
+const { Given, When, Then, After } = require('@cucumber/cucumber')
+
+const { connect, Unroutable } = require('../../')
+const { parse } = require('./yaml')
+const { getAddress, USER, PASSWORD } = require('./brokers')
+const { timeout, until } = require('../../test/helpers')
+
+Given('a holder answering {token} under the {token} key:',
+  /**
+   * @param {string} exchange
+   * @param {string} key
+   * @param {string} javascript
+   * @this {comq.features.Context}
+   */
+  async function (exchange, key, javascript) {
+    // eslint-disable-next-line no-new-func
+    const producer = new Function('return ' + javascript)()
+
+    this.io.diagnose('taken', () => (this.taken = true))
+
+    await this.io.back(exchange, key, producer)
+  })
+
+Given('a holder on another connection answering {token} under the {token} key',
+  /**
+   * @param {string} exchange
+   * @param {string} key
+   * @this {comq.features.Context}
+   */
+  async function (exchange, key) {
+    this.holder = await another(this)
+
+    await this.holder.back(exchange, key, echo)
+  })
+
+Given('a holder on another connection answering {token} under the {token} key in {number}ms',
+  /**
+   * @param {string} exchange
+   * @param {string} key
+   * @param {number} delay
+   * @this {comq.features.Context}
+   */
+  async function (exchange, key, delay) {
+    this.holder = await another(this)
+
+    await this.holder.back(exchange, key, async (payload) => {
+      await timeout(delay)
+
+      return echo(payload)
+    })
+  })
+
+Given('a holder connected to broker {number} answering {token} under the {token} key',
+  /**
+   * @param {number} broker
+   * @param {string} exchange
+   * @param {string} key
+   * @this {comq.features.Context}
+   */
+  async function (broker, exchange, key) {
+    this.holder = await another(this, broker)
+
+    await this.holder.back(exchange, key, echo)
+  })
+
+Given('a holder never answering {token} under the {token} key',
+  /**
+   * Holds the key as comq does, and takes a Request without ever answering it, so that losing
+   * its connection loses the Request with it.
+   *
+   * @param {string} exchange
+   * @param {string} key
+   * @this {comq.features.Context}
+   */
+  async function (exchange, key) {
+    const connection = await amqplib.connect(url(0))
+    const channel = await connection.createChannel()
+    const queue = exchange + '.' + key
+
+    await channel.assertExchange(exchange, 'direct', { durable: true })
+    await channel.assertQueue(queue, { exclusive: true })
+    await channel.bindQueue(queue, exchange, key)
+    await channel.consume(queue, () => undefined)
+
+    this.silent = connection
+  })
+
+When('the silent holder crashes',
+  /**
+   * Its socket is destroyed without a word, as a process that dies leaves it.
+   *
+   * @this {comq.features.Context}
+   */
+  function () {
+    const connection = this.silent
+
+    this.silent = undefined
+
+    connection.on('error', () => undefined)
+    connection.connection.stream.destroy()
+  })
+
+When('the consumer calls {token} under the {token} key',
+  /**
+   * @param {string} exchange
+   * @param {string} key
+   * @this {comq.features.Context}
+   */
+  async function (exchange, key) {
+    call(this, this.io.call(exchange, key, { key }))
+  })
+
+When('the consumer calls {token} under the {token} key with:',
+  /**
+   * @param {string} exchange
+   * @param {string} key
+   * @param {string} yaml
+   * @this {comq.features.Context}
+   */
+  async function (exchange, key, yaml) {
+    call(this, this.io.call(exchange, key, parse(yaml)))
+  })
+
+When('the consumer calls {token} under the {token} key with a {number}ms timeout',
+  /**
+   * @param {string} exchange
+   * @param {string} key
+   * @param {number} ms
+   * @this {comq.features.Context}
+   */
+  async function (exchange, key, ms) {
+    call(this, this.io.call(exchange, key, { key }, { timeout: ms }))
+  })
+
+When('the consumer calls {token} under the {token} key {number} times',
+  /**
+   * @param {string} exchange
+   * @param {string} key
+   * @param {number} times
+   * @this {comq.features.Context}
+   */
+  async function (exchange, key, times) {
+    const calls = Array.from({ length: times }, (_, n) => this.io.call(exchange, key, n))
+
+    this.replies = await Promise.all(calls)
+  })
+
+When('the consumer sends a request to the {token} queue with a {number}ms timeout',
+  /**
+   * @param {string} queue
+   * @param {number} ms
+   * @this {comq.features.Context}
+   */
+  async function (queue, ms) {
+    call(this, this.io.request(queue, { queue }, { timeout: ms }))
+  })
+
+When('the holder is sealed',
+  /**
+   * @this {comq.features.Context}
+   */
+  async function () {
+    await this.holder.seal()
+  })
+
+When('a second holder on another connection holds {token} under the {token} key',
+  /**
+   * @param {string} exchange
+   * @param {string} key
+   * @this {comq.features.Context}
+   */
+  async function (exchange, key) {
+    const second = await another(this)
+
+    second.diagnose('taken', () => (this.taken = true))
+
+    this.holding = second.back(exchange, key, echo)
+
+    // asserted on by a later step, and an unhandled rejection would end the run before it
+    this.holding.catch(() => undefined)
+  })
+
+When('the first holder disconnects',
+  /**
+   * @this {comq.features.Context}
+   */
+  async function () {
+    await this.holder.close()
+  })
+
+When('a producer counting requests to the {token} queue',
+  /**
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  async function (queue) {
+    this.produced = 0
+
+    await this.io.reply(queue, () => {
+      this.produced++
+
+      return null
+    })
+  })
+
+Then('the call is refused as unroutable',
+  /**
+   * @this {comq.features.Context}
+   */
+  async function () {
+    await assert.rejects(this.reply, (error) => error instanceof Unroutable)
+  })
+
+Then('the consumer stops waiting',
+  /**
+   * @this {comq.features.Context}
+   */
+  async function () {
+    await assert.rejects(this.reply, (error) => error.name === 'TimeoutError')
+  })
+
+Given('a holder connected to broker {number} answering {token} under the {token} key as {token}',
+  /**
+   * @param {number} broker
+   * @param {string} exchange
+   * @param {string} key
+   * @param {string} name
+   * @this {comq.features.Context}
+   */
+  async function (broker, exchange, key, name) {
+    this.holder = await another(this, broker)
+
+    await this.holder.back(exchange, key, () => name)
+  })
+
+When('a second holder on another connection holds {token} under the {token} key as {token}',
+  /**
+   * @param {string} exchange
+   * @param {string} key
+   * @param {string} name
+   * @this {comq.features.Context}
+   */
+  async function (exchange, key, name) {
+    const second = await another(this)
+
+    second.diagnose('taken', () => (this.taken = true))
+
+    await second.back(exchange, key, () => name)
+  })
+
+Then('the second holder finds the key taken',
+  /**
+   * @this {comq.features.Context}
+   */
+  async function () {
+    assert.equal(await until(() => this.taken === true), true, 'The key was not found taken')
+  })
+
+Then('every call to {token} under the {token} key is answered by {token} within {number} seconds', { timeout: 120_000 },
+  /**
+   * @param {string} exchange
+   * @param {string} key
+   * @param {string} name
+   * @param {number} seconds
+   * @this {comq.features.Context}
+   */
+  async function (exchange, key, name, seconds) {
+    const answered = await eventually(async () => {
+      const calls = Array.from({ length: 20 }, () => this.io.call(exchange, key, null, { timeout: 1000 }))
+      const replies = await Promise.all(calls)
+
+      if (!replies.every((reply) => reply === name)) throw new Error('Answered by another')
+    }, seconds)
+
+    assert.equal(answered, true, `Calls were not all answered by ${name}`)
+  })
+
+Then('the second holder holds the key',
+  /**
+   * @this {comq.features.Context}
+   */
+  async function () {
+    await this.holding
+  })
+
+Then('the holder finds its key taken while the broker holds the silent connection', { timeout: 60_000 },
+  /**
+   * @this {comq.features.Context}
+   */
+  async function () {
+    assert.equal(await until(() => this.taken === true, 50_000), true, 'The key was not found taken')
+  })
+
+Then('a call to {token} under the {token} key is answered within {number} seconds', { timeout: 120_000 },
+  /**
+   * @param {string} exchange
+   * @param {string} key
+   * @param {number} seconds
+   * @this {comq.features.Context}
+   */
+  async function (exchange, key, seconds) {
+    const answered = await eventually(() => this.io.call(exchange, key, { key }, { timeout: 1000 }), seconds)
+
+    assert.equal(answered, true, 'The call was not answered')
+  })
+
+Then('a call to {token} under the {token} key from another connection is answered within {number} seconds', { timeout: 120_000 },
+  /**
+   * @param {string} exchange
+   * @param {string} key
+   * @param {number} seconds
+   * @this {comq.features.Context}
+   */
+  async function (exchange, key, seconds) {
+    const caller = await another(this)
+    const answered = await eventually(() => caller.call(exchange, key, { key }, { timeout: 1000 }), seconds)
+
+    assert.equal(answered, true, 'The call was not answered')
+  })
+
+Then('a call to {token} under the {token} key is refused within {number} seconds', { timeout: 60_000 },
+  /**
+   * @param {string} exchange
+   * @param {string} key
+   * @param {number} seconds
+   * @this {comq.features.Context}
+   */
+  async function (exchange, key, seconds) {
+    const refused = await eventually(async () => {
+      try {
+        await this.io.call(exchange, key, { key }, { timeout: 1000 })
+      } catch (error) {
+        if (error instanceof Unroutable) return
+
+        throw error
+      }
+
+      throw new Error('Answered')
+    }, seconds)
+
+    assert.equal(refused, true, 'The call was not refused')
+  })
+
+Then('the call ends', { timeout: 90_000 },
+  /**
+   * @this {comq.features.Context}
+   */
+  async function () {
+    await this.reply.catch(() => undefined)
+  })
+
+Then('the consumer stops waiting within {number} second(s)', { timeout: 90_000 },
+  /**
+   * @param {number} seconds
+   * @this {comq.features.Context}
+   */
+  async function (seconds) {
+    const stopped = this.reply.then(() => 'answered', (error) => error.name === 'TimeoutError' ? 'stopped' : error.message)
+    const late = timeout(seconds * 1000).then(() => 'still waiting')
+
+    assert.equal(await Promise.race([stopped, late]), 'stopped')
+  })
+
+Then('every call is answered',
+  /**
+   * @this {comq.features.Context}
+   */
+  async function () {
+    assert.deepEqual(this.replies, this.replies.map((_, n) => n))
+  })
+
+Then('the producer receives nothing',
+  /**
+   * @this {comq.features.Context}
+   */
+  async function () {
+    await timeout(500)
+
+    assert.equal(this.produced, 0, 'An expired request was processed')
+  })
+
+After(
+  /**
+   * @this {comq.features.Context}
+   */
+  async function () {
+    await Promise.all((this.holders ?? []).map((io) => io.close()))
+    await this.silent?.close().catch(() => undefined)
+
+    this.holders = []
+    this.silent = undefined
+  })
+
+/**
+ * Another connection, to the brokers the consumer is connected to, or to one of them.
+ *
+ * @param {comq.features.Context} context
+ * @param {number} [broker]
+ * @returns {Promise<comq.IO>}
+ */
+async function another (context, broker) {
+  const urls = broker !== undefined
+    ? [url(broker)]
+    : context.sharded ? [url(0), url(1)] : [url(0)]
+
+  const io = await connect(...urls)
+
+  context.holders ??= []
+  context.holders.push(io)
+
+  return io
+}
+
+/**
+ * @param {comq.features.Context} context
+ * @param {Promise<any>} reply
+ */
+function call (context, reply) {
+  // asserted on by a later step, and an unhandled rejection would end the run before it
+  reply.catch(() => undefined)
+
+  context.reply = reply
+}
+
+/**
+ * @param {number} broker
+ * @returns {string}
+ */
+function url (broker) {
+  return `amqp://${USER}:${PASSWORD}@${getAddress(broker)}`
+}
+
+/**
+ * Makes an attempt until it succeeds, or until the time is up.
+ *
+ * @param {() => Promise<unknown>} attempt
+ * @param {number} seconds
+ * @returns {Promise<boolean>} whether it succeeded
+ */
+async function eventually (attempt, seconds) {
+  const deadline = Date.now() + seconds * 1000
+
+  while (Date.now() < deadline) {
+    try {
+      await attempt()
+
+      return true
+    } catch {
+      await timeout(500)
+    }
+  }
+
+  return false
+}
+
+function echo (payload) {
+  return payload
+}

package/features/steps/context.js

@@ -21,6 +21,8 @@ class Context extends World {
   attempts = []
   counts = {}
   parked = {}
+  awaited
+  answer
   processed
   enqueued
   tasksProcessedCount = 0
@@ -69,6 +71,8 @@ class Context extends World {
     this.attempts = []
     this.counts = {}
     this.parked = {}
+    this.awaited = undefined
+    this.answer = undefined
   }
 
   /**