comq 0.19.0 → 0.20.1

package/features/steps/network.js

@@ -26,6 +26,32 @@ When('the network goes silent',
     silence.call(this)
   })
 
+When('the network goes silent and refuses new connections',
+  /**
+   * @this {comq.features.Context}
+   */
+  function () {
+    for (const network of this.networks) network.refuse()
+
+    silence.call(this)
+  })
+
+When('the network admits new connections',
+  /**
+   * @this {comq.features.Context}
+   */
+  function () {
+    for (const network of this.networks) network.admit()
+  })
+
+When('the silent connection is let go',
+  /**
+   * @this {comq.features.Context}
+   */
+  function () {
+    for (const network of this.networks) network.release()
+  })
+
 // a request that is answered over a connection that is already silent proves
 // nothing, hence the window between the two is left as short as a step boundary
 When('the consumer sends a request to the {token} queue as the network goes silent',

package/features/steps/networks.js

@@ -23,6 +23,8 @@ class Network {
   /** @type {number} */
   #port
 
+  #refusing = false
+
   /**
    * @param {number} [n] broker index
    */
@@ -51,6 +53,26 @@ class Network {
     for (const tunnel of this.#tunnels) tunnel.silent = true
   }
 
+  /**
+   * A connection established from now on is closed as soon as it is accepted, so
+   * a connection that is lost stays lost until the network admits it.
+   */
+  refuse () {
+    this.#refusing = true
+  }
+
+  admit () {
+    this.#refusing = false
+  }
+
+  /**
+   * Closes the tunnels that went silent, at both ends, so the broker learns that the
+   * connection it was still holding is gone — the moment its heartbeat would otherwise tell it.
+   */
+  release () {
+    for (const tunnel of this.#tunnels) if (tunnel.silent) this.#collapse(tunnel)
+  }
+
   async close () {
     for (const tunnel of this.#tunnels) this.#collapse(tunnel)
 
@@ -61,6 +83,8 @@ class Network {
    * @param {net.Socket} client
    */
   #tunnel = (client) => {
+    if (this.#refusing) return client.destroy()
+
     const upstream = net.connect(this.#port, this.#host)
 
     /** @type {comq.features.Tunnel} */
@@ -71,10 +95,22 @@ class Network {
     client.on('data', (chunk) => { if (!tunnel.silent) upstream.write(chunk) })
     upstream.on('data', (chunk) => { if (!tunnel.silent) client.write(chunk) })
 
-    for (const socket of [client, upstream]) {
-      socket.on('error', () => this.#collapse(tunnel))
-      socket.on('close', () => this.#collapse(tunnel))
-    }
+    // a silent network tells neither end that the other has gone: the client giving up on a
+    // silent connection leaves the broker holding its end, until the network is let go
+    client.on('error', () => this.#hangUp(tunnel))
+    client.on('close', () => this.#hangUp(tunnel))
+
+    upstream.on('error', () => this.#collapse(tunnel))
+    upstream.on('close', () => this.#collapse(tunnel))
+  }
+
+  /**
+   * @param {comq.features.Tunnel} tunnel
+   */
+  #hangUp (tunnel) {
+    if (!tunnel.silent) return this.#collapse(tunnel)
+
+    tunnel.client.destroy()
   }
 
   /**

package/features/steps/parked.js

@@ -0,0 +1,102 @@
+'use strict'
+
+const assert = require('node:assert')
+const amqplib = require('amqplib')
+const { Given, When, Then } = require('@cucumber/cucumber')
+
+const { getAddress, USER, PASSWORD } = require('./brokers')
+const { until, timeout } = require('../../test/helpers')
+
+Given('a producer failing every request to the {token} queue',
+  /**
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  async function (queue) {
+    await this.io.reply(queue, () => {
+      throw new Error('Expected failure')
+    })
+  })
+
+When('a request is sent to the {token} queue',
+  /**
+   * The caller is left waiting on purpose: the promise is what the last step asserts on.
+   *
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  async function (queue) {
+    this.awaited = this.io.request(queue, { question: 'the answer?' }, 'application/json')
+
+    // nothing settles it unless the parked request is answered, and an unhandled
+    // rejection would take the run down before the assertion is reached
+    this.awaited.catch(() => undefined)
+  })
+
+Then('the request is parked',
+  /**
+   * @this {comq.features.Context}
+   */
+  async function () {
+    await until(() => this.events.discard === true)
+
+    assert.equal(this.events.discard, true, 'The request was not parked')
+  })
+
+When('the parked request from the {token} queue is answered by hand',
+  /**
+   * What an operator draining the parking queue would do: take the message, read who
+   * was asking, and publish the reply they are still waiting for.
+   *
+   * @param {string} queue
+   * @this {comq.features.Context}
+   */
+  async function (queue) {
+    const connection = await amqplib.connect(`amqp://${USER}:${PASSWORD}@${getAddress(0)}`)
+    const channel = await connection.createChannel()
+
+    try {
+      let parked
+
+      await channel.consume('comq.parked.' + queue, (message) => {
+        parked = message
+        channel.ack(message)
+      })
+
+      await until(() => parked !== undefined)
+
+      assert.notEqual(parked, undefined, `Nothing was parked from ${queue}`)
+
+      const { replyTo, correlationId, contentType } = parked.properties
+
+      assert.notEqual(replyTo, undefined, 'The parked request lost its replyTo')
+      assert.notEqual(correlationId, undefined, 'The parked request lost its correlationId')
+
+      this.answer = { answered: true, was: JSON.parse(parked.content.toString()) }
+
+      channel.sendToQueue(replyTo, Buffer.from(JSON.stringify(this.answer)),
+        { correlationId, contentType })
+
+      // let the frame reach the broker before the connection goes
+      await timeout(100)
+    } finally {
+      await channel.close()
+      await connection.close()
+    }
+  })
+
+Then('the caller receives the reply',
+  /**
+   * @this {comq.features.Context}
+   */
+  async function () {
+    const reply = await Promise.race([
+      this.awaited,
+      timeout(5000).then(() => TIMED_OUT)
+    ])
+
+    assert.notEqual(reply, TIMED_OUT, 'The caller is still waiting')
+    assert.deepEqual(reply, this.answer, 'The caller received something else')
+  })
+
+const TIMED_OUT = Symbol('the caller was still waiting')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "comq",
-  "version": "0.19.0",
+  "version": "0.20.1",
   "description": "ComQ",
   "author": {
     "name": "Artem Gurtovoi",

package/readme.md

@@ -6,7 +6,8 @@ for distributed, eventually consistent systems running on Node.js.
 ## Features
 
 - [Dynamic topology](#topology)
-- [Request](#request)-[reply](#reply) (RPC)
+- [Request](#request)-[reply](#reply) (RPC), with a [timeout](#timeout)
+- [Addressed requests](#addressed-requests) to the one connection holding a Key
 - Events ([pub](#emission)/[sub](#consumption)), fanned out or [routed](#routing)
 - [Tasks](#tasks)
 - [Pipelines](#pipelines)
@@ -106,19 +107,50 @@ await io.reply('add_numbers', ({ a, b }) => (a + b))
 
 ## Request
 
-`async IO.request(queue: string, payload: any, encoding?: string): any`
+`async IO.request(queue: string, payload: any, options?: string | RequestOptions): any`
 
 Send encoded Request message with `replyTo` and `correlationId` properties set and
-return decoded Reply content. The promise stays pending until the Reply arrives.
+return decoded Reply content. The promise stays pending until the Reply arrives, or until the
+[timeout](#timeout) passes.
 
 On the initial call, queues for Requests and Replies are asserted.
 
+`options` is the encoding, or an object:
+
+| Option     | Type          | Default            |
+|------------|---------------|--------------------|
+| `encoding` | `string`      | `application/json` |
+| `timeout`  | `number`, ms  | none               |
+| `signal`   | `AbortSignal` | none               |
+
 ### Example
 
 ```javascript
 const sum = await io.request('add_numbers', { a: 1, b: 2 })
 ```
 
+### Timeout
+
+A Request with a `timeout` rejects once it passes, with the `TimeoutError` of
+[`AbortSignal.timeout`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal/timeout_static).
+It is published with that much
+[expiration](https://www.rabbitmq.com/docs/ttl#per-message-ttl-in-publishers), so a Request no
+Producer has taken by then is dropped by the broker and never processed. A Request a Producer has
+already taken is processed to the end, and its Reply is discarded. A Request re-sent after a lost
+connection carries the time it has left.
+
+A `signal` rejects the Request with its reason once aborted, within the `timeout` where both are
+given. It ends the wait and leaves the Request where it is: in its queue until the `timeout`
+passes, or, without a `timeout`, until a Producer takes it. A Request abandoned by its `signal` may
+therefore still be processed.
+
+A Request that failed and waits for its [next attempt](#retries) loses its expiration on the way
+back to its queue, so it may be processed after its caller has stopped waiting.
+
+```javascript
+const sum = await io.request('add_numbers', { a: 1, b: 2 }, { timeout: 5000 })
+```
+
 ## Consumption
 
 `async IO.consume(exchange: string, group?: string, consumer): void`
@@ -194,6 +226,65 @@ await io.route('records', 'store.orders', { id: 1, status: 'paid' })
 await io.route('records', 'store.customers', { id: 2 }) // not delivered to the above
 ```
 
+## Addressed Requests
+
+`async IO.back(exchange: string, key: string, producer): void`
+
+`async IO.call(exchange: string, key: string, payload: any, options?: string | RequestOptions): any`
+
+A Request to the one connection holding a Key, where a [Request](#request) goes to whichever
+Producer takes it first.
+
+`back` asserts a [direct exchange](#routing) and a queue named `<exchange>.<key>`, *exclusive* to
+the connection, binds it under the `key` and starts consuming Requests, as [`reply`](#reply) does.
+**On each broker, one connection holds a Key at a time.** While another connection holds it,
+`back` claims it again and again, with the backoff a lost connection is
+[restored](#connection-tolerance) with, and emits [`taken`](#diagnostics) on every refusal. The Key
+is let go when its connection closes, however it closes — once the broker has noticed, for a
+connection that went without a word. `back` returns once a broker holds the Key. The rest of the
+connection works throughout, and a connection that is restored claims its Keys again the same way.
+
+`call` publishes the encoded Request to the exchange under the `key` and returns the decoded Reply,
+taking the same options as [`request`](#request).
+
+A call ends in one of three ways:
+
+- **A Reply.**
+- **Refused at once**, rejecting with `Unroutable`, when no connection holds the Key: it never did,
+  its holder has closed, or its holder is [sealed](#sealing). A refused call reached no one.
+- **At its [timeout](#timeout)**, when its holder has gone without being sealed — a crashed process,
+  a lost connection — or takes longer. A call nobody has taken by then is dropped.
+
+A call waits for as long as it takes unless it has a timeout, and a holder that is gone never
+answers it. **Give every call a timeout.**
+
+Sealing withdraws every Key before it stops consuming: a call published from then on is refused,
+and one published before it is delivered and answered. Calls queued beyond the
+[prefetch](#channels) at that moment go with the connection and end at their timeout.
+
+A Key is as alive as its connection: while the holder reconnects, calls to it are refused, and
+calls queued for it are lost.
+
+Over a [sharded connection](#sharded-connection), `back` claims the Key on every shard and returns
+once one of them holds it; a shard where it is taken goes on claiming it. Two connections given the
+same Key can therefore hold it on different shards and both answer calls, and `taken` is what says
+so. A call returned by one shard is published on the next, and refused once every shard has
+returned it.
+
+### Example
+
+```javascript
+const { Unroutable } = require('comq')
+
+await io.back('sessions', 'a1', (message) => sessions.get(message.id))
+
+try {
+  const session = await io.call('sessions', 'a1', { id: 7 }, { timeout: 5000 })
+} catch (error) {
+  if (error instanceof Unroutable) console.log('Nobody holds', error.key)
+}
+```
+
 ## Tasks
 
 `async IO.enqueue(queue: string, payload: any, encoding?: string): void`
@@ -425,24 +516,6 @@ dynamic, such as those that depend on runtime data like incoming messages, makin
 impossible or hard to maintain. The tradeoff of potentially encountering runtime topology
 declaration exceptions, which are more likely to happen during development, is deemed acceptable.
 
-### Settings
-
-Each channel type has a preset, and the trailing argument of `connect` overrides it:
-
-```javascript
-const io = await comq.connect(url, {
-  event: { delay: [5000, 60000] },  // two retries
-  request: { delay: 1000 }          // one
-})
-```
-
-`delay` governs [retries](#retries); the rest of [the settings](./types/topology.d.ts) are not
-meant to be changed.
-
-> Changing `delay` declares new retry queues rather than redeclaring the existing ones, so a
-> rolling deploy that changes it has no window in which either version fails. The queues left
-> behind are empty and can be removed once nothing is publishing to them.
-
 ### Channels
 
 `IO` lazy creates individual channels for Requests, Replies, and Events.
@@ -467,6 +540,8 @@ requests and are expecting replies.
   and [Consumption](#consumption), and as _direct_ for [Routing](#routing). One name is one or
   the other: asserting it as both is what the broker refuses.
 - Queues for Replies are _exclusive_ and _auto deleted_.
+- A queue [`back`](#addressed-requests) holds is _exclusive_, bound under its Key to a _direct_
+  exchange.
 
 comq declares two kinds of queue of its own, for [failed messages](#retries):
 
@@ -508,8 +583,8 @@ other consumers nor that consumer's next message; only the message that failed i
 | Event | 1s, 10s, 30s, 90s | 5 | 131s |
 | Request | 1s, 3s, 5s, 10s | 5 | 19s |
 
-Requests are shorter because a caller is blocked on one with no timeout, and a Reply arriving
-long after it gave up has nowhere useful to land. Nobody waits on an Event.
+Requests are shorter because a caller is blocked on one, and a Reply arriving long after it
+stopped waiting has nowhere useful to land. Nobody waits on an Event.
 
 One retry queue and one exchange are declared per distinct wait and shared by every queue that
 uses them, so their number grows with the length of the ladder rather than with the number of
@@ -566,11 +641,12 @@ Parked queues grow until somebody drains them, which is deliberate — the alter
 evidence. Alert on [`discard`](#diagnostics), and do not delete `comq.retry.*` or `comq.parked.*`
 queues on a running system.
 
-> **A parked Request is never answered.** A Consumer awaiting its Reply waits indefinitely, and
-> with a limited prefetch that can deadlock it. Parking keeps the Request rather than deleting
+> **A parked Request is never answered.** A Consumer awaiting its Reply waits until its
+> [timeout](#timeout), or indefinitely without one, and with a limited prefetch that can deadlock
+> it. Parking keeps the Request rather than deleting
 > it — and it keeps `replyTo` and `correlationId`, so a Reply can still be produced from it by
 > hand while the caller is alive — but comq itself sends no Reply and reports no error to the
-> caller. Give a Request a timeout on the calling side if you cannot tolerate that.
+> caller.
 
 #### What is guaranteed
 
@@ -585,11 +661,21 @@ A retried message re-enters its queue behind the messages published while it wai
 Retries and parked messages are published *persistent* whatever the channel is, so they survive a
 restart of the broker even on the Request channel. Ordinary publishing is untouched: Requests and
 Replies stay [delivery mode 1](#messages), and only a message that has already failed is written
-to disk. Two weaker points remain on that channel: it
-does not use publisher confirms, so comq has no positive acknowledgement that the broker took the
-copy; and the return hop of a retry is performed by the broker's dead-lettering, which on classic
-queues is at-most-once and can lose the message if the source queue is unavailable when the delay
-expires.
+to disk.
+
+On the Request channel the copy is published without [publisher confirms](#channels). Those are
+an Events property here, because a confirm is a round trip and Requests are where that is felt —
+the same reason they are not persistent. Confirm mode belongs to the channel rather than to a
+publish, so the failure path cannot ask for it on its own.
+
+What that costs is narrow. Commands on a channel are handled in order, so the broker takes the
+copy before it releases the original, and `mandatory` brings back a copy it could not route. What
+is left uncovered is a broker that accepted the frame and then failed to keep it.
+
+The return hop of a retry — the broker moving a message out of the retry queue when its wait
+expires — is dead-lettering, and on classic queues that is at-most-once: a retry can be lost if
+its source queue is unavailable at the moment the delay expires. This applies to every channel,
+not only Requests.
 
 See:
 
@@ -605,6 +691,25 @@ See:
 | Reply   | unlimited | no       | exclusive | automatic      | no         | —                |
 | Event   | limited   | yes      | durable   | manual         | yes        | 1s, 10s, 30s, 90s |
 
+### Settings
+
+Each channel type has a [preset](./source/topology), and the trailing argument of `connect`
+overrides any of its fields:
+
+```javascript
+const io = await comq.connect(url, {
+  event: { delay: [5000, 60000] },  // two retries
+  request: { delay: 1000 }          // one
+})
+```
+
+`delay` is the one meant to be set. Changing the rest will change what a Request, a Reply and an
+Event *are*.
+
+> Changing `delay` declares new retry queues rather than redeclaring the existing ones, so a
+> rolling deploy that changes it has no window in which either version fails. The queues left
+> behind are empty and can be removed once nothing is publishing to them.
+
 ## Graceful shutdown
 
 ### Sealing
@@ -615,6 +720,9 @@ See:
 Requests.
 Sending Requests, receiving Replies, and emitting Events will still be available.
 
+Keys held by [`back`](#addressed-requests) are withdrawn first, so a call published from then on is
+refused.
+
 ### Disconnection
 
 `async IO.close(): void`
@@ -685,6 +793,8 @@ Subscribe to one of the diagnostic events:
   [amqp message object](https://amqp-node.github.io/amqplib/channel_api.html#channel_publish) are
   passed as arguments. In the case of a [sharded connection](#sharded-connection), the message is
   reported only once every shard has rejected it.
+- `taken`: a Key [`back`](#addressed-requests) claims is held by another connection on this broker, and
+  is claimed again. Channel type and the queue name are passed.
 - `pause`: channel is paused. Channel type is passed.
   In the case of a [sharded connection](#sharded-connection), it means that there is no shard left
   to publish to, be it because every one of them has rejected a publish or lost its connection.

package/source/attributes/recall.js

@@ -15,7 +15,8 @@ const recorder = (context, method) => async function (...args) {
 
   const result = await method.apply(context, args)
 
-  method[CALLS].push(args)
+  // forgotten while it ran, by a seal: a sealed context replays nothing
+  method[CALLS]?.push(args)
 
   return result
 }

package/source/channel.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const { Promex } = require('promex')
+const { retry } = require('reretry')
 const { failsafe, lazy, recall } = require('./attributes')
 const { verdictOf, PARK } = require('./verdicts')
 const emitter = require('./emitter')
@@ -34,6 +35,13 @@ class Channel {
    */
   #queues = new Map()
 
+  /**
+   * The exchange and key each held queue is bound under, so that sealing withdraws them.
+   *
+   * @type {Map<string, [string, string]>}
+   */
+  #held = new Map()
+
   /** @type {Promex | null} */
   #paused = null
 
@@ -80,8 +88,9 @@ class Channel {
     // the consumers of the previous channel went down with it, their tags mean nothing here
     this.#tags = []
 
-    // a fresh channel has declared nothing
+    // a fresh channel has declared nothing, and bound nothing
     this.#queues.clear()
+    this.#held.clear()
 
     await this.#channel.prefetch(this.#topology.prefetch)
 
@@ -131,6 +140,33 @@ class Channel {
           if (!this.#sealed) await this.#consume(queue, callback)
         })))
 
+  /**
+   * Consumes a queue exclusive to this connection, bound to a direct exchange under a key:
+   * whatever is published under that key reaches this connection and no other. While another
+   * connection holds the queue, it is claimed again and again until it is let go, and every
+   * refusal is reported as `taken`.
+   */
+  held = recall(this,
+    failsafe(this, this.#recover,
+      lazy(this, this.#assertRouted,
+        /**
+         * @param {string} exchange
+         * @param {string} queue
+         * @param {string} key
+         * @param {comq.channels.Consumer} callback
+         * @returns {Promise<void>}
+         */
+        async (exchange, queue, key, callback) => {
+          if (this.#sealed) return
+
+          const holding = this.#hold(exchange, queue, key, callback)
+
+          // a recovery restores the rest of the connection while a key another connection holds
+          // is claimed, and a claim the connection is lost under is replayed by the next recovery
+          if (this.#recovering) holding.catch(noop)
+          else await holding
+        })))
+
   send = failsafe(this, this.#recover,
     lazy(this, this.#assertQueue,
       /**
@@ -201,6 +237,15 @@ class Channel {
   async seal () {
     this.#sealed = true
 
+    // a key is withdrawn before its consumer goes: a Request published from here on is
+    // returned to its caller, and one published before it is delivered and answered
+    const unbindings = Array.from(this.#held,
+      ([queue, [exchange, key]]) => this.#channel.unbindQueue(queue, exchange, key))
+
+    await Promise.allSettled(unbindings)
+
+    this.#held.clear()
+
     const cancellations = this.#tags.map((tag) => this.#channel.cancel(tag))
 
     await Promise.all(cancellations).catch(noop) // won't recover anyway
@@ -249,6 +294,84 @@ class Channel {
     this.#diagnostics.emit('recover')
   }
 
+  /**
+   * @param {string} exchange
+   * @param {string} queue
+   * @param {string} key
+   * @param {comq.channels.Consumer} callback
+   * @returns {Promise<void>}
+   */
+  async #hold (exchange, queue, key, callback) {
+    const claimed = await this.#claim(queue)
+
+    if (!claimed) return
+
+    await this.#assertQueue(queue, EXCLUSIVE)
+    await this.#channel.bindQueue(queue, exchange, key)
+
+    this.#held.set(queue, [exchange, key])
+
+    if (!this.#sealed) await this.#consume(queue, callback)
+  }
+
+  /**
+   * Claims a queue exclusive to this connection, again and again while another connection
+   * holds it. The broker announces nothing when a queue is let go, so it is asked again, with
+   * the backoff a lost connection is restored with.
+   *
+   * @param {string} queue
+   * @returns {Promise<boolean>} whether the queue is claimed, as opposed to sealed while claiming
+   */
+  async #claim (queue) {
+    const connection = this.#connection
+
+    return await retry(async (again) => {
+      if (this.#sealed) return false
+
+      // a connection that is gone is recovered from, like any other interruption
+      if (connection !== this.#connection) throw INTERRUPTION
+
+      if (await this.#declare(connection, queue)) return true
+
+      this.#diagnostics.emit('taken', queue)
+
+      return again
+    }, global.COMQ_TESTING_CLAIM_BACKOFF)
+  }
+
+  /**
+   * Declares a queue exclusive to this connection, and says whether another connection holds it.
+   *
+   * The broker refuses such a queue by closing the channel that asked for it, and a channel error
+   * nobody listens for closes the whole connection. So the declaration is made on a channel of its
+   * own, which is all that closes. The queue belongs to the connection, and outlives that channel.
+   *
+   * @param {comq.amqp.Connection} connection
+   * @param {string} queue
+   * @returns {Promise<boolean>}
+   */
+  async #declare (connection, queue) {
+    const probe = await connection.createChannel().catch(() => { throw INTERRUPTION })
+
+    /** @type {Error & { code?: number } | undefined} */
+    let refusal
+
+    probe.on('error', (error) => { refusal = error })
+
+    try {
+      await probe.assertQueue(queue, EXCLUSIVE)
+    } catch (exception) {
+      if (refusal?.code === RESOURCE_LOCKED) return false
+
+      // what the broker refused with says more than the channel it closed
+      throw refusal ?? exception
+    }
+
+    await probe.close().catch(noop)
+
+    return true
+  }
+
   // region initializers
 
   /**
@@ -648,6 +771,9 @@ const EXCLUSIVE = { exclusive: true }
 
 const INTERRUPTION = /** @type {Error} */ Symbol('internal interruption')
 
+// the AMQP reply code a broker closes a channel with when another connection holds the queue
+const RESOURCE_LOCKED = 405
+
 const RETRY_PREFIX = 'comq.retry.'
 const PARKED_PREFIX = 'comq.parked.'
 
@@ -672,3 +798,4 @@ const PARKED_AT_HEADER = 'x-comq-at'
 function noop () {}
 
 exports.create = create
+exports.RETRY_PREFIX = RETRY_PREFIX

package/source/connection.js

@@ -371,6 +371,8 @@ const SHUTDOWN_MS = 5_000
 
 const SOCKET_OPTIONS = {
   timeout: CONNECT_MS,
+  // a message is written once and is small, which is what Nagle's algorithm holds back
+  noDelay: true,
   // a peer that went away without a word is noticed by the kernel as well
   keepAlive: true,
   keepAliveDelay: KEEPALIVE_MS

package/source/events.js

@@ -4,4 +4,4 @@
 exports.connection = ['open', 'close', 'error', 'reconnect', 'exhausted']
 
 /** @type {comq.diagnostics.Event[]} */
-exports.channel = ['flow', 'drain', 'recover', 'discard', 'retry', 'pause', 'resume', 'return', 'lost', 'remove']
+exports.channel = ['flow', 'drain', 'recover', 'discard', 'retry', 'pause', 'resume', 'return', 'lost', 'remove', 'taken']