@yarkivaev/source-to-sink 1.3.8 → 1.3.10

package/README.md

@@ -38,3 +38,14 @@ source.start();
 ## License
 
 MIT
+
+### STOMP delivery window
+
+`stompSource(url, destination, collector, { prefetch: 32 })` bounds broker deliveries
+that are waiting for acknowledgement. The default is 32; use a positive safe
+integer to tune throughput. A zero (unlimited) window is rejected because a
+backlog can otherwise exhaust the consumer's heap. With `manualAck: true`, the
+collector remains responsible for settling frames. If it flushes only by count,
+its batch must not exceed the prefetch window; a timed flush also avoids stalls.
+
+Manual collectors may call `settle.nack(false)` to dead-letter invalid messages. Configure a durable dead-letter queue and broker policy before enabling this; without a route RabbitMQ discards rejected messages. `settle.nack()` retains retry behaviour.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/source-to-sink",
-  "version": "1.3.8",
+  "version": "1.3.10",
   "description": "Generic library for building data streaming pipelines",
   "repository": {
     "type": "git",

package/src/stompSource.js

@@ -62,8 +62,8 @@ function settleOf(channel, message) {
     ack() {
       channel.ack(message);
     },
-    nack() {
-      channel.nack(message);
+    nack(requeue = true) {
+      channel.nack(message, { requeue: String(requeue) });
     }
   };
 }
@@ -115,13 +115,13 @@ function read(message) {
  * @param {object} channel - STOMP channel
  * @param {string} destination - queue destination
  * @param {object} collector - message collector
- * @param {boolean} serial - wait for persist before the next arrival
- * @param {boolean} manualAck - collector settles frames
+ * @param {object} delivery - serial, manualAck and prefetch settings
  * @returns {undefined}
  */
-function bind(channel, destination, collector, serial, manualAck) {
+function bind(channel, destination, collector, delivery) {
+  const { serial, manualAck, prefetch } = delivery;
   let pending = Promise.resolve();
-  channel.subscribe({ destination, ack: 'client-individual' }, (err, message) => {
+  channel.subscribe({ destination, ack: 'client-individual', 'prefetch-count': String(prefetch) }, (err, message) => {
     if (err) {
       return;
     }
@@ -140,11 +140,13 @@ function bind(channel, destination, collector, serial, manualAck) {
  * Messages are `{destination, payload, settle}`. Default ack-after-accept;
  * with `manualAck` the collector owns settle. Arrivals enqueue in
  * subscribe order. With `serial: false` persist may overlap after enqueue.
+ * A finite prefetch (default 32) bounds unacknowledged broker deliveries.
  *
  * @param {string} url - STOMP broker URL
  * @param {string} destination - STOMP destination
  * @param {object} collector - Collector with accept()
  * @param {object} [options] - Connection and delivery options
+ * @param {number} [options.prefetch=32] - Positive integer delivery window
  * @returns {object} Source with start() and stop() methods
  */
 export default function stompSource(url, destination, collector, options = {}) {
@@ -157,6 +159,10 @@ export default function stompSource(url, destination, collector, options = {}) {
   if (!collector || typeof collector.accept !== 'function') {
     throw new Error('Collector must have an accept() method');
   }
+  const prefetch = options.prefetch ?? 32;
+  if (!Number.isSafeInteger(prefetch) || prefetch < 1) {
+    throw new Error('prefetch must be a positive safe integer');
+  }
   const parsed = new URL(url);
   const serial = options.serial !== false;
   const manualAck = options.manualAck === true;
@@ -172,7 +178,7 @@ export default function stompSource(url, destination, collector, options = {}) {
         maxReconnects: -1
       });
       const channel = new stompit.Channel(failover);
-      bind(channel, destination, collector, serial, manualAck);
+      bind(channel, destination, collector, { serial, manualAck, prefetch });
       state = subscribed(channel);
     },
     stop() {

package/test/test_stompSource.js

@@ -1,6 +1,7 @@
 import assert from 'node:assert';
 import { describe, it } from 'mocha';
 import stompSource from '../src/stompSource.js';
+import stompit from 'stompit';
 
 describe('stompSource', () => {
   it('throws on missing url', () => {
@@ -61,3 +62,85 @@ describe('stompSource', () => {
     assert.strictEqual(typeof source.start, 'function', 'manualAck source missing start');
   });
 });
+
+/**
+ * Captures the protocol headers using an inert channel.
+ *
+ * @param {object} options - Source options
+ * @returns {object} SUBSCRIBE headers
+ */
+function subscription(options) {
+  const original = stompit.Channel;
+  let headers;
+  stompit.Channel = function Channel() {
+    return {
+      subscribe(value) { headers = value; },
+      close() { return undefined; }
+    };
+  };
+  try {
+    const source = stompSource('stomp://localhost:61613', '/queue/backlog', {
+      accept() { return undefined; }
+    }, options);
+    source.start();
+    source.stop();
+    return headers;
+  } finally {
+    stompit.Channel = original;
+  }
+}
+
+describe('STOMP delivery window', function() {
+  it('bounds the default backlog without changing acknowledgements', function() {
+    assert.deepStrictEqual(subscription({ manualAck: true, serial: false }), {
+      destination: '/queue/backlog',
+      ack: 'client-individual',
+      'prefetch-count': '32'
+    });
+  });
+
+  it('sends a configured finite delivery window', function() {
+    const prefetch = Math.floor(Math.random() * 100) + 1;
+    assert.strictEqual(subscription({ prefetch })['prefetch-count'], String(prefetch));
+  });
+
+  for (const prefetch of [0, -1, 1.5, Infinity, NaN, '32']) {
+    it(`rejects invalid prefetch ${prefetch}`, function() {
+      assert.throws(function() {
+        stompSource('stomp://localhost:61613', '/queue/backlog', {
+          accept() { return undefined; }
+        }, { prefetch });
+      }, /prefetch must be a positive safe integer/u);
+    });
+  }
+});
+
+
+describe('STOMP dead lettering', function() {
+  it('sends requeue=false for a rejected poison message', async function() {
+    const original = stompit.Channel;
+    let receive;
+    let headers;
+    stompit.Channel = function Channel() {
+      return {
+        subscribe(value, callback) { receive = callback; },
+        nack(message, value) { headers = value; },
+        close() {}
+      };
+    };
+    try {
+      const source = stompSource('stomp://localhost:61613', '/queue/test', {
+        accept({ settle }) { settle.nack(false); }
+      }, { manualAck: true });
+      source.start();
+      receive(null, { readString(encoding, callback) { callback(null, '{}'); } });
+      await new Promise((resolve) => { setImmediate(resolve); });
+      source.stop();
+      assert.deepStrictEqual(headers, { requeue: 'false' });
+    } finally {
+      // Test restores its synchronous module stub after delivery completes.
+      // eslint-disable-next-line require-atomic-updates
+      stompit.Channel = original;
+    }
+  });
+});