@yarkivaev/source-to-sink 1.3.7 → 1.3.9

package/README.md

@@ -38,3 +38,12 @@ source.start();
 ## License
 
 MIT
+
+### STOMP delivery window
+
+`stompSource(url, destination, collector, { prefetch: 32 })` bounds broker deliveries
+that are waiting for acknowledgement. The default is 32; use a positive safe
+integer to tune throughput. A zero (unlimited) window is rejected because a
+backlog can otherwise exhaust the consumer's heap. With `manualAck: true`, the
+collector remains responsible for settling frames. If it flushes only by count,
+its batch must not exceed the prefetch window; a timed flush also avoids stalls.

package/eslint.config.js

@@ -150,6 +150,7 @@ export default [
         files: ['src/modbusSource.js'],
         rules: {
             'max-params': ['error', 8],
+            'max-lines': ['error', { 'max': 250, 'skipBlankLines': true, 'skipComments': true }],
             'max-lines-per-function': ['error', { 'max': 60, 'skipBlankLines': true, 'skipComments': true }],
             'max-statements': ['error', 20]
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/source-to-sink",
-  "version": "1.3.7",
+  "version": "1.3.9",
   "description": "Generic library for building data streaming pipelines",
   "repository": {
     "type": "git",

package/src/modbusSource.js

@@ -124,6 +124,10 @@ function attempt(bag) {
     }
     bag.delay = bag.interval;
     bag.state = connected(bag.client);
+    bag.client.setTimeout(bag.interval * 1000);
+    bag.client.once('close', () => {
+      bag.lost();
+    });
     bag.source.start();
   }).catch((err) => {
     if (token !== bag.epoch) {
@@ -139,6 +143,28 @@ function attempt(bag) {
   });
 }
 
+/**
+ * Resets a live session and retries through attempt() backoff.
+ *
+ * @param {object} bag - Mutable lifecycle bag
+ */
+function resume(bag) {
+  if (!bag.state.connected()) {
+    return;
+  }
+  bag.source.stop();
+  const { client } = bag;
+  bag.client = undefined;
+  bag.state = idle();
+  drop(client);
+  bag.log.error(`Connection to ${bag.target} lost, retrying in ${bag.delay}s`);
+  const handle = setTimeout(() => {
+    attempt(bag);
+  }, bag.delay * 1000);
+  bag.state = retrying(handle);
+  bag.delay = Math.min(bag.delay * 2, bag.maxDelay);
+}
+
 /**
  * Shared Modbus polling lifecycle for TCP and RTU transports.
  *
@@ -155,9 +181,17 @@ function modbusPollingSource(options) {
     maxDelay: 300,
     delay: interval
   };
+  bag.lost = function lost() {
+    resume(bag);
+  };
   async function fetch() {
-    const result = await bag.client.readHoldingRegisters(address, count);
-    return [result.data];
+    try {
+      const result = await bag.client.readHoldingRegisters(address, count);
+      return [result.data];
+    } catch (err) {
+      bag.lost();
+      throw err;
+    }
   }
   bag.source = pollingSource(fetch, interval, collector, clk);
   return {
@@ -179,9 +213,10 @@ function modbusPollingSource(options) {
       if (bag.state.retrying()) {
         bag.state.cancel();
       }
-      drop(bag.client);
+      const { client } = bag;
       bag.client = undefined;
       bag.state = idle();
+      drop(client);
     }
   };
 }

package/src/stompSource.js

@@ -115,13 +115,13 @@ function read(message) {
  * @param {object} channel - STOMP channel
  * @param {string} destination - queue destination
  * @param {object} collector - message collector
- * @param {boolean} serial - wait for persist before the next arrival
- * @param {boolean} manualAck - collector settles frames
+ * @param {object} delivery - serial, manualAck and prefetch settings
  * @returns {undefined}
  */
-function bind(channel, destination, collector, serial, manualAck) {
+function bind(channel, destination, collector, delivery) {
+  const { serial, manualAck, prefetch } = delivery;
   let pending = Promise.resolve();
-  channel.subscribe({ destination, ack: 'client-individual' }, (err, message) => {
+  channel.subscribe({ destination, ack: 'client-individual', 'prefetch-count': String(prefetch) }, (err, message) => {
     if (err) {
       return;
     }
@@ -140,11 +140,13 @@ function bind(channel, destination, collector, serial, manualAck) {
  * Messages are `{destination, payload, settle}`. Default ack-after-accept;
  * with `manualAck` the collector owns settle. Arrivals enqueue in
  * subscribe order. With `serial: false` persist may overlap after enqueue.
+ * A finite prefetch (default 32) bounds unacknowledged broker deliveries.
  *
  * @param {string} url - STOMP broker URL
  * @param {string} destination - STOMP destination
  * @param {object} collector - Collector with accept()
  * @param {object} [options] - Connection and delivery options
+ * @param {number} [options.prefetch=32] - Positive integer delivery window
  * @returns {object} Source with start() and stop() methods
  */
 export default function stompSource(url, destination, collector, options = {}) {
@@ -157,6 +159,10 @@ export default function stompSource(url, destination, collector, options = {}) {
   if (!collector || typeof collector.accept !== 'function') {
     throw new Error('Collector must have an accept() method');
   }
+  const prefetch = options.prefetch ?? 32;
+  if (!Number.isSafeInteger(prefetch) || prefetch < 1) {
+    throw new Error('prefetch must be a positive safe integer');
+  }
   const parsed = new URL(url);
   const serial = options.serial !== false;
   const manualAck = options.manualAck === true;
@@ -172,7 +178,7 @@ export default function stompSource(url, destination, collector, options = {}) {
         maxReconnects: -1
       });
       const channel = new stompit.Channel(failover);
-      bind(channel, destination, collector, serial, manualAck);
+      bind(channel, destination, collector, { serial, manualAck, prefetch });
       state = subscribed(channel);
     },
     stop() {

package/test/test_modbusSource.js

@@ -105,6 +105,114 @@ function until(probe, limit) {
     return new Promise(tick);
 }
 
+/**
+ * TCP listener that counts accepted sockets and can drop them.
+ *
+ * @returns {Promise<object>} Gate with port, accepted, drop, close
+ */
+function sessionListen() {
+    let accepted = 0;
+    const sockets = new Set();
+    const server = net.createServer((sock) => {
+        accepted += 1;
+        sockets.add(sock);
+        sock.on('close', () => {
+            sockets.delete(sock);
+        });
+    });
+    return new Promise((resolve) => {
+        server.listen(0, '127.0.0.1', () => {
+            resolve({
+                port: server.address().port,
+                accepted() {
+                    return accepted;
+                },
+                drop() {
+                    for (const sock of sockets) {
+                        sock.destroy();
+                    }
+                },
+                close() {
+                    for (const sock of sockets) {
+                        sock.destroy();
+                    }
+                    return new Promise((done) => {
+                        server.close(done);
+                    });
+                }
+            });
+        });
+    });
+}
+
+/**
+ * Builds a Modbus TCP Read Holding Registers response.
+ *
+ * @param {Buffer} req - Request frame
+ * @param {number[]} values - Register values
+ * @returns {Buffer} Response frame
+ */
+function holding(req, values) {
+    const tid = req.readUInt16BE(0);
+    const uid = req.readUInt8(6);
+    const bytes = values.length * 2;
+    const out = Buffer.alloc(9 + bytes);
+    out.writeUInt16BE(tid, 0);
+    out.writeUInt16BE(0, 2);
+    out.writeUInt16BE(3 + bytes, 4);
+    out.writeUInt8(uid, 6);
+    out.writeUInt8(3, 7);
+    out.writeUInt8(bytes, 8);
+    for (let i = 0; i < values.length; i += 1) {
+        out.writeUInt16BE(values[i], 9 + i * 2);
+    }
+    return out;
+}
+
+/**
+ * TCP listener that stays silent on the first socket then answers later ones.
+ *
+ * @param {number[]} values - Holding register values
+ * @returns {Promise<object>} Gate with port, accepted, close
+ */
+function recoverListen(values) {
+    let accepted = 0;
+    const sockets = new Set();
+    const server = net.createServer((sock) => {
+        accepted += 1;
+        sockets.add(sock);
+        sock.on('close', () => {
+            sockets.delete(sock);
+        });
+        if (accepted === 1) {
+            return;
+        }
+        sock.on('data', (req) => {
+            if (req.length >= 12 && req.readUInt8(7) === 3) {
+                sock.write(holding(req, values));
+            }
+        });
+    });
+    return new Promise((resolve) => {
+        server.listen(0, '127.0.0.1', () => {
+            resolve({
+                port: server.address().port,
+                accepted() {
+                    return accepted;
+                },
+                close() {
+                    for (const sock of sockets) {
+                        sock.destroy();
+                    }
+                    return new Promise((done) => {
+                        server.close(done);
+                    });
+                }
+            });
+        });
+    });
+}
+
 describe('modbusSource', () => {
   it('throws on empty host', () => {
     const collector = { accept: () => {} };
@@ -311,4 +419,64 @@ describe('modbusSource', () => {
     await gate.close();
     assert.strictEqual(live, 0, 'stop during connect left a Modbus TCP socket open');
   });
+
+  it('opens a new TCP socket after the device stays silent or drops the session', async function() {
+    this.timeout(6000);
+    const gate = await sessionListen();
+    const source = modbusSource(
+      '127.0.0.1',
+      gate.port,
+      4000,
+      3 + Math.floor(Math.random() * 8),
+      0.05,
+      { accept() {} },
+      fakeClock(11 + Math.floor(Math.random() * 100)),
+      { error() {} }
+    );
+    source.start();
+    await until(() => {
+      return gate.accepted() >= 1;
+    }, 2000);
+    await settle(80);
+    gate.drop();
+    await until(() => {
+      return gate.accepted() >= 2;
+    }, 3000);
+    const accepted = gate.accepted();
+    source.stop();
+    await gate.close();
+    assert.ok(accepted >= 2, 'silent or dropped session did not open a new Modbus TCP socket');
+  });
+
+  it('forwards a new sample after the device answers again', async function() {
+    this.timeout(6000);
+    const count = 3 + Math.floor(Math.random() * 6);
+    const values = Array.from({ length: count }, () => {
+      return 1000 + Math.floor(Math.random() * 500);
+    });
+    const samples = [];
+    const gate = await recoverListen(values);
+    const source = modbusSource(
+      '127.0.0.1',
+      gate.port,
+      4000,
+      count,
+      0.05,
+      {
+        accept(entry) {
+          samples.push(entry);
+        }
+      },
+      fakeClock(13 + Math.floor(Math.random() * 100)),
+      { error() {} }
+    );
+    source.start();
+    await until(() => {
+      return samples.length >= 1;
+    }, 4000);
+    const sample = samples[0];
+    source.stop();
+    await gate.close();
+    assert.deepStrictEqual(sample, values, 'restored device did not deliver a new sample');
+  });
 });

package/test/test_stompSource.js

@@ -1,6 +1,7 @@
 import assert from 'node:assert';
 import { describe, it } from 'mocha';
 import stompSource from '../src/stompSource.js';
+import stompit from 'stompit';
 
 describe('stompSource', () => {
   it('throws on missing url', () => {
@@ -61,3 +62,55 @@ describe('stompSource', () => {
     assert.strictEqual(typeof source.start, 'function', 'manualAck source missing start');
   });
 });
+
+/**
+ * Captures the protocol headers using an inert channel.
+ *
+ * @param {object} options - Source options
+ * @returns {object} SUBSCRIBE headers
+ */
+function subscription(options) {
+  const original = stompit.Channel;
+  let headers;
+  stompit.Channel = function Channel() {
+    return {
+      subscribe(value) { headers = value; },
+      close() { return undefined; }
+    };
+  };
+  try {
+    const source = stompSource('stomp://localhost:61613', '/queue/backlog', {
+      accept() { return undefined; }
+    }, options);
+    source.start();
+    source.stop();
+    return headers;
+  } finally {
+    stompit.Channel = original;
+  }
+}
+
+describe('STOMP delivery window', function() {
+  it('bounds the default backlog without changing acknowledgements', function() {
+    assert.deepStrictEqual(subscription({ manualAck: true, serial: false }), {
+      destination: '/queue/backlog',
+      ack: 'client-individual',
+      'prefetch-count': '32'
+    });
+  });
+
+  it('sends a configured finite delivery window', function() {
+    const prefetch = Math.floor(Math.random() * 100) + 1;
+    assert.strictEqual(subscription({ prefetch })['prefetch-count'], String(prefetch));
+  });
+
+  for (const prefetch of [0, -1, 1.5, Infinity, NaN, '32']) {
+    it(`rejects invalid prefetch ${prefetch}`, function() {
+      assert.throws(function() {
+        stompSource('stomp://localhost:61613', '/queue/backlog', {
+          accept() { return undefined; }
+        }, { prefetch });
+      }, /prefetch must be a positive safe integer/u);
+    });
+  }
+});