@yarkivaev/source-to-sink 1.2.3 → 1.2.7

package/.github/workflows/ci.yml

@@ -8,5 +8,5 @@ jobs:
   ci:
     uses: yarkivaev/npm-workflows/.github/workflows/ci.yml@main
     with:
-      node-version: '22'
+      node-version: '22.22.1'
       lint: true

package/.github/workflows/release.yml

@@ -11,5 +11,5 @@ jobs:
       contents: write
       id-token: write
     with:
-      node-version: '22'
+      node-version: '22.22.1'
       lint: false

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/source-to-sink",
-  "version": "1.2.3",
+  "version": "1.2.7",
   "description": "Generic library for building data streaming pipelines",
   "repository": {
     "type": "git",

package/src/batch.js

@@ -31,12 +31,8 @@ export default function batch(sink, size, circuit) {
   }
   let records = [];
   function perform() {
-    if (records.length === 0) {
-      return;
-    }
-    if (!circuit.allowing()) {
-      return;
-    }
+    if (records.length === 0) { return; }
+    if (!circuit.allowing()) { return; }
     const pending = records;
     records = [];
     try {

package/src/mqttSource.js

@@ -83,6 +83,9 @@ export default function mqttSource(url, topics, collector, options = {}) {
           sessionExpiryInterval: options.sessionExpiryInterval || 3600
         } : undefined
       });
+      // @todo Implement manualAcknowledge to defer ack until batch writes to sink
+      // Currently MQTT.js auto-acks after this callback returns, before the batch
+      // flushes to the database. Up to 5 seconds of data may be lost on crash
       function handler(tp, message) {
         collector.accept({ topic: tp, payload: message.toString() });
       }

package/src/stompSource.js

@@ -99,17 +99,18 @@ export default function stompSource(url, destination, collector, options = {}) {
         maxReconnects: -1
       });
       const channel = new stompit.Channel(failover);
+      let pending = Promise.resolve();
       const subscribe = { destination, ack: 'client-individual' };
       channel.subscribe(subscribe, (err, message) => {
-        if (err) {
-          return;
-        }
+        if (err) { return; }
         message.readString('utf-8', (readErr, body) => {
-          if (readErr) {
-            return;
-          }
-          collector.accept({ destination, payload: body.toString() });
-          channel.ack(message);
+          if (readErr) { return; }
+          pending = pending.then(async () => {
+            try {
+              await collector.accept({ destination, payload: body.toString() });
+              channel.ack(message);
+            } catch { channel.nack(message); }
+          });
         });
       });
       state = subscribed(channel);