@yarkivaev/source-to-sink 1.2.3 → 1.2.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/source-to-sink",
-  "version": "1.2.3",
+  "version": "1.2.4",
   "description": "Generic library for building data streaming pipelines",
   "repository": {
     "type": "git",

package/src/batch.js

@@ -31,12 +31,8 @@ export default function batch(sink, size, circuit) {
   }
   let records = [];
   function perform() {
-    if (records.length === 0) {
-      return;
-    }
-    if (!circuit.allowing()) {
-      return;
-    }
+    if (records.length === 0) { return; }
+    if (!circuit.allowing()) { return; }
     const pending = records;
     records = [];
     try {

package/src/mqttSource.js

@@ -83,6 +83,9 @@ export default function mqttSource(url, topics, collector, options = {}) {
           sessionExpiryInterval: options.sessionExpiryInterval || 3600
         } : undefined
       });
+      // @todo Implement manualAcknowledge to defer ack until batch writes to sink
+      // Currently MQTT.js auto-acks after this callback returns, before the batch
+      // flushes to the database. Up to 5 seconds of data may be lost on crash
       function handler(tp, message) {
         collector.accept({ topic: tp, payload: message.toString() });
       }

package/src/stompSource.js

@@ -101,15 +101,13 @@ export default function stompSource(url, destination, collector, options = {}) {
       const channel = new stompit.Channel(failover);
       const subscribe = { destination, ack: 'client-individual' };
       channel.subscribe(subscribe, (err, message) => {
-        if (err) {
-          return;
-        }
-        message.readString('utf-8', (readErr, body) => {
-          if (readErr) {
-            return;
-          }
-          collector.accept({ destination, payload: body.toString() });
-          channel.ack(message);
+        if (err) { return; }
+        message.readString('utf-8', async (readErr, body) => {
+          if (readErr) { return; }
+          try {
+            await collector.accept({ destination, payload: body.toString() });
+            channel.ack(message);
+          } catch { channel.nack(message); }
         });
       });
       state = subscribed(channel);