@yarkivaev/source-to-sink 1.3.5 → 1.3.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/source-to-sink",
-  "version": "1.3.5",
+  "version": "1.3.7",
   "description": "Generic library for building data streaming pipelines",
   "repository": {
     "type": "git",

package/src/follow.js

@@ -0,0 +1,41 @@
+/**
+ * Continues a delivery chain after a rejected prior step.
+ *
+ * @param {Promise} pending - prior chain
+ * @returns {Promise} settled chain
+ */
+function resume(pending) {
+    return pending.then((value) => {
+        return value;
+    }, (error) => {
+        return error;
+    });
+}
+
+/**
+ * Chains one arrival: read the body, then start deliver.
+ * Serial waits for deliver. Otherwise persist may overlap after enqueue.
+ *
+ * @param {Promise} pending - prior arrival
+ * @param {Function} read - returns a promise for the body
+ * @param {Function} deliver - starts persist for one body
+ * @param {boolean} serial - wait for persist before the next arrival
+ * @returns {Promise} chain
+ *
+ * @example
+ *   let pending = Promise.resolve();
+ *   pending = follow(pending, () => Promise.resolve('a'), write, false);
+ */
+export default function follow(pending, read, deliver, serial) {
+    return resume(pending).then(async () => {
+        const body = await read();
+        const run = deliver(body);
+        if (serial) {
+            return run;
+        }
+        void run.catch((error) => {
+            return error;
+        });
+        return undefined;
+    });
+}

package/src/modbusSource.js

@@ -85,6 +85,28 @@ function connected(client) {
   };
 }
 
+/**
+ * Destroys a Modbus client without throwing if already closed.
+ *
+ * @param {object} client - ModbusRTU client
+ */
+function drop(client) {
+  if (client && typeof client.destroy === 'function') {
+    client.destroy(() => {
+      return undefined;
+    });
+  }
+}
+
+/**
+ * Builds a ModbusRTU client for one connect attempt.
+ *
+ * @returns {object} Fresh ModbusRTU client
+ */
+function create() {
+  return new ModbusRTU();
+}
+
 /**
  * Starts one connect attempt and ignores results after stop().
  *
@@ -93,9 +115,11 @@ function connected(client) {
 function attempt(bag) {
   const token = bag.epoch;
   bag.state = opening();
+  drop(bag.client);
+  bag.client = bag.create();
   bag.connect(bag.client).then(() => {
     if (token !== bag.epoch) {
-      bag.client.close();
+      drop(bag.client);
       return;
     }
     bag.delay = bag.interval;
@@ -105,6 +129,7 @@ function attempt(bag) {
     if (token !== bag.epoch) {
       return;
     }
+    drop(bag.client);
     bag.log.error(`Connection to ${bag.target} failed, retrying in ${bag.delay}s: ${err.message}`);
     const handle = setTimeout(() => {
       attempt(bag);
@@ -121,19 +146,20 @@ function attempt(bag) {
  * @returns {object} Source with start() and stop() methods
  */
 function modbusPollingSource(options) {
-  const { address, count, interval, collector, clk, client } = options;
-  async function fetch() {
-    const result = await client.readHoldingRegisters(address, count);
-    return [result.data];
-  }
+  const { address, count, interval, collector, clk } = options;
   const bag = {
     ...options,
+    client: undefined,
     state: idle(),
     epoch: 0,
     maxDelay: 300,
-    source: pollingSource(fetch, interval, collector, clk),
     delay: interval
   };
+  async function fetch() {
+    const result = await bag.client.readHoldingRegisters(address, count);
+    return [result.data];
+  }
+  bag.source = pollingSource(fetch, interval, collector, clk);
   return {
     /**
      * Connects to the Modbus device and starts polling.
@@ -153,9 +179,8 @@ function modbusPollingSource(options) {
       if (bag.state.retrying()) {
         bag.state.cancel();
       }
-      if (bag.state.connected() || bag.state.opening()) {
-        bag.client.close();
-      }
+      drop(bag.client);
+      bag.client = undefined;
       bag.state = idle();
     }
   };
@@ -195,11 +220,11 @@ export default function modbusSource(host, port, address, count, interval, colle
   if (typeof count !== 'number' || count <= 0) {
     throw new Error(`Count must be a positive number, got: ${count}`);
   }
-  const client = new ModbusRTU();
   function connect(modbusClient) {
     return modbusClient.connectTCP(host, { port });
   }
   return modbusPollingSource({
+    create,
     connect,
     target: `${host}:${port}`,
     address,
@@ -207,7 +232,6 @@ export default function modbusSource(host, port, address, count, interval, colle
     interval,
     collector,
     clk,
-    client,
     log
   });
 }
@@ -248,7 +272,6 @@ export function modbusRtuSource(path, serial, address, count, interval, collecto
   if (typeof count !== 'number' || count <= 0) {
     throw new Error(`Count must be a positive number, got: ${count}`);
   }
-  const client = new ModbusRTU();
   const slaveId = serial.slaveId || 1;
   const dataBits = serial.dataBits || 8;
   const stopBits = serial.stopBits || 1;
@@ -264,6 +287,7 @@ export function modbusRtuSource(path, serial, address, count, interval, collecto
     });
   }
   return modbusPollingSource({
+    create,
     connect,
     target: path,
     address,
@@ -271,7 +295,6 @@ export function modbusRtuSource(path, serial, address, count, interval, collecto
     interval,
     collector,
     clk,
-    client,
     log
   });
 }

package/src/pollingSource.js

@@ -64,14 +64,17 @@ export default function pollingSource(fetch, interval, collector, clk) {
   }
   let state = idle();
   let since = clk.millis();
-  async function poll() {
+  function poll() {
     const from = since;
     const until = clk.millis();
     since = until;
-    const result = await fetch(from, until);
-    for (const entry of result) {
-      collector.accept(entry);
-    }
+    Promise.resolve(fetch(from, until)).then((result) => {
+      for (const entry of result) {
+        collector.accept(entry);
+      }
+    }).catch(() => {
+      return undefined;
+    });
   }
   return {
     /**

package/src/postgresSink.js

@@ -32,7 +32,7 @@ export function deduplicate(records, keys) {
 /**
  * Builds the ON CONFLICT SQL suffix from options.
  *
- * @param {object} options - Sink options with conflict and update arrays
+ * @param {object} options - Sink options with conflict, update, and optional updateWhere
  * @returns {string} SQL suffix or empty string
  */
 function buildSuffix(options) {
@@ -40,7 +40,10 @@ function buildSuffix(options) {
   const cols = options.conflict.join(', ');
   if (Array.isArray(options.update) && options.update.length > 0) {
     const sets = options.update.map((col) => {return `${col} = EXCLUDED.${col}`}).join(', ');
-    return ` ON CONFLICT (${cols}) DO UPDATE SET ${sets}`;
+    const guard = typeof options.updateWhere === 'string' && options.updateWhere.length > 0
+      ? ` WHERE ${options.updateWhere}`
+      : '';
+    return ` ON CONFLICT (${cols}) DO UPDATE SET ${sets}${guard}`;
   }
   return ` ON CONFLICT (${cols}) DO NOTHING`;
 }
@@ -67,6 +70,7 @@ function buildSuffix(options) {
  * @param {object} [options] - Optional configuration
  * @param {Array<string>} [options.conflict] - Columns for ON CONFLICT clause
  * @param {Array<string>} [options.update] - Columns for DO UPDATE SET clause
+ * @param {string} [options.updateWhere] - Optional WHERE guard on DO UPDATE
  * @param {object} [options.pool] - Existing pg Pool to reuse
  * @returns {object} Sink with write(records, client?) method
  */

package/src/stompSource.js

@@ -1,4 +1,5 @@
 import stompit from 'stompit';
+import follow from './follow.js';
 
 /**
  * Idle state for STOMP source.
@@ -90,41 +91,46 @@ async function deliver(ctx, body) {
 }
 
 /**
- * Ignores a rejected delivery promise when running non-serial.
+ * Reads one STOMP frame body as text.
  *
- * @param {Error} error - rejected reason
- * @returns {undefined}
+ * @param {object} message - STOMP frame
+ * @returns {Promise<string>} UTF-8 body
  */
-function ignoreDelivery(error) {
-  return error;
+function read(message) {
+  return new Promise((resolve, reject) => {
+    message.readString('utf-8', (error, body) => {
+      if (error) {
+        reject(error);
+        return;
+      }
+      resolve(body.toString());
+    });
+  });
 }
 
 /**
  * Subscribes a channel and routes frames to the collector.
+ * Arrivals stay in subscribe order through follow, even when persist overlaps.
  *
  * @param {object} channel - STOMP channel
  * @param {string} destination - queue destination
  * @param {object} collector - message collector
- * @param {boolean} serial - global serial delivery
+ * @param {boolean} serial - wait for persist before the next arrival
  * @param {boolean} manualAck - collector settles frames
  * @returns {undefined}
  */
 function bind(channel, destination, collector, serial, manualAck) {
   let pending = Promise.resolve();
   channel.subscribe({ destination, ack: 'client-individual' }, (err, message) => {
-    if (err) { return; }
-    message.readString('utf-8', (readErr, body) => {
-      if (readErr) { return; }
-      const ctx = { channel, message, destination, collector, manualAck };
-      function work() {
-        return deliver(ctx, body.toString());
-      }
-      if (serial) {
-        pending = pending.then(work, work);
-      } else {
-        void work().catch(ignoreDelivery);
-      }
-    });
+    if (err) {
+      return;
+    }
+    const ctx = { channel, message, destination, collector, manualAck };
+    pending = follow(pending, () => {
+      return read(message);
+    }, (body) => {
+      return deliver(ctx, body);
+    }, serial);
   });
 }
 
@@ -132,8 +138,8 @@ function bind(channel, destination, collector, serial, manualAck) {
  * STOMP subscription source for streaming messages to a collector.
  *
  * Messages are `{destination, payload, settle}`. Default ack-after-accept;
- * with `manualAck` the collector owns settle. With `serial: false`
- * deliveries are not globally sequenced.
+ * with `manualAck` the collector owns settle. Arrivals enqueue in
+ * subscribe order. With `serial: false` persist may overlap after enqueue.
  *
  * @param {string} url - STOMP broker URL
  * @param {string} destination - STOMP destination

package/test/test_follow.js

@@ -0,0 +1,35 @@
+import assert from 'node:assert';
+import { describe, it } from 'mocha';
+import follow from '../src/follow.js';
+
+describe('follow', () => {
+    it('accepts in subscribe order when the first body is slower', async () => {
+        const order = [];
+        const first = `ä-${Math.random().toString(16).slice(2)}`;
+        const second = `ö-${Math.random().toString(16).slice(2)}`;
+        const pause = (ms) => {
+            return new Promise((resolve) => {
+                setTimeout(resolve, ms);
+            });
+        };
+        const deliver = (body) => {
+            order.push(body);
+            return Promise.resolve();
+        };
+        let pending = Promise.resolve();
+        pending = follow(pending, () => {
+            return pause(40).then(() => {
+                return first;
+            });
+        }, deliver, false);
+        pending = follow(pending, () => {
+            return Promise.resolve(second);
+        }, deliver, false);
+        await pending;
+        assert.deepStrictEqual(
+            order,
+            [first, second],
+            'a slower first body reordered accept ahead of subscribe'
+        );
+    });
+});

package/test/test_modbusSource.js

@@ -59,6 +59,27 @@ function settle(ms) {
     });
 }
 
+/**
+ * Collects unhandledRejection events until stop().
+ *
+ * @returns {object} Collector with seen() and stop()
+ */
+function watchRejects() {
+    const seen = [];
+    function onReject(err) {
+        seen.push(err);
+    }
+    process.on('unhandledRejection', onReject);
+    return {
+        seen() {
+            return seen.length;
+        },
+        stop() {
+            process.off('unhandledRejection', onReject);
+        }
+    };
+}
+
 /**
  * Waits until probe is true or the timeout elapses.
  *
@@ -200,6 +221,28 @@ describe('modbusSource', () => {
     assert.strictEqual(true, true, 'Should not crash on unreachable host');
   });
 
+  it('does not emit unhandled rejection after connection retry', async function() {
+    this.timeout(4000);
+    const watch = watchRejects();
+    const log = { error() {} };
+    const port = 65000 + Math.floor(Math.random() * 500);
+    const source = modbusSource(
+      '127.0.0.1',
+      port,
+      4000,
+      14,
+      0.05,
+      { accept() {} },
+      fakeClock(1 + Math.floor(Math.random() * 100)),
+      log
+    );
+    source.start();
+    await settle(400);
+    source.stop();
+    watch.stop();
+    assert.strictEqual(watch.seen(), 0, 'retry leaked an unhandled rejection');
+  });
+
   it('does not open a second TCP socket when start is called twice', async function() {
     const gate = await countingListen();
     const source = modbusSource(

package/test/test_pollingSource.js

@@ -91,6 +91,26 @@ describe('pollingSource', () => {
     assert.strictEqual(count, before, 'Should stop polling after stop');
   });
 
+  it('does not emit unhandled rejection when fetch rejects', async () => {
+    const seen = [];
+    function onReject(err) {
+      seen.push(err);
+    }
+    process.on('unhandledRejection', onReject);
+    const label = `\u00e9${Math.random()}`;
+    const fetch = () => {
+      return Promise.reject(new Error(label));
+    };
+    const source = pollingSource(fetch, 0.03, { accept() {} }, fakeClock(1 + Math.floor(Math.random() * 100)));
+    source.start();
+    await new Promise((resolve) => {
+      setTimeout(resolve, 80);
+    });
+    source.stop();
+    process.off('unhandledRejection', onReject);
+    assert.strictEqual(seen.length, 0, 'poll rejection leaked as unhandled');
+  });
+
   it('throws on missing fetch', () => {
     const collector = { accept: () => {} };
     const clk = fakeClock(0);

package/test/test_postgresSink.js

@@ -122,6 +122,28 @@ describe('postgresSink', () => {
     assert.strictEqual(queries.length, 1, 'injected pool was not used for write');
   });
 
+  it('appends updateWhere to the conflict update', async () => {
+    const queries = [];
+    const pool = {
+      query(sql) {
+        queries.push(sql);
+        return Promise.resolve({ rowCount: 1 });
+      }
+    };
+    const sink = postgresSink('postgresql://localhost:5432/db', 'segments',
+      ['machine', 'duration'], {
+        pool,
+        conflict: ['machine'],
+        update: ['duration'],
+        updateWhere: 'segments.duration = 0 OR EXCLUDED.duration > 0'
+      });
+    await sink.write([{ machine: `ä-${Math.random()}`, duration: 0 }]);
+    assert.ok(
+      queries[0].includes('WHERE segments.duration = 0 OR EXCLUDED.duration > 0'),
+      'conflict update did not keep the monotonic duration guard'
+    );
+  });
+
   it('routes write through an explicit client when provided', async () => {
     const pool = { query() { return Promise.reject(new Error('pool must not be used')); } };
     const clientQueries = [];