@yarkivaev/source-to-sink 1.3.2 → 1.3.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/source-to-sink",
-  "version": "1.3.2",
+  "version": "1.3.4",
   "description": "Generic library for building data streaming pipelines",
   "repository": {
     "type": "git",

package/src/modbusSource.js

@@ -4,7 +4,7 @@ import pollingSource from './pollingSource.js';
 /**
  * Idle state for Modbus connection.
  *
- * @returns {object} State with connected() returning false and no retry handle
+ * @returns {object} State with no open attempt
  */
 function idle() {
   return {
@@ -13,6 +13,9 @@ function idle() {
     },
     retrying() {
       return false;
+    },
+    opening() {
+      return false;
     }
   };
 }
@@ -31,12 +34,34 @@ function retrying(handle) {
     retrying() {
       return true;
     },
+    opening() {
+      return false;
+    },
     cancel() {
       clearTimeout(handle);
     }
   };
 }
 
+/**
+ * In-flight TCP or RTU connect state.
+ *
+ * @returns {object} State with opening() returning true
+ */
+function opening() {
+  return {
+    connected() {
+      return false;
+    },
+    retrying() {
+      return false;
+    },
+    opening() {
+      return true;
+    }
+  };
+}
+
 /**
  * Connected state for Modbus connection.
  *
@@ -51,12 +76,44 @@ function connected(client) {
     retrying() {
       return false;
     },
+    opening() {
+      return false;
+    },
     disconnect() {
       client.close();
     }
   };
 }
 
+/**
+ * Starts one connect attempt and ignores results after stop().
+ *
+ * @param {object} bag - Mutable lifecycle bag
+ */
+function attempt(bag) {
+  const token = bag.epoch;
+  bag.state = opening();
+  bag.connect(bag.client).then(() => {
+    if (token !== bag.epoch) {
+      bag.client.close();
+      return;
+    }
+    bag.delay = bag.interval;
+    bag.state = connected(bag.client);
+    bag.source.start();
+  }).catch((err) => {
+    if (token !== bag.epoch) {
+      return;
+    }
+    bag.log.error(`Connection to ${bag.target} failed, retrying in ${bag.delay}s: ${err.message}`);
+    const handle = setTimeout(() => {
+      attempt(bag);
+    }, bag.delay * 1000);
+    bag.state = retrying(handle);
+    bag.delay = Math.min(bag.delay * 2, bag.maxDelay);
+  });
+}
+
 /**
  * Shared Modbus polling lifecycle for TCP and RTU transports.
  *
@@ -64,60 +121,42 @@ function connected(client) {
  * @returns {object} Source with start() and stop() methods
  */
 function modbusPollingSource(options) {
-  const {
-    connect,
-    target,
-    address,
-    count,
-    interval,
-    collector,
-    clk,
-    client,
-    log
-  } = options;
-  let state = idle();
+  const { address, count, interval, collector, clk, client } = options;
   async function fetch() {
     const result = await client.readHoldingRegisters(address, count);
     return [result.data];
   }
-  const maxDelay = 300;
-  const source = pollingSource(fetch, interval, collector, clk);
-  let delay = interval;
-  function attempt() {
-    connect(client).then(() => {
-      delay = interval;
-      state = connected(client);
-      source.start();
-    }).catch((err) => {
-      log.error(`Connection to ${target} failed, retrying in ${delay}s: ${err.message}`);
-      const handle = setTimeout(attempt, delay * 1000);
-      state = retrying(handle);
-      delay = Math.min(delay * 2, maxDelay);
-    });
-  }
+  const bag = {
+    ...options,
+    state: idle(),
+    epoch: 0,
+    maxDelay: 300,
+    source: pollingSource(fetch, interval, collector, clk),
+    delay: interval
+  };
   return {
     /**
      * Connects to the Modbus device and starts polling.
      */
     start() {
-      if (state.connected() || state.retrying()) {
+      if (bag.state.connected() || bag.state.retrying() || bag.state.opening()) {
         return;
       }
-      attempt();
+      attempt(bag);
     },
     /**
      * Stops polling and disconnects from the Modbus device.
      */
     stop() {
-      source.stop();
-      if (state.retrying()) {
-        state.cancel();
-        state = idle();
+      bag.epoch += 1;
+      bag.source.stop();
+      if (bag.state.retrying()) {
+        bag.state.cancel();
       }
-      if (state.connected()) {
-        state.disconnect();
-        state = idle();
+      if (bag.state.connected() || bag.state.opening()) {
+        bag.client.close();
       }
+      bag.state = idle();
     }
   };
 }

package/src/postgresSink.js

@@ -1,30 +1,5 @@
 import pg from 'pg';
 
-/**
- * PostgreSQL sink for batch record insertion.
- *
- * Creates a pg.Pool internally and implements the Sink
- * interface for use with batch collectors. Supports optional
- * conflict resolution via ON CONFLICT DO NOTHING or
- * ON CONFLICT DO UPDATE SET.
- *
- * @example
- * const sink = postgresSink('postgresql://localhost:5432/db', 'metrics', ['ts', 'value']);
- * await sink.write([{ ts: Date.now(), value: 42 }]);
- *
- * @example
- * const sink = postgresSink('postgresql://localhost:5432/db', 'segments',
- *   ['machine', 'name', 'start_time'], { conflict: ['machine', 'start_time'] });
- * await sink.write([{ machine: 'a', name: 'on', start_time: '2024-01-01' }]);
- *
- * @param {string} url - PostgreSQL URL (e.g., 'postgresql://localhost:5432/db')
- * @param {string} table - Target table name
- * @param {Array<string>} columns - Column names for insertion
- * @param {object} [options] - Optional configuration
- * @param {Array<string>} [options.conflict] - Columns for ON CONFLICT clause
- * @param {Array<string>} [options.update] - Columns for DO UPDATE SET clause
- * @returns {object} Sink with write(records) method
- */
 /**
  * Removes duplicate records by conflict key within a batch.
  *
@@ -70,6 +45,31 @@ function buildSuffix(options) {
   return ` ON CONFLICT (${cols}) DO NOTHING`;
 }
 
+/**
+ * PostgreSQL sink for batch record insertion.
+ *
+ * Creates a pg.Pool internally unless `options.pool` is supplied.
+ * Supports optional conflict resolution via ON CONFLICT DO NOTHING or
+ * ON CONFLICT DO UPDATE SET. `write` accepts an optional queryable
+ * client so callers can run inserts inside an existing transaction.
+ *
+ * @example
+ * const sink = postgresSink('postgresql://localhost:5432/db', 'metrics', ['ts', 'value']);
+ * await sink.write([{ ts: Date.now(), value: 42 }]);
+ *
+ * @example
+ * const sink = postgresSink(url, 'segments', cols, { pool, conflict: ['machine', 'start_time'] });
+ * await sink.write([{ machine: 'a', name: 'on', start_time: '2024-01-01' }], client);
+ *
+ * @param {string} url - PostgreSQL URL (e.g., 'postgresql://localhost:5432/db')
+ * @param {string} table - Target table name
+ * @param {Array<string>} columns - Column names for insertion
+ * @param {object} [options] - Optional configuration
+ * @param {Array<string>} [options.conflict] - Columns for ON CONFLICT clause
+ * @param {Array<string>} [options.update] - Columns for DO UPDATE SET clause
+ * @param {object} [options.pool] - Existing pg Pool to reuse
+ * @returns {object} Sink with write(records, client?) method
+ */
 export default function postgresSink(url, table, columns, options = {}) {
   if (typeof url !== 'string' || url.length === 0) {
     throw new Error('URL must be a non-empty string');
@@ -80,7 +80,7 @@ export default function postgresSink(url, table, columns, options = {}) {
   if (!Array.isArray(columns) || columns.length === 0) {
     throw new Error('Columns must be a non-empty array');
   }
-  const pool = new pg.Pool({ connectionString: url });
+  const pool = options.pool || new pg.Pool({ connectionString: url });
   const suffix = buildSuffix(options);
   const conflict = Array.isArray(options.conflict) ? options.conflict : [];
   return {
@@ -88,9 +88,10 @@ export default function postgresSink(url, table, columns, options = {}) {
      * Writes records to PostgreSQL table.
      *
      * @param {Array} records - Array of objects with keys matching columns
+     * @param {object} [client] - Optional pg client/pool with query()
      * @returns {Promise} Promise resolving when insert completes
      */
-    write(records) {
+    write(records, client) {
       const unique = deduplicate(records, conflict);
       const placeholders = unique.map((record, i) => {
         const offset = i * columns.length;
@@ -99,7 +100,8 @@ export default function postgresSink(url, table, columns, options = {}) {
       }).join(', ');
       const query = `INSERT INTO ${table} (${columns.join(', ')}) VALUES ${placeholders}${suffix}`;
       const values = unique.flatMap((record) => {return columns.map((col) => {return record[col]})});
-      return pool.query(query, values);
+      const runner = client && typeof client.query === 'function' ? client : pool;
+      return runner.query(query, values);
     }
   };
 }

package/src/stompSource.js

@@ -30,30 +30,6 @@ function subscribed(channel) {
   };
 }
 
-/**
- * STOMP subscription source for streaming messages to a collector.
- *
- * Subscribes to a STOMP destination and forwards raw messages to the collector.
- * Messages are passed as {destination, payload} objects without parsing.
- * Uses ConnectFailover for automatic reconnection with exponential backoff
- * and Channel for transparent re-subscription. Acknowledges messages only
- * after the collector has accepted them.
- *
- * @example
- * const source = stompSource('stomp://localhost:61613', '/queue/sensors', collector);
- * source.start();
- * // ... later
- * source.stop();
- *
- * @param {string} url - STOMP broker URL (e.g., 'stomp://localhost:61613')
- * @param {string} destination - STOMP destination to subscribe (e.g., '/queue/sensors')
- * @param {object} collector - Collector with accept() method receiving {destination, payload}
- * @param {object} [options] - Optional STOMP connection options
- * @param {string} [options.login] - Login for STOMP auth
- * @param {string} [options.passcode] - Passcode for STOMP auth
- * @param {string} [options.host] - Virtual host for STOMP connection
- * @returns {object} Source with start() and stop() methods
- */
 /**
  * Builds STOMP server configuration from a URL.
  *
@@ -73,6 +49,98 @@ function serverConfig(parsed, options) {
   };
 }
 
+/**
+ * Builds settle helpers for one STOMP frame.
+ *
+ * @param {object} channel - STOMP channel
+ * @param {object} message - STOMP frame
+ * @returns {{ ack: Function, nack: Function }}
+ */
+function settleOf(channel, message) {
+  return {
+    ack() {
+      channel.ack(message);
+    },
+    nack() {
+      channel.nack(message);
+    }
+  };
+}
+
+/**
+ * Delivers one STOMP body to the collector and settles the frame.
+ *
+ * @param {object} ctx - delivery context with channel/message/destination/collector/manualAck
+ * @param {string} body - UTF-8 payload
+ * @returns {Promise<void>}
+ */
+async function deliver(ctx, body) {
+  const settle = settleOf(ctx.channel, ctx.message);
+  try {
+    await ctx.collector.accept({ destination: ctx.destination, payload: body, settle });
+    if (!ctx.manualAck) {
+      settle.ack();
+    }
+  } catch (error) {
+    if (!ctx.manualAck) {
+      settle.nack();
+    }
+    throw error;
+  }
+}
+
+/**
+ * Ignores a rejected delivery promise when running non-serial.
+ *
+ * @param {Error} error - rejected reason
+ * @returns {undefined}
+ */
+function ignoreDelivery(error) {
+  return error;
+}
+
+/**
+ * Subscribes a channel and routes frames to the collector.
+ *
+ * @param {object} channel - STOMP channel
+ * @param {string} destination - queue destination
+ * @param {object} collector - message collector
+ * @param {boolean} serial - global serial delivery
+ * @param {boolean} manualAck - collector settles frames
+ * @returns {undefined}
+ */
+function bind(channel, destination, collector, serial, manualAck) {
+  let pending = Promise.resolve();
+  channel.subscribe({ destination, ack: 'client-individual' }, (err, message) => {
+    if (err) { return; }
+    message.readString('utf-8', (readErr, body) => {
+      if (readErr) { return; }
+      const ctx = { channel, message, destination, collector, manualAck };
+      function work() {
+        return deliver(ctx, body.toString());
+      }
+      if (serial) {
+        pending = pending.then(work, work);
+      } else {
+        void work().catch(ignoreDelivery);
+      }
+    });
+  });
+}
+
+/**
+ * STOMP subscription source for streaming messages to a collector.
+ *
+ * Messages are `{destination, payload, settle}`. Default ack-after-accept;
+ * with `manualAck` the collector owns settle. With `serial: false`
+ * deliveries are not globally sequenced.
+ *
+ * @param {string} url - STOMP broker URL
+ * @param {string} destination - STOMP destination
+ * @param {object} collector - Collector with accept()
+ * @param {object} [options] - Connection and delivery options
+ * @returns {object} Source with start() and stop() methods
+ */
 export default function stompSource(url, destination, collector, options = {}) {
   if (typeof url !== 'string' || url.length === 0) {
     throw new Error('URL must be a non-empty string');
@@ -84,11 +152,10 @@ export default function stompSource(url, destination, collector, options = {}) {
     throw new Error('Collector must have an accept() method');
   }
   const parsed = new URL(url);
+  const serial = options.serial !== false;
+  const manualAck = options.manualAck === true;
   let state = idle();
   return {
-    /**
-     * Starts subscribing to the STOMP destination.
-     */
     start() {
       if (state.subscribed()) {
         return;
@@ -99,25 +166,9 @@ export default function stompSource(url, destination, collector, options = {}) {
         maxReconnects: -1
       });
       const channel = new stompit.Channel(failover);
-      let pending = Promise.resolve();
-      const subscribe = { destination, ack: 'client-individual' };
-      channel.subscribe(subscribe, (err, message) => {
-        if (err) { return; }
-        message.readString('utf-8', (readErr, body) => {
-          if (readErr) { return; }
-          pending = pending.then(async () => {
-            try {
-              await collector.accept({ destination, payload: body.toString() });
-              channel.ack(message);
-            } catch { channel.nack(message); }
-          });
-        });
-      });
+      bind(channel, destination, collector, serial, manualAck);
       state = subscribed(channel);
     },
-    /**
-     * Stops subscribing to the STOMP destination.
-     */
     stop() {
       if (!state.subscribed()) {
         return;

package/test/test_modbusSource.js

@@ -1,8 +1,89 @@
 import assert from 'node:assert';
+import net from 'node:net';
 import { describe, it } from 'mocha';
 import modbusSource from '../src/modbusSource.js';
 import fakeClock from './fakeClock.js';
 
+/**
+ * TCP listener that counts concurrent sockets on an ephemeral port.
+ *
+ * @returns {Promise<object>} Gate with port, live, peak, close
+ */
+function countingListen() {
+    let live = 0;
+    let peak = 0;
+    const sockets = new Set();
+    const server = net.createServer((sock) => {
+        live += 1;
+        if (live > peak) {
+            peak = live;
+        }
+        sockets.add(sock);
+        sock.on('close', () => {
+            sockets.delete(sock);
+            live -= 1;
+        });
+    });
+    return new Promise((resolve) => {
+        server.listen(0, '127.0.0.1', () => {
+            resolve({
+                port: server.address().port,
+                live() {
+                    return live;
+                },
+                peak() {
+                    return peak;
+                },
+                close() {
+                    for (const sock of sockets) {
+                        sock.destroy();
+                    }
+                    return new Promise((done) => {
+                        server.close(done);
+                    });
+                }
+            });
+        });
+    });
+}
+
+/**
+ * Resolves after the given delay.
+ *
+ * @param {number} ms - Delay in milliseconds
+ * @returns {Promise<void>} Timer promise
+ */
+function settle(ms) {
+    return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+    });
+}
+
+/**
+ * Waits until probe is true or the timeout elapses.
+ *
+ * @param {Function} probe - Condition
+ * @param {number} limit - Timeout in milliseconds
+ * @returns {Promise<void>} Resolves when probe holds
+ */
+function until(probe, limit) {
+    const deadline = Date.now() + limit;
+    function tick(resolve, reject) {
+        if (probe()) {
+            resolve();
+            return;
+        }
+        if (Date.now() >= deadline) {
+            reject(new Error('condition did not hold before timeout'));
+            return;
+        }
+        setTimeout(() => {
+            tick(resolve, reject);
+        }, 20);
+    }
+    return new Promise(tick);
+}
+
 describe('modbusSource', () => {
   it('throws on empty host', () => {
     const collector = { accept: () => {} };
@@ -118,4 +199,73 @@ describe('modbusSource', () => {
     source.stop();
     assert.strictEqual(true, true, 'Should not crash on unreachable host');
   });
+
+  it('does not open a second TCP socket when start is called twice', async function() {
+    const gate = await countingListen();
+    const source = modbusSource(
+      '127.0.0.1',
+      gate.port,
+      4000,
+      14,
+      5,
+      { accept() {} },
+      fakeClock(1 + Math.floor(Math.random() * 100)),
+      { error() {} }
+    );
+    source.start();
+    await until(() => {
+      return gate.live() >= 1;
+    }, 2000);
+    source.start();
+    await settle(150);
+    const peak = gate.peak();
+    source.stop();
+    await gate.close();
+    assert.strictEqual(peak, 1, 'second start opened another Modbus TCP socket');
+  });
+
+  it('closes the TCP socket when stop is called after connect', async function() {
+    const gate = await countingListen();
+    const source = modbusSource(
+      '127.0.0.1',
+      gate.port,
+      4000,
+      14,
+      5,
+      { accept() {} },
+      fakeClock(3 + Math.floor(Math.random() * 100)),
+      { error() {} }
+    );
+    source.start();
+    await until(() => {
+      return gate.live() >= 1;
+    }, 2000);
+    source.stop();
+    await until(() => {
+      return gate.live() === 0;
+    }, 2000);
+    const live = gate.live();
+    await gate.close();
+    assert.strictEqual(live, 0, 'stop after connect left a Modbus TCP socket open');
+  });
+
+  it('does not keep a TCP socket when stop is called during connect', async function() {
+    const gate = await countingListen();
+    const source = modbusSource(
+      '127.0.0.1',
+      gate.port,
+      4000,
+      14,
+      5,
+      { accept() {} },
+      fakeClock(7 + Math.floor(Math.random() * 100)),
+      { error() {} }
+    );
+    source.start();
+    source.stop();
+    await settle(400);
+    const live = gate.live();
+    await gate.close();
+    assert.strictEqual(live, 0, 'stop during connect left a Modbus TCP socket open');
+  });
 });

package/test/test_postgresSink.js

@@ -108,4 +108,31 @@ describe('postgresSink', () => {
     const result = deduplicate(records, []);
     assert.strictEqual(result.length, 2, 'Should not deduplicate without conflict columns');
   });
+
+  it('reuses an injected pool when writing', async () => {
+    const queries = [];
+    const pool = {
+      query(sql, values) {
+        queries.push({ sql, values });
+        return Promise.resolve({ rowCount: 1 });
+      }
+    };
+    const sink = postgresSink('postgresql://localhost:5432/db', 'metrics', ['ts', 'value'], { pool });
+    await sink.write([{ ts: 1700000000, value: 42 }]);
+    assert.strictEqual(queries.length, 1, 'injected pool was not used for write');
+  });
+
+  it('routes write through an explicit client when provided', async () => {
+    const pool = { query() { return Promise.reject(new Error('pool must not be used')); } };
+    const clientQueries = [];
+    const client = {
+      query(sql, values) {
+        clientQueries.push({ sql, values });
+        return Promise.resolve({ rowCount: 1 });
+      }
+    };
+    const sink = postgresSink('postgresql://localhost:5432/db', 'metrics', ['ts', 'value'], { pool });
+    await sink.write([{ ts: 1700000001, value: 7 }], client);
+    assert.strictEqual(clientQueries.length, 1, 'explicit client was not used for write');
+  });
 });

package/test/test_stompSource.js

@@ -51,4 +51,13 @@ describe('stompSource', () => {
     assert.strictEqual(typeof source.start, 'function', 'Should have start method');
     assert.strictEqual(typeof source.stop, 'function', 'Should have stop method');
   });
+
+  it('accepts manualAck and serial options without throwing', () => {
+    const collector = { accept: () => {} };
+    const source = stompSource('stomp://localhost:61613', '/queue/test', collector, {
+      manualAck: true,
+      serial: false
+    });
+    assert.strictEqual(typeof source.start, 'function', 'manualAck source missing start');
+  });
 });