@yarkivaev/source-to-sink 1.0.3 → 1.1.0

package/index.js

@@ -12,6 +12,8 @@
  * - mqttSource: MQTT subscription source (accepts URL)
  * - stompSource: STOMP subscription source (accepts URL)
  * - lokiSource: Loki polling source (accepts URL)
+ * - modbusSource: Modbus TCP polling source (accepts host/port)
+ * - mqttSink: MQTT publishing sink (accepts URL)
  *
  * @example
  * import { batch, circuit, timedBatch, clock, clickhouseSink, mqttSource } from 'source-to-sink';
@@ -33,3 +35,5 @@ export { default as postgresSink } from './src/postgresSink.js';
 export { default as mqttSource } from './src/mqttSource.js';
 export { default as stompSource } from './src/stompSource.js';
 export { default as lokiSource } from './src/lokiSource.js';
+export { default as modbusSource } from './src/modbusSource.js';
+export { default as mqttSink } from './src/mqttSink.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/source-to-sink",
-  "version": "1.0.3",
+  "version": "1.1.0",
   "description": "Generic library for building data streaming pipelines",
   "repository": {
     "type": "git",
@@ -12,7 +12,8 @@
     "mqtt": "^5.0.0",
     "@clickhouse/client": "^1.0.0",
     "stompit": "^1.0.0",
-    "pg": "^8.0.0"
+    "pg": "^8.0.0",
+    "modbus-serial": "^8.0.0"
   },
   "devDependencies": {
     "mocha": "^10.0.0",

package/src/modbusSource.js

@@ -0,0 +1,99 @@
+import ModbusRTU from 'modbus-serial';
+import pollingSource from './pollingSource.js';
+
+/**
+ * Idle state for Modbus connection.
+ *
+ * @returns {object} State with connected() returning false
+ */
+function idle() {
+  return {
+    connected() {
+      return false;
+    }
+  };
+}
+
+/**
+ * Connected state for Modbus connection.
+ *
+ * @param {object} client - ModbusRTU client
+ * @returns {object} State with connected() returning true
+ */
+function connected(client) {
+  return {
+    connected() {
+      return true;
+    },
+    disconnect() {
+      client.close();
+    }
+  };
+}
+
+/**
+ * Modbus TCP polling source for reading holding registers.
+ *
+ * Connects to a Modbus TCP device and polls holding registers
+ * at a specified interval. Forwards register arrays to the
+ * collector via pollingSource.
+ *
+ * @example
+ * const source = modbusSource('192.168.2.148', 502, 4000, 14, 5, collector, clock);
+ * source.start();
+ * // ... later
+ * source.stop();
+ *
+ * @param {string} host - Modbus TCP host address
+ * @param {number} port - Modbus TCP port
+ * @param {number} address - Starting register address
+ * @param {number} count - Number of registers to read
+ * @param {number} interval - Polling interval in seconds
+ * @param {object} collector - Collector with accept() method
+ * @param {object} clk - Clock with millis() method
+ * @returns {object} Source with start() and stop() methods
+ */
+export default function modbusSource(host, port, address, count, interval, collector, clk) {
+  if (typeof host !== 'string' || host.length === 0) {
+    throw new Error('Host must be a non-empty string');
+  }
+  if (typeof port !== 'number' || port <= 0) {
+    throw new Error(`Port must be a positive number, got: ${port}`);
+  }
+  if (typeof address !== 'number' || address < 0) {
+    throw new Error(`Address must be a non-negative number, got: ${address}`);
+  }
+  if (typeof count !== 'number' || count <= 0) {
+    throw new Error(`Count must be a positive number, got: ${count}`);
+  }
+  const client = new ModbusRTU();
+  let state = idle();
+  const fetch = async () => {
+    const result = await client.readHoldingRegisters(address, count);
+    return [result.data];
+  };
+  const source = pollingSource(fetch, interval, collector, clk);
+  return {
+    /**
+     * Connects to the Modbus device and starts polling.
+     */
+    async start() {
+      if (state.connected()) {
+        return;
+      }
+      await client.connectTCP(host, { port });
+      state = connected(client);
+      source.start();
+    },
+    /**
+     * Stops polling and disconnects from the Modbus device.
+     */
+    stop() {
+      source.stop();
+      if (state.connected()) {
+        state.disconnect();
+        state = idle();
+      }
+    }
+  };
+}

package/src/mqttSink.js

@@ -0,0 +1,98 @@
+import mqtt from 'mqtt';
+
+/**
+ * Idle state for MQTT sink.
+ *
+ * @returns {object} State with connected() returning false
+ */
+function idle() {
+  return {
+    connected() {
+      return false;
+    }
+  };
+}
+
+/**
+ * Connected state for MQTT sink.
+ *
+ * @param {object} client - MQTT client
+ * @returns {object} State with connected() returning true
+ */
+function connected(client) {
+  return {
+    connected() {
+      return true;
+    },
+    publish(topic, payload, qos) {
+      client.publish(topic, payload, { qos });
+    },
+    disconnect() {
+      client.end();
+    }
+  };
+}
+
+/**
+ * MQTT sink for publishing records to MQTT topics.
+ *
+ * Each record must have a topic and payload field.
+ * Creates an MQTT client internally and manages the connection.
+ *
+ * @example
+ * const sink = mqttSink('mqtt://localhost:1883');
+ * sink.start();
+ * sink.write([{ topic: 'sensors/temp', payload: '42.5' }]);
+ * sink.stop();
+ *
+ * @param {string} url - MQTT broker URL (e.g., 'mqtt://localhost:1883')
+ * @param {object} [options] - Optional MQTT connection options
+ * @param {string} [options.clientId] - Client ID for the connection
+ * @param {number} [options.qos] - QoS level for publishing (default 0)
+ * @returns {object} Sink with write(), start(), and stop() methods
+ */
+export default function mqttSink(url, options = {}) {
+  if (typeof url !== 'string' || url.length === 0) {
+    throw new Error('URL must be a non-empty string');
+  }
+  const qos = options.qos || 0;
+  let state = idle();
+  return {
+    /**
+     * Writes records to MQTT topics.
+     *
+     * @param {Array} records - Array of {topic, payload} records
+     */
+    write(records) {
+      if (!state.connected()) {
+        throw new Error('Sink is not connected');
+      }
+      for (const record of records) {
+        state.publish(record.topic, record.payload, qos);
+      }
+    },
+    /**
+     * Connects to the MQTT broker.
+     */
+    start() {
+      if (state.connected()) {
+        return;
+      }
+      const client = mqtt.connect(url, {
+        clientId: options.clientId,
+        protocolVersion: 5
+      });
+      state = connected(client);
+    },
+    /**
+     * Disconnects from the MQTT broker.
+     */
+    stop() {
+      if (!state.connected()) {
+        return;
+      }
+      state.disconnect();
+      state = idle();
+    }
+  };
+}

package/src/postgresSink.js

@@ -25,6 +25,35 @@ import pg from 'pg';
  * @param {Array<string>} [options.update] - Columns for DO UPDATE SET clause
  * @returns {object} Sink with write(records) method
  */
+/**
+ * Removes duplicate records by conflict key within a batch.
+ *
+ * PostgreSQL rejects INSERT batches where ON CONFLICT would update
+ * the same row twice. This keeps the last occurrence for each
+ * unique combination of conflict column values.
+ *
+ * @example
+ *   deduplicate(
+ *     [{ machine: 'a', start: 1, name: 'pending' },
+ *      { machine: 'a', start: 1, name: 'completed' }],
+ *     ['machine', 'start']
+ *   );
+ *   // => [{ machine: 'a', start: 1, name: 'completed' }]
+ *
+ * @param {Array} records - Array of record objects
+ * @param {Array<string>} keys - Conflict column names
+ * @returns {Array} Deduplicated records
+ */
+export function deduplicate(records, keys) {
+  if (keys.length === 0) return records;
+  const seen = new Map();
+  for (const record of records) {
+    const key = keys.map(k => record[k]).join('\0');
+    seen.set(key, record);
+  }
+  return Array.from(seen.values());
+}
+
 /**
  * Builds the ON CONFLICT SQL suffix from options.
  *
@@ -53,6 +82,7 @@ export default function postgresSink(url, table, columns, options = {}) {
   }
   const pool = new pg.Pool({ connectionString: url });
   const suffix = buildSuffix(options);
+  const conflict = Array.isArray(options.conflict) ? options.conflict : [];
   return {
     /**
      * Writes records to PostgreSQL table.
@@ -61,13 +91,14 @@ export default function postgresSink(url, table, columns, options = {}) {
      * @returns {Promise} Promise resolving when insert completes
      */
     write(records) {
-      const placeholders = records.map((_, i) => {
+      const unique = deduplicate(records, conflict);
+      const placeholders = unique.map((_, i) => {
         const offset = i * columns.length;
         const row = columns.map((__, j) => `$${offset + j + 1}`).join(', ');
         return `(${row})`;
       }).join(', ');
       const query = `INSERT INTO ${table} (${columns.join(', ')}) VALUES ${placeholders}${suffix}`;
-      const values = records.flatMap((record) => columns.map((col) => record[col]));
+      const values = unique.flatMap((record) => columns.map((col) => record[col]));
       return pool.query(query, values);
     }
   };

package/test/integration_mqttSink.js

@@ -0,0 +1,101 @@
+import assert from 'node:assert';
+import { describe, it, before, after } from 'mocha';
+import { GenericContainer } from 'testcontainers';
+import mqtt from 'mqtt';
+import mqttSink from '../src/mqttSink.js';
+
+describe('mqttSink integration', function() {
+  let container;
+  let url;
+  let subscriber;
+
+  before(async function() {
+    this.timeout(120000);
+    container = await new GenericContainer('eclipse-mosquitto:2')
+      .withExposedPorts(1883)
+      .withCommand(['mosquitto', '-c', '/mosquitto-no-auth.conf'])
+      .withStartupTimeout(30000)
+      .start();
+    const host = container.getHost();
+    const port = container.getMappedPort(1883);
+    url = `mqtt://${host}:${port}`;
+    subscriber = mqtt.connect(url);
+    await new Promise((resolve, reject) => {
+      subscriber.on('connect', resolve);
+      subscriber.on('error', reject);
+      setTimeout(() => reject(new Error('MQTT connection timeout')), 10000);
+    });
+  });
+
+  after(async function() {
+    this.timeout(30000);
+    if (subscriber) {
+      subscriber.end();
+    }
+    if (container) {
+      await container.stop();
+    }
+  });
+
+  it('publishes records that subscriber receives', async function() {
+    this.timeout(10000);
+    const received = [];
+    const topic = `sink/${Math.random().toString(36).slice(2)}`;
+    subscriber.subscribe(topic);
+    subscriber.on('message', (t, payload) => {
+      received.push({ topic: t, payload: payload.toString() });
+    });
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+    const sink = mqttSink(url);
+    sink.start();
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+    const value = `${Math.random()}`;
+    sink.write([{ topic, payload: value }]);
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+    sink.stop();
+    assert.strictEqual(received.length >= 1, true, 'Subscriber should receive published record');
+  });
+
+  it('publishes unicode payload', async function() {
+    this.timeout(10000);
+    const received = [];
+    const topic = `юникод/${Math.random().toString(36).slice(2)}`;
+    subscriber.subscribe(topic);
+    subscriber.on('message', (t, payload) => {
+      if (t === topic) {
+        received.push(payload.toString());
+      }
+    });
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+    const sink = mqttSink(url);
+    sink.start();
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+    const text = `Данные температуры ${Math.random()}`;
+    sink.write([{ topic, payload: text }]);
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+    sink.stop();
+    assert.strictEqual(received.length >= 1, true, 'Subscriber should receive unicode payload');
+  });
+
+  it('does not publish after stop', async function() {
+    this.timeout(10000);
+    const received = [];
+    const topic = `stopped/${Math.random().toString(36).slice(2)}`;
+    subscriber.subscribe(topic);
+    subscriber.on('message', (t, payload) => {
+      if (t === topic) {
+        received.push(payload.toString());
+      }
+    });
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+    const sink = mqttSink(url);
+    sink.start();
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+    sink.stop();
+    assert.throws(
+      () => sink.write([{ topic, payload: `${Math.random()}` }]),
+      /not connected/,
+      'Should throw when writing after stop'
+    );
+  });
+});

package/test/test_modbusSource.js

@@ -0,0 +1,96 @@
+import assert from 'node:assert';
+import { describe, it } from 'mocha';
+import modbusSource from '../src/modbusSource.js';
+import fakeClock from './fakeClock.js';
+
+describe('modbusSource', () => {
+  it('throws on empty host', () => {
+    const collector = { accept: () => {} };
+    const clk = fakeClock(Math.floor(Math.random() * 10000));
+    assert.throws(
+      () => modbusSource('', 502, 4000, 14, 5, collector, clk),
+      /Host must be a non-empty string/,
+      'Should reject empty host'
+    );
+  });
+
+  it('throws on missing host', () => {
+    const collector = { accept: () => {} };
+    const clk = fakeClock(Math.floor(Math.random() * 10000));
+    assert.throws(
+      () => modbusSource(null, 502, 4000, 14, 5, collector, clk),
+      /Host must be a non-empty string/,
+      'Should reject null host'
+    );
+  });
+
+  it('throws on invalid port', () => {
+    const collector = { accept: () => {} };
+    const clk = fakeClock(Math.floor(Math.random() * 10000));
+    assert.throws(
+      () => modbusSource('192.168.2.148', -1, 4000, 14, 5, collector, clk),
+      /Port must be a positive number/,
+      'Should reject negative port'
+    );
+  });
+
+  it('throws on negative address', () => {
+    const collector = { accept: () => {} };
+    const clk = fakeClock(Math.floor(Math.random() * 10000));
+    assert.throws(
+      () => modbusSource('192.168.2.148', 502, -1, 14, 5, collector, clk),
+      /Address must be a non-negative number/,
+      'Should reject negative address'
+    );
+  });
+
+  it('throws on invalid count', () => {
+    const collector = { accept: () => {} };
+    const clk = fakeClock(Math.floor(Math.random() * 10000));
+    assert.throws(
+      () => modbusSource('192.168.2.148', 502, 4000, 0, 5, collector, clk),
+      /Count must be a positive number/,
+      'Should reject zero count'
+    );
+  });
+
+  it('throws on missing collector', () => {
+    const clk = fakeClock(Math.floor(Math.random() * 10000));
+    assert.throws(
+      () => modbusSource('192.168.2.148', 502, 4000, 14, 5, null, clk),
+      /Collector must have an accept\(\) method/,
+      'Should reject null collector'
+    );
+  });
+
+  it('throws on missing clock', () => {
+    const collector = { accept: () => {} };
+    assert.throws(
+      () => modbusSource('192.168.2.148', 502, 4000, 14, 5, collector, null),
+      /Clock must have a millis\(\) method/,
+      'Should reject null clock'
+    );
+  });
+
+  it('creates source with start and stop methods', () => {
+    const collector = { accept: () => {} };
+    const clk = fakeClock(Math.floor(Math.random() * 10000));
+    const source = modbusSource(`\u00e9${Math.random()}`, 502, 4000, 14, 5, collector, clk);
+    assert.strictEqual(typeof source.start, 'function', 'Should have start method');
+  });
+
+  it('creates source with stop method', () => {
+    const collector = { accept: () => {} };
+    const clk = fakeClock(Math.floor(Math.random() * 10000));
+    const source = modbusSource(`\u4e2d${Math.random()}`, 502, 4000, 14, 5, collector, clk);
+    assert.strictEqual(typeof source.stop, 'function', 'Should have stop method');
+  });
+
+  it('stops without error when not started', () => {
+    const collector = { accept: () => {} };
+    const clk = fakeClock(Math.floor(Math.random() * 10000));
+    const source = modbusSource(`\u3042${Math.random()}`, 502, 4000, 14, 5, collector, clk);
+    source.stop();
+    assert.strictEqual(true, true, 'Should not throw when stopping without start');
+  });
+});

package/test/test_mqttSink.js

@@ -0,0 +1,51 @@
+import assert from 'node:assert';
+import { describe, it } from 'mocha';
+import mqttSink from '../src/mqttSink.js';
+
+describe('mqttSink', () => {
+  it('throws on empty url', () => {
+    assert.throws(
+      () => mqttSink(''),
+      /URL must be a non-empty string/,
+      'Should reject empty URL'
+    );
+  });
+
+  it('throws on missing url', () => {
+    assert.throws(
+      () => mqttSink(null),
+      /URL must be a non-empty string/,
+      'Should reject null URL'
+    );
+  });
+
+  it('creates sink with write method', () => {
+    const sink = mqttSink(`mqtt://\u00e9${Math.random()}`);
+    assert.strictEqual(typeof sink.write, 'function', 'Should have write method');
+  });
+
+  it('creates sink with start method', () => {
+    const sink = mqttSink(`mqtt://\u4e2d${Math.random()}`);
+    assert.strictEqual(typeof sink.start, 'function', 'Should have start method');
+  });
+
+  it('creates sink with stop method', () => {
+    const sink = mqttSink(`mqtt://\u3042${Math.random()}`);
+    assert.strictEqual(typeof sink.stop, 'function', 'Should have stop method');
+  });
+
+  it('throws on write when not connected', () => {
+    const sink = mqttSink(`mqtt://\u00df${Math.random()}`);
+    assert.throws(
+      () => sink.write([{ topic: `\u00e9${Math.random()}`, payload: `${Math.random()}` }]),
+      /Sink is not connected/,
+      'Should reject write when not connected'
+    );
+  });
+
+  it('stops without error when not started', () => {
+    const sink = mqttSink(`mqtt://\u0420${Math.random()}`);
+    sink.stop();
+    assert.strictEqual(true, true, 'Should not throw when stopping without start');
+  });
+});

package/test/test_postgresSink.js

@@ -1,6 +1,6 @@
 import assert from 'node:assert';
 import { describe, it } from 'mocha';
-import postgresSink from '../src/postgresSink.js';
+import postgresSink, { deduplicate } from '../src/postgresSink.js';
 
 describe('postgresSink', () => {
   it('throws on missing url', () => {
@@ -80,4 +80,32 @@ describe('postgresSink', () => {
       { conflict: ['machine', 'start_time'], update: ['name', 'end_time', 'duration'] });
     assert.strictEqual(typeof sink.write, 'function', 'Should have write method with update option');
   });
+
+  it('deduplicates batch by conflict columns keeping last occurrence', () => {
+    const records = [
+      { machine: 'ü-1', name: 'pending', start_time: '2024-01-01' },
+      { machine: 'ü-1', name: 'completed', start_time: '2024-01-01' },
+      { machine: 'ö-2', name: 'pending', start_time: '2024-01-02' }
+    ];
+    const result = deduplicate(records, ['machine', 'start_time']);
+    assert.strictEqual(result.length, 2, 'Should remove duplicate conflict key');
+  });
+
+  it('keeps last record when conflict columns match', () => {
+    const records = [
+      { machine: 'ü-1', name: 'pending', start_time: '2024-01-01' },
+      { machine: 'ü-1', name: 'completed', start_time: '2024-01-01' }
+    ];
+    const result = deduplicate(records, ['machine', 'start_time']);
+    assert.strictEqual(result[0].name, 'completed', 'Should keep the last occurrence');
+  });
+
+  it('returns records unchanged when no conflict columns', () => {
+    const records = [
+      { machine: 'ä-1', name: 'à', start_time: '2024-01-01' },
+      { machine: 'ä-1', name: 'à', start_time: '2024-01-01' }
+    ];
+    const result = deduplicate(records, []);
+    assert.strictEqual(result.length, 2, 'Should not deduplicate without conflict columns');
+  });
 });