@yarkivaev/source-to-sink 1.1.1 → 1.2.3

package/.github/workflows/ci.yml

@@ -9,4 +9,4 @@ jobs:
     uses: yarkivaev/npm-workflows/.github/workflows/ci.yml@main
     with:
       node-version: '22'
-      lint: false
+      lint: true

package/eslint.config.js

@@ -0,0 +1,176 @@
+import js from '@eslint/js';
+import globals from 'globals';
+
+export default [
+    js.configs.recommended,
+    {
+        languageOptions: {
+            ecmaVersion: 'latest',
+            sourceType: 'module',
+            globals: {
+                ...globals.node,
+                ...globals.mocha
+            }
+        },
+        rules: {
+            // Possible Problems
+            'array-callback-return': 'error',
+            'no-await-in-loop': 'error',
+            'no-constant-binary-expression': 'error',
+            'no-constructor-return': 'error',
+            'no-duplicate-imports': 'error',
+            'no-new-native-nonconstructor': 'error',
+            'no-promise-executor-return': 'error',
+            'no-self-compare': 'error',
+            'no-template-curly-in-string': 'error',
+            'no-unmodified-loop-condition': 'error',
+            'no-unreachable-loop': 'error',
+            'no-unused-private-class-members': 'error',
+            'no-use-before-define': 'error',
+            'require-atomic-updates': 'error',
+
+            // Suggestions
+            'accessor-pairs': 'error',
+            'arrow-body-style': ['error', 'always'],
+            'block-scoped-var': 'error',
+            'camelcase': 'error',
+            'consistent-return': 'error',
+            'consistent-this': ['error', 'self'],
+            'curly': ['error', 'all'],
+            'default-case': 'error',
+            'default-case-last': 'error',
+            'default-param-last': 'error',
+            'dot-notation': 'error',
+            'eqeqeq': ['error', 'always'],
+            'func-name-matching': 'error',
+            'func-names': ['error', 'always'],
+            'func-style': ['error', 'declaration'],
+            'grouped-accessor-pairs': ['error', 'getBeforeSet'],
+            'guard-for-in': 'error',
+            'id-denylist': 'off',
+            'id-length': ['error', { 'min': 2, 'exceptions': ['i', 'j', 'k', 'x', 'y', 'z', 'a', 'b', 'p', 'w', 'e', 'r'] }],
+            'max-classes-per-file': ['error', 1],
+            'max-depth': ['error', 3],
+            'max-lines': ['error', { 'max': 200, 'skipBlankLines': true, 'skipComments': true }],
+            'max-lines-per-function': ['error', { 'max': 50, 'skipBlankLines': true, 'skipComments': true }],
+            'max-nested-callbacks': ['error', 3],
+            'max-params': ['error', 5],
+            'max-statements': ['error', 15],
+            'new-cap': 'error',
+            'no-alert': 'error',
+            'no-array-constructor': 'error',
+            'no-bitwise': 'error',
+            'no-caller': 'error',
+            'no-console': 'error',
+            'no-continue': 'error',
+            'no-div-regex': 'error',
+            'no-else-return': 'error',
+            'no-empty-function': 'error',
+            'no-eq-null': 'error',
+            'no-eval': 'error',
+            'no-extend-native': 'error',
+            'no-extra-bind': 'error',
+            'no-extra-label': 'error',
+            'no-implicit-coercion': 'error',
+            'no-implicit-globals': 'error',
+            'no-implied-eval': 'error',
+            'no-inline-comments': 'error',
+            'no-invalid-this': 'error',
+            'no-iterator': 'error',
+            'no-label-var': 'error',
+            'no-labels': 'error',
+            'no-lone-blocks': 'error',
+            'no-lonely-if': 'error',
+            'no-loop-func': 'error',
+            'no-magic-numbers': 'off',
+            'no-multi-assign': 'error',
+            'no-multi-str': 'error',
+            'no-negated-condition': 'error',
+            'no-nested-ternary': 'error',
+            'no-new': 'error',
+            'no-new-func': 'error',
+            'no-new-wrappers': 'error',
+            'no-object-constructor': 'error',
+            'no-octal-escape': 'error',
+            'no-param-reassign': 'error',
+            'no-plusplus': 'error',
+            'no-proto': 'error',
+            'no-return-assign': 'error',
+            'no-script-url': 'error',
+            'no-sequences': 'error',
+            'no-shadow': 'error',
+            'no-throw-literal': 'error',
+            'no-undef-init': 'error',
+            'no-undefined': 'off',
+            'no-underscore-dangle': 'error',
+            'no-unneeded-ternary': 'error',
+            'no-unused-expressions': 'error',
+            'no-useless-call': 'error',
+            'no-useless-computed-key': 'error',
+            'no-useless-concat': 'error',
+            'no-useless-constructor': 'error',
+            'no-useless-rename': 'error',
+            'no-useless-return': 'error',
+            'no-var': 'error',
+            'no-void': 'off',
+            'no-warning-comments': 'warn',
+            'object-shorthand': ['error', 'always'],
+            'one-var': ['error', 'never'],
+            'operator-assignment': ['error', 'always'],
+            'prefer-arrow-callback': 'error',
+            'prefer-const': 'error',
+            'prefer-destructuring': ['error', { 'array': false, 'object': true }],
+            'prefer-exponentiation-operator': 'error',
+            'prefer-named-capture-group': 'error',
+            'prefer-numeric-literals': 'error',
+            'prefer-object-has-own': 'error',
+            'prefer-object-spread': 'error',
+            'prefer-promise-reject-errors': 'error',
+            'prefer-regex-literals': 'error',
+            'prefer-rest-params': 'error',
+            'prefer-spread': 'error',
+            'prefer-template': 'error',
+            'radix': 'error',
+            'require-await': 'error',
+            'require-unicode-regexp': 'error',
+            'sort-imports': ['error', { 'ignoreCase': true, 'ignoreDeclarationSort': true }],
+            'sort-keys': 'off',
+            'sort-vars': 'error',
+            'strict': ['error', 'never'],
+            'symbol-description': 'error',
+            'vars-on-top': 'error',
+            'yoda': 'error',
+
+            // Layout & Formatting
+            'line-comment-position': ['error', { 'position': 'above' }],
+            'unicode-bom': ['error', 'never']
+        }
+    },
+    {
+        files: ['src/modbusSource.js'],
+        rules: {
+            'max-params': ['error', 8],
+            'max-lines-per-function': ['error', { 'max': 60, 'skipBlankLines': true, 'skipComments': true }],
+            'max-statements': ['error', 20]
+        }
+    },
+    {
+        files: ['test/**/*.js'],
+        rules: {
+            'max-lines': 'off',
+            'max-lines-per-function': 'off',
+            'max-statements': 'off',
+            'max-nested-callbacks': ['error', 5],
+            'prefer-arrow-callback': 'off',
+            'func-names': 'off',
+            'id-denylist': 'off',
+            'no-undefined': 'off',
+            'no-empty-function': 'off',
+            'func-style': 'off',
+            'no-invalid-this': 'off',
+            'no-await-in-loop': 'off',
+            'camelcase': ['error', { 'properties': 'never' }],
+            'id-length': ['error', { 'min': 2, 'exceptions': ['i', 'j', 'k', 'x', 'y', 'z', 'a', 'b', 'p', 'w', 'e', 'r', 'c', 't', 'v', '_'] }]
+        }
+    }
+];

package/index.js

@@ -37,3 +37,4 @@ export { default as stompSource } from './src/stompSource.js';
 export { default as lokiSource } from './src/lokiSource.js';
 export { default as modbusSource } from './src/modbusSource.js';
 export { default as mqttSink } from './src/mqttSink.js';
+export { default as sqliteSink } from './src/sqliteSink.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/source-to-sink",
-  "version": "1.1.1",
+  "version": "1.2.3",
   "description": "Generic library for building data streaming pipelines",
   "repository": {
     "type": "git",
@@ -13,15 +13,21 @@
     "@clickhouse/client": "^1.0.0",
     "stompit": "^1.0.0",
     "pg": "^8.0.0",
-    "modbus-serial": "^8.0.0"
+    "modbus-serial": "^8.0.0",
+    "better-sqlite3": "^11.0.0"
   },
   "devDependencies": {
-    "mocha": "^10.0.0",
+    "@eslint/js": "^9.39.2",
     "c8": "^8.0.0",
+    "eslint": "^9.39.2",
+    "globals": "^17.0.0",
+    "mocha": "^10.0.0",
     "testcontainers": "^10.13.0"
   },
   "scripts": {
-    "test": "mocha test/*.js",
-    "coverage": "c8 mocha test/*.js"
+    "test": "mocha test/*.js --exit",
+    "coverage": "c8 mocha test/*.js",
+    "lint": "eslint .",
+    "lint:fix": "eslint . --fix"
   }
 }

package/src/batch.js

@@ -30,7 +30,7 @@ export default function batch(sink, size, circuit) {
     throw new Error('Circuit must have an allowing() method');
   }
   let records = [];
-  const perform = () => {
+  function perform() {
     if (records.length === 0) {
       return;
     }
@@ -38,15 +38,23 @@ export default function batch(sink, size, circuit) {
       return;
     }
     const pending = records;
+    records = [];
     try {
-      sink.write(pending);
-      records = [];
-      circuit.succeed();
+      const result = sink.write(pending);
+      if (result && typeof result.then === 'function') {
+        result.then(
+          () => { circuit.succeed(); },
+          () => { circuit.fail(); records = pending.concat(records); }
+        );
+      } else {
+        circuit.succeed();
+      }
     } catch (err) {
       circuit.fail();
+      records = pending.concat(records);
       throw err;
     }
-  };
+  }
   return {
     /**
      * Accepts a record into the batch.

package/src/lokiSource.js

@@ -26,9 +26,9 @@ export default function lokiSource(url, query, interval, collector, clk) {
   if (typeof query !== 'string' || query.length === 0) {
     throw new Error('Query must be a non-empty string');
   }
-  const fetch = async (since, until) => {
+  async function fetch(since, until) {
     const params = new URLSearchParams({
-      query: query,
+      query,
       start: (since * 1000000).toString(),
       end: (until * 1000000).toString(),
       limit: '1000'
@@ -44,6 +44,6 @@ export default function lokiSource(url, query, interval, collector, clk) {
       }
     }
     return entries;
-  };
+  }
   return pollingSource(fetch, interval, collector, clk);
 }

package/src/modbusSource.js

@@ -95,14 +95,14 @@ export default function modbusSource(host, port, address, count, interval, colle
   }
   const client = new ModbusRTU();
   let state = idle();
-  const fetch = async () => {
+  async function fetch() {
     const result = await client.readHoldingRegisters(address, count);
     return [result.data];
-  };
+  }
   const maxDelay = 300;
   const source = pollingSource(fetch, interval, collector, clk);
   let delay = interval;
-  const attempt = () => {
+  function attempt() {
     client.connectTCP(host, { port }).then(() => {
       delay = interval;
       state = connected(client);
@@ -113,7 +113,7 @@ export default function modbusSource(host, port, address, count, interval, colle
       state = retrying(handle);
       delay = Math.min(delay * 2, maxDelay);
     });
-  };
+  }
   return {
     /**
      * Connects to the Modbus device and starts polling.

package/src/mqttSource.js

@@ -65,7 +65,7 @@ export default function mqttSource(url, topics, collector, options = {}) {
   if (!collector || typeof collector.accept !== 'function') {
     throw new Error('Collector must have an accept() method');
   }
-  const list = topics.split(',').map((t) => t.trim()).filter((t) => t.length > 0);
+  const list = topics.split(',').map((tp) => {return tp.trim()}).filter((tp) => {return tp.length > 0});
   let state = idle();
   return {
     /**
@@ -77,15 +77,15 @@ export default function mqttSource(url, topics, collector, options = {}) {
       }
       const client = mqtt.connect(url, {
         clientId: options.clientId,
-        clean: options.clientId ? false : true,
+        clean: !options.clientId,
         protocolVersion: 5,
         properties: options.clientId ? {
           sessionExpiryInterval: options.sessionExpiryInterval || 3600
         } : undefined
       });
-      const handler = (t, message) => {
-        collector.accept({ topic: t, payload: message.toString() });
-      };
+      function handler(tp, message) {
+        collector.accept({ topic: tp, payload: message.toString() });
+      }
       client.on('message', handler);
       client.on('connect', () => {
         client.subscribe(list, { qos: options.clientId ? 1 : 0 });

package/src/pollingSource.js

@@ -64,14 +64,15 @@ export default function pollingSource(fetch, interval, collector, clk) {
   }
   let state = idle();
   let since = clk.millis();
-  const poll = async () => {
+  async function poll() {
+    const from = since;
     const until = clk.millis();
-    const result = await fetch(since, until);
     since = until;
+    const result = await fetch(from, until);
     for (const entry of result) {
       collector.accept(entry);
     }
-  };
+  }
   return {
     /**
      * Starts polling.

package/src/postgresSink.js

@@ -45,10 +45,10 @@ import pg from 'pg';
  * @returns {Array} Deduplicated records
  */
 export function deduplicate(records, keys) {
-  if (keys.length === 0) return records;
+  if (keys.length === 0) {return records;}
   const seen = new Map();
   for (const record of records) {
-    const key = keys.map(k => record[k]).join('\0');
+    const key = keys.map(k => {return record[k]}).join('\0');
     seen.set(key, record);
   }
   return Array.from(seen.values());
@@ -61,10 +61,10 @@ export function deduplicate(records, keys) {
  * @returns {string} SQL suffix or empty string
  */
 function buildSuffix(options) {
-  if (!Array.isArray(options.conflict) || options.conflict.length === 0) return '';
+  if (!Array.isArray(options.conflict) || options.conflict.length === 0) {return '';}
   const cols = options.conflict.join(', ');
   if (Array.isArray(options.update) && options.update.length > 0) {
-    const sets = options.update.map(c => `${c} = EXCLUDED.${c}`).join(', ');
+    const sets = options.update.map((col) => {return `${col} = EXCLUDED.${col}`}).join(', ');
     return ` ON CONFLICT (${cols}) DO UPDATE SET ${sets}`;
   }
   return ` ON CONFLICT (${cols}) DO NOTHING`;
@@ -92,13 +92,13 @@ export default function postgresSink(url, table, columns, options = {}) {
      */
     write(records) {
       const unique = deduplicate(records, conflict);
-      const placeholders = unique.map((_, i) => {
+      const placeholders = unique.map((record, i) => {
         const offset = i * columns.length;
-        const row = columns.map((__, j) => `$${offset + j + 1}`).join(', ');
+        const row = columns.map((__, j) => {return `$${offset + j + 1}`}).join(', ');
         return `(${row})`;
       }).join(', ');
       const query = `INSERT INTO ${table} (${columns.join(', ')}) VALUES ${placeholders}${suffix}`;
-      const values = unique.flatMap((record) => columns.map((col) => record[col]));
+      const values = unique.flatMap((record) => {return columns.map((col) => {return record[col]})});
       return pool.query(query, values);
     }
   };

package/src/sqliteSink.js

@@ -0,0 +1,55 @@
+import Database from 'better-sqlite3';
+
+/**
+ * SQLite sink for batch record insertion.
+ *
+ * Opens a SQLite database in WAL mode and implements the Sink
+ * interface for use with batch collectors. Creates the target
+ * table with the given columns if it does not exist.
+ *
+ * @example
+ * const sink = sqliteSink('/tmp/metrics.db', 'metrics', ['topic', 'ts', 'value']);
+ * sink.write([{ topic: 'sensor/1', ts: 1700000000000, value: 42 }]);
+ *
+ * @param {string} path - Filesystem path to the SQLite database file
+ * @param {string} table - Target table name
+ * @param {Array<string>} columns - Column names for insertion
+ * @returns {object} Sink with write(records) and close() methods
+ */
+export default function sqliteSink(path, table, columns) {
+  if (typeof path !== 'string' || path.length === 0) {
+    throw new Error('Path must be a non-empty string');
+  }
+  if (typeof table !== 'string' || table.length === 0) {
+    throw new Error('Table must be a non-empty string');
+  }
+  if (!Array.isArray(columns) || columns.length === 0) {
+    throw new Error('Columns must be a non-empty array');
+  }
+  const db = new Database(path);
+  db.pragma('journal_mode = WAL');
+  const placeholders = columns.map(() => {return '?'}).join(', ');
+  const sql = `INSERT INTO ${table} (${columns.join(', ')}) VALUES (${placeholders})`;
+  const statement = db.prepare(sql);
+  const transaction = db.transaction((records) => {
+    for (const record of records) {
+      statement.run(...columns.map((col) => {return record[col]}));
+    }
+  });
+  return {
+    /**
+     * Writes records to SQLite table inside a transaction.
+     *
+     * @param {Array} records - Array of objects with keys matching columns
+     */
+    write(records) {
+      transaction(records);
+    },
+    /**
+     * Closes the database connection.
+     */
+    close() {
+      db.close();
+    }
+  };
+}

package/src/stompSource.js

@@ -54,6 +54,25 @@ function subscribed(channel) {
  * @param {string} [options.host] - Virtual host for STOMP connection
  * @returns {object} Source with start() and stop() methods
  */
+/**
+ * Builds STOMP server configuration from a URL.
+ *
+ * @param {URL} parsed - Parsed URL object
+ * @param {object} options - Connection options
+ * @returns {object} Server config for ConnectFailover
+ */
+function serverConfig(parsed, options) {
+  return {
+    host: parsed.hostname,
+    port: parseInt(parsed.port, 10) || 61613,
+    connectHeaders: {
+      host: options.host || parsed.hostname,
+      login: options.login || '',
+      passcode: options.passcode || ''
+    }
+  };
+}
+
 export default function stompSource(url, destination, collector, options = {}) {
   if (typeof url !== 'string' || url.length === 0) {
     throw new Error('URL must be a non-empty string');
@@ -74,17 +93,7 @@ export default function stompSource(url, destination, collector, options = {}) {
       if (state.subscribed()) {
         return;
       }
-      const headers = {
-        host: options.host || parsed.hostname,
-        login: options.login || '',
-        passcode: options.passcode || ''
-      };
-      const server = {
-        host: parsed.hostname,
-        port: parseInt(parsed.port, 10) || 61613,
-        connectHeaders: headers
-      };
-      const failover = new stompit.ConnectFailover([server], {
+      const failover = new stompit.ConnectFailover([serverConfig(parsed, options)], {
         initialReconnectDelay: 10,
         maxReconnectDelay: 30000,
         maxReconnects: -1

package/src/timedBatch.js

@@ -63,7 +63,7 @@ export default function timedBatch(origin, interval) {
     throw new Error(`Interval must be a positive number, got: ${interval}`);
   }
   let timer = idle();
-  const schedule = () => {
+  function schedule() {
     if (!timer.scheduled()) {
       const handle = setTimeout(() => {
         timer = idle();
@@ -71,11 +71,11 @@ export default function timedBatch(origin, interval) {
       }, interval * 1000);
       timer = scheduled(handle);
     }
-  };
-  const cancel = () => {
+  }
+  function cancel() {
     timer.cancel();
     timer = idle();
-  };
+  }
   return {
     /**
      * Accepts a record and schedules flush timer.

package/test/integration_clickhouseSink.js

@@ -1,5 +1,5 @@
 import assert from 'node:assert';
-import { describe, it, before, after } from 'mocha';
+import { after, before, describe, it } from 'mocha';
 import { GenericContainer } from 'testcontainers';
 import { createClient } from '@clickhouse/client';
 import clickhouseSink from '../src/clickhouseSink.js';
@@ -26,7 +26,7 @@ describe('clickhouseSink integration', function() {
         break;
       } catch {
         retries -= 1;
-        await new Promise((resolve) => setTimeout(resolve, 1000));
+        await new Promise((resolve) => { setTimeout(resolve, 1000); });
       }
     }
     await client.command({ query: 'CREATE DATABASE IF NOT EXISTS test' });
@@ -55,7 +55,7 @@ describe('clickhouseSink integration', function() {
     const ts = Date.now();
     const sink = clickhouseSink(url, 'test.metrics');
     await sink.write([{ topic, ts, value }]);
-    await new Promise((resolve) => setTimeout(resolve, 500));
+    await new Promise((resolve) => { setTimeout(resolve, 500); });
     const result = await client.query({
       query: `SELECT * FROM test.metrics WHERE topic = '${topic}'`,
       format: 'JSONEachRow'
@@ -73,7 +73,7 @@ describe('clickhouseSink integration', function() {
     ];
     const sink = clickhouseSink(url, 'test.metrics');
     await sink.write(records);
-    await new Promise((resolve) => setTimeout(resolve, 500));
+    await new Promise((resolve) => { setTimeout(resolve, 500); });
     const result = await client.query({
       query: `SELECT * FROM test.metrics WHERE topic = '${topic}'`,
       format: 'JSONEachRow'
@@ -88,7 +88,7 @@ describe('clickhouseSink integration', function() {
     const ts = Date.now();
     const sink = clickhouseSink(url, 'test.metrics');
     await sink.write([{ topic, ts, value }]);
-    await new Promise((resolve) => setTimeout(resolve, 500));
+    await new Promise((resolve) => { setTimeout(resolve, 500); });
     const result = await client.query({
       query: `SELECT * FROM test.metrics WHERE topic = '${topic}'`,
       format: 'JSONEachRow'

package/test/integration_lokiSource.js

@@ -1,5 +1,5 @@
 import assert from 'node:assert';
-import { describe, it, before, after } from 'mocha';
+import { after, before, describe, it } from 'mocha';
 import { GenericContainer } from 'testcontainers';
 import lokiSource from '../src/lokiSource.js';
 import clock from '../src/clock.js';
@@ -50,7 +50,7 @@ describe('lokiSource integration', function() {
         // Loki not ready yet
       }
       retries -= 1;
-      await new Promise((resolve) => setTimeout(resolve, 1000));
+      await new Promise((resolve) => { setTimeout(resolve, 1000); });
     }
   });
 
@@ -65,13 +65,13 @@ describe('lokiSource integration', function() {
     this.timeout(30000);
     const received = [];
     const app = `app${Math.random().toString(36).slice(2)}`;
-    const collector = { accept: (entry) => received.push(entry) };
+    const collector = { accept: (entry) => {return received.push(entry)} };
     const clk = clock();
     const source = lokiSource(url, `{app="${app}"}`, 0.5, collector, clk);
     source.start();
-    await new Promise((resolve) => setTimeout(resolve, 500));
+    await new Promise((resolve) => { setTimeout(resolve, 500); });
     await pushToLoki(url, { app }, `test entry ${Math.random()}`);
-    await new Promise((resolve) => setTimeout(resolve, 3000));
+    await new Promise((resolve) => { setTimeout(resolve, 3000); });
     source.stop();
     assert.strictEqual(received.length >= 1, true, 'Should receive entries from Loki');
   });
@@ -80,13 +80,13 @@ describe('lokiSource integration', function() {
     this.timeout(30000);
     const received = [];
     const app = `app${Math.random().toString(36).slice(2)}`;
-    const collector = { accept: (entry) => received.push(entry) };
+    const collector = { accept: (entry) => {return received.push(entry)} };
     const clk = clock();
     const source = lokiSource(url, `{app="${app}"}`, 0.5, collector, clk);
     source.start();
-    await new Promise((resolve) => setTimeout(resolve, 500));
+    await new Promise((resolve) => { setTimeout(resolve, 500); });
     await pushToLoki(url, { app }, `тест запись ${Math.random()}`);
-    await new Promise((resolve) => setTimeout(resolve, 3000));
+    await new Promise((resolve) => { setTimeout(resolve, 3000); });
     source.stop();
     assert.strictEqual(received.length >= 1, true, 'Should receive entries with unicode');
   });
@@ -95,15 +95,15 @@ describe('lokiSource integration', function() {
     this.timeout(30000);
     const received = [];
     const app = `app${Math.random().toString(36).slice(2)}`;
-    const collector = { accept: (entry) => received.push(entry) };
+    const collector = { accept: (entry) => {return received.push(entry)} };
     const clk = clock();
     const source = lokiSource(url, `{app="${app}"}`, 0.5, collector, clk);
     source.start();
-    await new Promise((resolve) => setTimeout(resolve, 1000));
+    await new Promise((resolve) => { setTimeout(resolve, 1000); });
     source.stop();
-    const before = received.length;
+    const snapshot = received.length;
     await pushToLoki(url, { app }, `after stop ${Math.random()}`);
-    await new Promise((resolve) => setTimeout(resolve, 2000));
-    assert.strictEqual(received.length, before, 'Should not receive entries after stop');
+    await new Promise((resolve) => { setTimeout(resolve, 2000); });
+    assert.strictEqual(received.length, snapshot, 'Should not receive entries after stop');
   });
 });