@yarkivaev/source-to-sink 1.0.0

package/.claude/settings.local.json

@@ -0,0 +1,17 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(npm test)",
+      "Bash(npm run coverage:*)",
+      "Bash(npx c8 npm test:*)",
+      "Bash(npm link)",
+      "Bash(npm link:*)",
+      "Bash(npm run test:integration:*)",
+      "Bash(npm run test:all:*)",
+      "Bash(npm test:*)",
+      "Bash(find:*)",
+      "Bash(poetry run pytest:*)",
+      "Bash(grep:*)"
+    ]
+  }
+}

package/.github/workflows/ci.yml

@@ -0,0 +1,12 @@
+name: CI
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  ci:
+    uses: yarkivaev/npm-workflows/.github/workflows/ci.yml@main
+    with:
+      node-version: '22'
+      lint: false

package/.github/workflows/release.yml

@@ -0,0 +1,15 @@
+name: Release
+
+on:
+  push:
+    branches: [main]
+
+jobs:
+  release:
+    uses: yarkivaev/npm-workflows/.github/workflows/release.yml@main
+    permissions:
+      contents: write
+      id-token: write
+    with:
+      node-version: '22'
+      lint: false

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,40 @@
+# source-to-sink
+
+A library for building data streaming pipelines with batching and circuit breaker support.
+
+## Installation
+
+```bash
+npm install source-to-sink
+```
+
+## Usage
+
+```javascript
+import { batch, circuit, timedBatch, clock, clickhouseSink, mqttSource } from 'source-to-sink';
+
+const clk = clock();
+const breaker = circuit(5, 60, clk);
+const sink = clickhouseSink('http://localhost:8123', 'metrics');
+const collector = timedBatch(batch(sink, 1000, breaker), 5.0);
+const source = mqttSource('mqtt://localhost:1883', 'sensors/#', collector);
+
+source.start();
+```
+
+## Components
+
+| Component | Description |
+|-----------|-------------|
+| `batch(sink, size, circuit)` | Collects records and flushes to sink when size is reached |
+| `circuit(threshold, timeout, clock)` | Circuit breaker for failure isolation |
+| `timedBatch(collector, interval)` | Adds time-based auto-flush to a collector |
+| `clock()` | System time provider |
+| `pollingSource(fetch, interval, collector, clock)` | Generic polling source with time window |
+| `clickhouseSink(url, table)` | ClickHouse sink adapter |
+| `mqttSource(url, topic, collector)` | MQTT subscription source |
+| `lokiSource(url, query, interval, collector, clock)` | Loki polling source |
+
+## License
+
+MIT

package/index.js

@@ -0,0 +1,31 @@
+/**
+ * Source-to-sink streaming library.
+ *
+ * Provides components for building data streaming pipelines:
+ * - batch: Collects records and flushes to sink
+ * - circuit: Circuit breaker for failure isolation
+ * - timedBatch: Decorator for time-based flushing
+ * - clock: Time provider for circuit breaker
+ * - pollingSource: Generic polling source with time window
+ * - clickhouseSink: ClickHouse sink (accepts URL)
+ * - mqttSource: MQTT subscription source (accepts URL)
+ * - lokiSource: Loki polling source (accepts URL)
+ *
+ * @example
+ * import { batch, circuit, timedBatch, clock, clickhouseSink, mqttSource } from 'source-to-sink';
+ *
+ * const clk = clock();
+ * const c = circuit(5, 60, clk);
+ * const sink = clickhouseSink('http://localhost:8123', 'metrics');
+ * const collector = timedBatch(batch(sink, 1000, c), 5.0);
+ * const source = mqttSource('mqtt://localhost:1883', 'sensors/#', collector);
+ * source.start();
+ */
+export { default as batch } from './src/batch.js';
+export { default as circuit } from './src/circuit.js';
+export { default as clock } from './src/clock.js';
+export { default as timedBatch } from './src/timedBatch.js';
+export { default as pollingSource } from './src/pollingSource.js';
+export { default as clickhouseSink } from './src/clickhouseSink.js';
+export { default as mqttSource } from './src/mqttSource.js';
+export { default as lokiSource } from './src/lokiSource.js';

package/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@yarkivaev/source-to-sink",
+  "version": "1.0.0",
+  "description": "Generic library for building data streaming pipelines",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/yarkivaev/source-to-sink"
+  },
+  "type": "module",
+  "main": "index.js",
+  "dependencies": {
+    "mqtt": "^5.0.0",
+    "@clickhouse/client": "^1.0.0"
+  },
+  "devDependencies": {
+    "mocha": "^10.0.0",
+    "c8": "^8.0.0",
+    "testcontainers": "^10.13.0"
+  },
+  "scripts": {
+    "test": "mocha test/*.js",
+    "coverage": "c8 mocha test/*.js"
+  }
+}

package/src/batch.js

@@ -0,0 +1,75 @@
+/**
+ * Batch collector that accumulates records and flushes to a sink.
+ *
+ * Collects records until batch size is reached, then flushes to
+ * the configured sink. Uses circuit breaker for failure isolation.
+ * For time-based flushing, wrap with timedBatch.
+ *
+ * @example
+ * const sink = { write: (records) => console.log(records) };
+ * const clk = clock();
+ * const c = circuit(5, 60, clk);
+ * const b = batch(sink, 100, c);
+ * b.accept({ value: 42 });
+ * b.flush();
+ * b.stop();
+ *
+ * @param {object} sink - Object with write(records) method
+ * @param {number} size - Maximum batch size before automatic flush
+ * @param {object} circuit - Circuit breaker with allowing(), succeed(), fail()
+ * @returns {object} Batch collector with accept(), flush(), and stop() methods
+ */
+export default function batch(sink, size, circuit) {
+  if (!sink || typeof sink.write !== 'function') {
+    throw new Error('Sink must have a write(records) method');
+  }
+  if (typeof size !== 'number' || size < 1) {
+    throw new Error(`Size must be a positive number, got: ${size}`);
+  }
+  if (!circuit || typeof circuit.allowing !== 'function') {
+    throw new Error('Circuit must have an allowing() method');
+  }
+  let records = [];
+  const perform = () => {
+    if (records.length === 0) {
+      return;
+    }
+    if (!circuit.allowing()) {
+      return;
+    }
+    const pending = records;
+    try {
+      sink.write(pending);
+      records = [];
+      circuit.succeed();
+    } catch (err) {
+      circuit.fail();
+      throw err;
+    }
+  };
+  return {
+    /**
+     * Accepts a record into the batch.
+     *
+     * @param {*} record - Record to accept
+     */
+    accept(record) {
+      records.push(record);
+      if (records.length >= size) {
+        perform();
+      }
+    },
+    /**
+     * Forces an immediate flush of all pending records.
+     */
+    flush() {
+      perform();
+    },
+    /**
+     * Stops the batch collector and clears pending records.
+     */
+    stop() {
+      records = [];
+    }
+  };
+}

package/src/circuit.js

@@ -0,0 +1,102 @@
+/**
+ * Closed circuit state allowing operations.
+ *
+ * @returns {object} State with allowing() returning true
+ */
+function closed() {
+  return {
+    allowing() {
+      return true;
+    }
+  };
+}
+
+/**
+ * Open circuit state blocking operations.
+ *
+ * @param {number} timestamp - When the circuit was opened
+ * @param {object} clk - Clock for time tracking
+ * @param {number} timeout - Seconds before expiration
+ * @returns {object} State with allowing() and expired()
+ */
+function open(timestamp, clk, timeout) {
+  return {
+    allowing() {
+      return false;
+    },
+    expired() {
+      return (clk.millis() - timestamp) / 1000 >= timeout;
+    }
+  };
+}
+
+/**
+ * Circuit breaker for failure isolation in data pipelines.
+ *
+ * Implements the circuit breaker pattern to prevent cascading failures.
+ * The circuit opens after a threshold of failures and closes after a timeout.
+ *
+ * @example
+ * const clk = clock();
+ * const c = circuit(5, 60, clk);
+ * if (c.allowing()) {
+ *   try {
+ *     await riskyOperation();
+ *     c.succeed();
+ *   } catch (err) {
+ *     c.fail();
+ *   }
+ * }
+ *
+ * @param {number} threshold - Number of failures before opening the circuit
+ * @param {number} timeout - Seconds to wait before attempting recovery
+ * @param {object} clk - Clock with millis() method for time tracking
+ * @returns {object} Circuit breaker with allowing(), succeed(), and fail() methods
+ */
+export default function circuit(threshold, timeout, clk) {
+  if (typeof threshold !== 'number' || threshold < 1) {
+    throw new Error(`Threshold must be a positive number, got: ${threshold}`);
+  }
+  if (typeof timeout !== 'number' || timeout < 0) {
+    throw new Error(`Timeout must be a non-negative number, got: ${timeout}`);
+  }
+  if (!clk || typeof clk.millis !== 'function') {
+    throw new Error('Clock must have a millis() method');
+  }
+  let failures = 0;
+  let state = closed();
+  return {
+    /**
+     * Checks if the circuit allows operations.
+     *
+     * @returns {boolean} True if circuit allows operations, false if open
+     */
+    allowing() {
+      if (state.allowing()) {
+        return true;
+      }
+      if (state.expired()) {
+        state = closed();
+        failures = 0;
+        return true;
+      }
+      return false;
+    },
+    /**
+     * Records a successful operation, resetting the failure count.
+     */
+    succeed() {
+      failures = 0;
+      state = closed();
+    },
+    /**
+     * Records a failed operation, potentially opening the circuit.
+     */
+    fail() {
+      failures += 1;
+      if (failures >= threshold) {
+        state = open(clk.millis(), clk, timeout);
+      }
+    }
+  };
+}

package/src/clickhouseSink.js

@@ -0,0 +1,36 @@
+import { createClient } from '@clickhouse/client';
+
+/**
+ * ClickHouse sink for batch record insertion.
+ *
+ * Creates a ClickHouse client internally and implements the Sink
+ * interface for use with batch collectors.
+ *
+ * @example
+ * const sink = clickhouseSink('http://localhost:8123', 'metrics');
+ * await sink.write([{ ts: Date.now(), value: 42 }]);
+ *
+ * @param {string} url - ClickHouse URL (e.g., 'http://localhost:8123')
+ * @param {string} table - Target table name
+ * @returns {object} Sink with write(records) method
+ */
+export default function clickhouseSink(url, table) {
+  if (typeof url !== 'string' || url.length === 0) {
+    throw new Error('URL must be a non-empty string');
+  }
+  if (typeof table !== 'string' || table.length === 0) {
+    throw new Error('Table must be a non-empty string');
+  }
+  const client = createClient({ url });
+  return {
+    /**
+     * Writes records to ClickHouse table.
+     *
+     * @param {Array} records - Array of records to insert
+     * @returns {Promise} Promise resolving when insert completes
+     */
+    write(records) {
+      return client.insert({ table, values: records, format: 'JSONEachRow' });
+    }
+  };
+}

package/src/clock.js

@@ -0,0 +1,24 @@
+/**
+ * Real clock implementation using system time.
+ *
+ * Provides current time in milliseconds for time-dependent operations.
+ * Use fakeClock in tests to control time deterministically.
+ *
+ * @example
+ * const clk = clock();
+ * const now = clk.millis();
+ *
+ * @returns {object} Clock with millis() method
+ */
+export default function clock() {
+  return {
+    /**
+     * Returns current time in milliseconds since epoch.
+     *
+     * @returns {number} Current time in milliseconds
+     */
+    millis() {
+      return Date.now();
+    }
+  };
+}

package/src/lokiSource.js

@@ -0,0 +1,49 @@
+import pollingSource from './pollingSource.js';
+
+/**
+ * Loki polling source for streaming log entries to a collector.
+ *
+ * Polls Loki at a specified interval and forwards log entries
+ * to the collector. Creates HTTP client internally.
+ *
+ * @example
+ * const source = lokiSource('http://localhost:3100', '{app="traefik"}', 10, collector, clock);
+ * source.start();
+ * // ... later
+ * source.stop();
+ *
+ * @param {string} url - Loki base URL (e.g., 'http://localhost:3100')
+ * @param {string} query - LogQL query string
+ * @param {number} interval - Polling interval in seconds
+ * @param {object} collector - Collector with accept() method
+ * @param {object} clk - Clock with millis() method
+ * @returns {object} Source with start() and stop() methods
+ */
+export default function lokiSource(url, query, interval, collector, clk) {
+  if (typeof url !== 'string' || url.length === 0) {
+    throw new Error('URL must be a non-empty string');
+  }
+  if (typeof query !== 'string' || query.length === 0) {
+    throw new Error('Query must be a non-empty string');
+  }
+  const fetch = async (since, until) => {
+    const params = new URLSearchParams({
+      query: query,
+      start: (since * 1000000).toString(),
+      end: (until * 1000000).toString(),
+      limit: '1000'
+    });
+    const response = await globalThis.fetch(`${url}/loki/api/v1/query_range?${params}`);
+    const data = await response.json();
+    const entries = [];
+    if (data.data && data.data.result) {
+      for (const stream of data.data.result) {
+        for (const [ts, line] of stream.values) {
+          entries.push({ ts: parseInt(ts, 10) / 1000000, line });
+        }
+      }
+    }
+    return entries;
+  };
+  return pollingSource(fetch, interval, collector, clk);
+}

package/src/mqttSource.js

@@ -0,0 +1,109 @@
+import mqtt from 'mqtt';
+
+/**
+ * Idle state for MQTT source.
+ *
+ * @returns {object} State with subscribed() returning false
+ */
+function idle() {
+  return {
+    subscribed() {
+      return false;
+    }
+  };
+}
+
+/**
+ * Subscribed state for MQTT source.
+ *
+ * @param {object} client - MQTT client
+ * @param {function} handler - Message handler function
+ * @returns {object} State with subscribed() returning true
+ */
+function subscribed(client, handler) {
+  return {
+    subscribed() {
+      return true;
+    },
+    client() {
+      return client;
+    },
+    handler() {
+      return handler;
+    }
+  };
+}
+
+/**
+ * MQTT subscription source for streaming messages to a collector.
+ *
+ * Subscribes to MQTT topics and forwards raw messages to the collector.
+ * Messages are passed as {topic, payload} objects without parsing.
+ * Creates MQTT client internally. Supports comma-separated topic patterns.
+ *
+ * @example
+ * const source = mqttSource('mqtt://localhost:1883', 'sensors/#,devices/#', collector);
+ * source.start();
+ * // ... later
+ * source.stop();
+ *
+ * @param {string} url - MQTT broker URL (e.g., 'mqtt://localhost:1883')
+ * @param {string} topics - Comma-separated MQTT topic patterns to subscribe
+ * @param {object} collector - Collector with accept() method receiving {topic, payload}
+ * @param {object} [options] - Optional MQTT connection options
+ * @param {string} [options.clientId] - Client ID for persistent sessions
+ * @param {number} [options.sessionExpiryInterval] - Session expiry in seconds (default 3600)
+ * @returns {object} Source with start() and stop() methods
+ */
+export default function mqttSource(url, topics, collector, options = {}) {
+  if (typeof url !== 'string' || url.length === 0) {
+    throw new Error('URL must be a non-empty string');
+  }
+  if (typeof topics !== 'string' || topics.length === 0) {
+    throw new Error('Topic must be a non-empty string');
+  }
+  if (!collector || typeof collector.accept !== 'function') {
+    throw new Error('Collector must have an accept() method');
+  }
+  const list = topics.split(',').map((t) => t.trim()).filter((t) => t.length > 0);
+  let state = idle();
+  return {
+    /**
+     * Starts subscribing to the MQTT topics.
+     */
+    start() {
+      if (state.subscribed()) {
+        return;
+      }
+      const client = mqtt.connect(url, {
+        clientId: options.clientId,
+        clean: options.clientId ? false : true,
+        protocolVersion: 5,
+        properties: options.clientId ? {
+          sessionExpiryInterval: options.sessionExpiryInterval || 3600
+        } : undefined
+      });
+      const handler = (t, message) => {
+        collector.accept({ topic: t, payload: message.toString() });
+      };
+      client.on('message', handler);
+      client.on('connect', () => {
+        client.subscribe(list, { qos: options.clientId ? 1 : 0 });
+      });
+      state = subscribed(client, handler);
+    },
+    /**
+     * Stops subscribing to the MQTT topics.
+     */
+    stop() {
+      if (!state.subscribed()) {
+        return;
+      }
+      const client = state.client();
+      client.unsubscribe(list);
+      client.off('message', state.handler());
+      client.end();
+      state = idle();
+    }
+  };
+}

package/src/pollingSource.js

@@ -0,0 +1,98 @@
+/**
+ * Idle state for polling source.
+ *
+ * @returns {object} State with polling() returning false
+ */
+function idle() {
+  return {
+    polling() {
+      return false;
+    }
+  };
+}
+
+/**
+ * Polling state for polling source.
+ *
+ * @param {object} handle - Timer handle from setInterval
+ * @returns {object} State with polling() returning true
+ */
+function polling(handle) {
+  return {
+    polling() {
+      return true;
+    },
+    cancel() {
+      clearInterval(handle);
+    }
+  };
+}
+
+/**
+ * Generic polling source with time window tracking.
+ *
+ * Polls at a specified interval and forwards records to
+ * the collector. Tracks time windows to avoid duplicates.
+ *
+ * @example
+ * const fetch = async (since, until) => {
+ *   return await api.query(since, until);
+ * };
+ * const source = pollingSource(fetch, 10, collector, clock);
+ * source.start();
+ * // ... later
+ * source.stop();
+ *
+ * @param {function} fetch - Async function(since, until) returning array
+ * @param {number} interval - Polling interval in seconds
+ * @param {object} collector - Collector with accept() method
+ * @param {object} clk - Clock with millis() method
+ * @returns {object} Source with start() and stop() methods
+ */
+export default function pollingSource(fetch, interval, collector, clk) {
+  if (typeof fetch !== 'function') {
+    throw new Error('Fetch must be a function');
+  }
+  if (typeof interval !== 'number' || interval <= 0) {
+    throw new Error(`Interval must be a positive number, got: ${interval}`);
+  }
+  if (!collector || typeof collector.accept !== 'function') {
+    throw new Error('Collector must have an accept() method');
+  }
+  if (!clk || typeof clk.millis !== 'function') {
+    throw new Error('Clock must have a millis() method');
+  }
+  let state = idle();
+  let since = clk.millis();
+  const poll = async () => {
+    const until = clk.millis();
+    const result = await fetch(since, until);
+    since = until;
+    for (const entry of result) {
+      collector.accept(entry);
+    }
+  };
+  return {
+    /**
+     * Starts polling.
+     */
+    start() {
+      if (state.polling()) {
+        return;
+      }
+      since = clk.millis();
+      const handle = setInterval(poll, interval * 1000);
+      state = polling(handle);
+    },
+    /**
+     * Stops polling.
+     */
+    stop() {
+      if (!state.polling()) {
+        return;
+      }
+      state.cancel();
+      state = idle();
+    }
+  };
+}