@yarkivaev/source-to-sink 1.3.3 → 1.3.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/source-to-sink",
-  "version": "1.3.3",
+  "version": "1.3.5",
   "description": "Generic library for building data streaming pipelines",
   "repository": {
     "type": "git",

package/src/postgresSink.js

@@ -1,30 +1,5 @@
 import pg from 'pg';
 
-/**
- * PostgreSQL sink for batch record insertion.
- *
- * Creates a pg.Pool internally and implements the Sink
- * interface for use with batch collectors. Supports optional
- * conflict resolution via ON CONFLICT DO NOTHING or
- * ON CONFLICT DO UPDATE SET.
- *
- * @example
- * const sink = postgresSink('postgresql://localhost:5432/db', 'metrics', ['ts', 'value']);
- * await sink.write([{ ts: Date.now(), value: 42 }]);
- *
- * @example
- * const sink = postgresSink('postgresql://localhost:5432/db', 'segments',
- *   ['machine', 'name', 'start_time'], { conflict: ['machine', 'start_time'] });
- * await sink.write([{ machine: 'a', name: 'on', start_time: '2024-01-01' }]);
- *
- * @param {string} url - PostgreSQL URL (e.g., 'postgresql://localhost:5432/db')
- * @param {string} table - Target table name
- * @param {Array<string>} columns - Column names for insertion
- * @param {object} [options] - Optional configuration
- * @param {Array<string>} [options.conflict] - Columns for ON CONFLICT clause
- * @param {Array<string>} [options.update] - Columns for DO UPDATE SET clause
- * @returns {object} Sink with write(records) method
- */
 /**
  * Removes duplicate records by conflict key within a batch.
  *
@@ -70,6 +45,31 @@ function buildSuffix(options) {
   return ` ON CONFLICT (${cols}) DO NOTHING`;
 }
 
+/**
+ * PostgreSQL sink for batch record insertion.
+ *
+ * Creates a pg.Pool internally unless `options.pool` is supplied.
+ * Supports optional conflict resolution via ON CONFLICT DO NOTHING or
+ * ON CONFLICT DO UPDATE SET. `write` accepts an optional queryable
+ * client so callers can run inserts inside an existing transaction.
+ *
+ * @example
+ * const sink = postgresSink('postgresql://localhost:5432/db', 'metrics', ['ts', 'value']);
+ * await sink.write([{ ts: Date.now(), value: 42 }]);
+ *
+ * @example
+ * const sink = postgresSink(url, 'segments', cols, { pool, conflict: ['machine', 'start_time'] });
+ * await sink.write([{ machine: 'a', name: 'on', start_time: '2024-01-01' }], client);
+ *
+ * @param {string} url - PostgreSQL URL (e.g., 'postgresql://localhost:5432/db')
+ * @param {string} table - Target table name
+ * @param {Array<string>} columns - Column names for insertion
+ * @param {object} [options] - Optional configuration
+ * @param {Array<string>} [options.conflict] - Columns for ON CONFLICT clause
+ * @param {Array<string>} [options.update] - Columns for DO UPDATE SET clause
+ * @param {object} [options.pool] - Existing pg Pool to reuse
+ * @returns {object} Sink with write(records, client?) method
+ */
 export default function postgresSink(url, table, columns, options = {}) {
   if (typeof url !== 'string' || url.length === 0) {
     throw new Error('URL must be a non-empty string');
@@ -80,7 +80,7 @@ export default function postgresSink(url, table, columns, options = {}) {
   if (!Array.isArray(columns) || columns.length === 0) {
     throw new Error('Columns must be a non-empty array');
   }
-  const pool = new pg.Pool({ connectionString: url });
+  const pool = options.pool || new pg.Pool({ connectionString: url });
   const suffix = buildSuffix(options);
   const conflict = Array.isArray(options.conflict) ? options.conflict : [];
   return {
@@ -88,9 +88,10 @@ export default function postgresSink(url, table, columns, options = {}) {
      * Writes records to PostgreSQL table.
      *
      * @param {Array} records - Array of objects with keys matching columns
+     * @param {object} [client] - Optional pg client/pool with query()
      * @returns {Promise} Promise resolving when insert completes
      */
-    write(records) {
+    write(records, client) {
       const unique = deduplicate(records, conflict);
       const placeholders = unique.map((record, i) => {
         const offset = i * columns.length;
@@ -99,7 +100,8 @@ export default function postgresSink(url, table, columns, options = {}) {
       }).join(', ');
       const query = `INSERT INTO ${table} (${columns.join(', ')}) VALUES ${placeholders}${suffix}`;
       const values = unique.flatMap((record) => {return columns.map((col) => {return record[col]})});
-      return pool.query(query, values);
+      const runner = client && typeof client.query === 'function' ? client : pool;
+      return runner.query(query, values);
     }
   };
 }

package/src/stompSend.js

@@ -7,6 +7,20 @@ import stompit from 'stompit';
  * @param {object} options - Connection options
  * @returns {object} stompit connect parameters
  */
+/**
+ * STOMP SEND headers that persist the body on a durable RabbitMQ queue.
+ *
+ * @param {string} destination - STOMP destination
+ * @returns {object} headers including persistent true
+ */
+export function stompHeaders(destination) {
+    return {
+        destination,
+        'content-type': 'application/json',
+        persistent: 'true'
+    };
+}
+
 function connectParams(parsed, options) {
     return {
         host: parsed.hostname,
@@ -55,10 +69,7 @@ export default function stompSend(url, destination, payload, options = {}) {
                 reject(error);
                 return;
             }
-            const frame = client.send({
-                destination,
-                'content-type': 'application/json'
-            });
+            const frame = client.send(stompHeaders(destination));
             frame.end(body);
             client.disconnect((disconnectError) => {
                 if (disconnectError) {

package/src/stompSource.js

@@ -30,30 +30,6 @@ function subscribed(channel) {
   };
 }
 
-/**
- * STOMP subscription source for streaming messages to a collector.
- *
- * Subscribes to a STOMP destination and forwards raw messages to the collector.
- * Messages are passed as {destination, payload} objects without parsing.
- * Uses ConnectFailover for automatic reconnection with exponential backoff
- * and Channel for transparent re-subscription. Acknowledges messages only
- * after the collector has accepted them.
- *
- * @example
- * const source = stompSource('stomp://localhost:61613', '/queue/sensors', collector);
- * source.start();
- * // ... later
- * source.stop();
- *
- * @param {string} url - STOMP broker URL (e.g., 'stomp://localhost:61613')
- * @param {string} destination - STOMP destination to subscribe (e.g., '/queue/sensors')
- * @param {object} collector - Collector with accept() method receiving {destination, payload}
- * @param {object} [options] - Optional STOMP connection options
- * @param {string} [options.login] - Login for STOMP auth
- * @param {string} [options.passcode] - Passcode for STOMP auth
- * @param {string} [options.host] - Virtual host for STOMP connection
- * @returns {object} Source with start() and stop() methods
- */
 /**
  * Builds STOMP server configuration from a URL.
  *
@@ -73,6 +49,98 @@ function serverConfig(parsed, options) {
   };
 }
 
+/**
+ * Builds settle helpers for one STOMP frame.
+ *
+ * @param {object} channel - STOMP channel
+ * @param {object} message - STOMP frame
+ * @returns {{ ack: Function, nack: Function }}
+ */
+function settleOf(channel, message) {
+  return {
+    ack() {
+      channel.ack(message);
+    },
+    nack() {
+      channel.nack(message);
+    }
+  };
+}
+
+/**
+ * Delivers one STOMP body to the collector and settles the frame.
+ *
+ * @param {object} ctx - delivery context with channel/message/destination/collector/manualAck
+ * @param {string} body - UTF-8 payload
+ * @returns {Promise<void>}
+ */
+async function deliver(ctx, body) {
+  const settle = settleOf(ctx.channel, ctx.message);
+  try {
+    await ctx.collector.accept({ destination: ctx.destination, payload: body, settle });
+    if (!ctx.manualAck) {
+      settle.ack();
+    }
+  } catch (error) {
+    if (!ctx.manualAck) {
+      settle.nack();
+    }
+    throw error;
+  }
+}
+
+/**
+ * Ignores a rejected delivery promise when running non-serial.
+ *
+ * @param {Error} error - rejected reason
+ * @returns {undefined}
+ */
+function ignoreDelivery(error) {
+  return error;
+}
+
+/**
+ * Subscribes a channel and routes frames to the collector.
+ *
+ * @param {object} channel - STOMP channel
+ * @param {string} destination - queue destination
+ * @param {object} collector - message collector
+ * @param {boolean} serial - global serial delivery
+ * @param {boolean} manualAck - collector settles frames
+ * @returns {undefined}
+ */
+function bind(channel, destination, collector, serial, manualAck) {
+  let pending = Promise.resolve();
+  channel.subscribe({ destination, ack: 'client-individual' }, (err, message) => {
+    if (err) { return; }
+    message.readString('utf-8', (readErr, body) => {
+      if (readErr) { return; }
+      const ctx = { channel, message, destination, collector, manualAck };
+      function work() {
+        return deliver(ctx, body.toString());
+      }
+      if (serial) {
+        pending = pending.then(work, work);
+      } else {
+        void work().catch(ignoreDelivery);
+      }
+    });
+  });
+}
+
+/**
+ * STOMP subscription source for streaming messages to a collector.
+ *
+ * Messages are `{destination, payload, settle}`. Default ack-after-accept;
+ * with `manualAck` the collector owns settle. With `serial: false`
+ * deliveries are not globally sequenced.
+ *
+ * @param {string} url - STOMP broker URL
+ * @param {string} destination - STOMP destination
+ * @param {object} collector - Collector with accept()
+ * @param {object} [options] - Connection and delivery options
+ * @returns {object} Source with start() and stop() methods
+ */
 export default function stompSource(url, destination, collector, options = {}) {
   if (typeof url !== 'string' || url.length === 0) {
     throw new Error('URL must be a non-empty string');
@@ -84,11 +152,10 @@ export default function stompSource(url, destination, collector, options = {}) {
     throw new Error('Collector must have an accept() method');
   }
   const parsed = new URL(url);
+  const serial = options.serial !== false;
+  const manualAck = options.manualAck === true;
   let state = idle();
   return {
-    /**
-     * Starts subscribing to the STOMP destination.
-     */
     start() {
       if (state.subscribed()) {
         return;
@@ -99,25 +166,9 @@ export default function stompSource(url, destination, collector, options = {}) {
         maxReconnects: -1
       });
       const channel = new stompit.Channel(failover);
-      let pending = Promise.resolve();
-      const subscribe = { destination, ack: 'client-individual' };
-      channel.subscribe(subscribe, (err, message) => {
-        if (err) { return; }
-        message.readString('utf-8', (readErr, body) => {
-          if (readErr) { return; }
-          pending = pending.then(async () => {
-            try {
-              await collector.accept({ destination, payload: body.toString() });
-              channel.ack(message);
-            } catch { channel.nack(message); }
-          });
-        });
-      });
+      bind(channel, destination, collector, serial, manualAck);
       state = subscribed(channel);
     },
-    /**
-     * Stops subscribing to the STOMP destination.
-     */
     stop() {
       if (!state.subscribed()) {
         return;

package/test/test_postgresSink.js

@@ -108,4 +108,31 @@ describe('postgresSink', () => {
     const result = deduplicate(records, []);
     assert.strictEqual(result.length, 2, 'Should not deduplicate without conflict columns');
   });
+
+  it('reuses an injected pool when writing', async () => {
+    const queries = [];
+    const pool = {
+      query(sql, values) {
+        queries.push({ sql, values });
+        return Promise.resolve({ rowCount: 1 });
+      }
+    };
+    const sink = postgresSink('postgresql://localhost:5432/db', 'metrics', ['ts', 'value'], { pool });
+    await sink.write([{ ts: 1700000000, value: 42 }]);
+    assert.strictEqual(queries.length, 1, 'injected pool was not used for write');
+  });
+
+  it('routes write through an explicit client when provided', async () => {
+    const pool = { query() { return Promise.reject(new Error('pool must not be used')); } };
+    const clientQueries = [];
+    const client = {
+      query(sql, values) {
+        clientQueries.push({ sql, values });
+        return Promise.resolve({ rowCount: 1 });
+      }
+    };
+    const sink = postgresSink('postgresql://localhost:5432/db', 'metrics', ['ts', 'value'], { pool });
+    await sink.write([{ ts: 1700000001, value: 7 }], client);
+    assert.strictEqual(clientQueries.length, 1, 'explicit client was not used for write');
+  });
 });

package/test/test_stompSend.js

@@ -1,6 +1,6 @@
 import assert from 'node:assert';
 import { describe, it } from 'mocha';
-import stompSend from '../src/stompSend.js';
+import stompSend, { stompHeaders } from '../src/stompSend.js';
 
 describe('stompSend', () => {
     it('throws on missing url', () => {
@@ -23,6 +23,15 @@ describe('stompSend', () => {
         );
     });
 
+    it('marks SEND frames persistent', () => {
+        const destination = `/exchange/\u00e9${Math.random().toString(36).slice(2)}`;
+        assert.strictEqual(
+            stompHeaders(destination).persistent,
+            'true',
+            'stompSend did not mark the frame persistent'
+        );
+    });
+
     it('returns a promise', () => {
         const pending = stompSend(
             `stomp://\u00e9${Math.random()}:61613`,

package/test/test_stompSource.js

@@ -51,4 +51,13 @@ describe('stompSource', () => {
     assert.strictEqual(typeof source.start, 'function', 'Should have start method');
     assert.strictEqual(typeof source.stop, 'function', 'Should have stop method');
   });
+
+  it('accepts manualAck and serial options without throwing', () => {
+    const collector = { accept: () => {} };
+    const source = stompSource('stomp://localhost:61613', '/queue/test', collector, {
+      manualAck: true,
+      serial: false
+    });
+    assert.strictEqual(typeof source.start, 'function', 'manualAck source missing start');
+  });
 });