@yarkivaev/source-to-sink 1.3.4 → 1.3.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/source-to-sink",
-  "version": "1.3.4",
+  "version": "1.3.6",
   "description": "Generic library for building data streaming pipelines",
   "repository": {
     "type": "git",

package/src/follow.js

@@ -0,0 +1,41 @@
+/**
+ * Continues a delivery chain after a rejected prior step.
+ *
+ * @param {Promise} pending - prior chain
+ * @returns {Promise} settled chain
+ */
+function resume(pending) {
+    return pending.then((value) => {
+        return value;
+    }, (error) => {
+        return error;
+    });
+}
+
+/**
+ * Chains one arrival: read the body, then start deliver.
+ * Serial waits for deliver. Otherwise persist may overlap after enqueue.
+ *
+ * @param {Promise} pending - prior arrival
+ * @param {Function} read - returns a promise for the body
+ * @param {Function} deliver - starts persist for one body
+ * @param {boolean} serial - wait for persist before the next arrival
+ * @returns {Promise} chain
+ *
+ * @example
+ *   let pending = Promise.resolve();
+ *   pending = follow(pending, () => Promise.resolve('a'), write, false);
+ */
+export default function follow(pending, read, deliver, serial) {
+    return resume(pending).then(async () => {
+        const body = await read();
+        const run = deliver(body);
+        if (serial) {
+            return run;
+        }
+        void run.catch((error) => {
+            return error;
+        });
+        return undefined;
+    });
+}

package/src/postgresSink.js

@@ -32,7 +32,7 @@ export function deduplicate(records, keys) {
 /**
  * Builds the ON CONFLICT SQL suffix from options.
  *
- * @param {object} options - Sink options with conflict and update arrays
+ * @param {object} options - Sink options with conflict, update, and optional updateWhere
  * @returns {string} SQL suffix or empty string
  */
 function buildSuffix(options) {
@@ -40,7 +40,10 @@ function buildSuffix(options) {
   const cols = options.conflict.join(', ');
   if (Array.isArray(options.update) && options.update.length > 0) {
     const sets = options.update.map((col) => {return `${col} = EXCLUDED.${col}`}).join(', ');
-    return ` ON CONFLICT (${cols}) DO UPDATE SET ${sets}`;
+    const guard = typeof options.updateWhere === 'string' && options.updateWhere.length > 0
+      ? ` WHERE ${options.updateWhere}`
+      : '';
+    return ` ON CONFLICT (${cols}) DO UPDATE SET ${sets}${guard}`;
   }
   return ` ON CONFLICT (${cols}) DO NOTHING`;
 }
@@ -67,6 +70,7 @@ function buildSuffix(options) {
  * @param {object} [options] - Optional configuration
  * @param {Array<string>} [options.conflict] - Columns for ON CONFLICT clause
  * @param {Array<string>} [options.update] - Columns for DO UPDATE SET clause
+ * @param {string} [options.updateWhere] - Optional WHERE guard on DO UPDATE
  * @param {object} [options.pool] - Existing pg Pool to reuse
  * @returns {object} Sink with write(records, client?) method
  */

package/src/stompSend.js

@@ -7,6 +7,20 @@ import stompit from 'stompit';
  * @param {object} options - Connection options
  * @returns {object} stompit connect parameters
  */
+/**
+ * STOMP SEND headers that persist the body on a durable RabbitMQ queue.
+ *
+ * @param {string} destination - STOMP destination
+ * @returns {object} headers including persistent true
+ */
+export function stompHeaders(destination) {
+    return {
+        destination,
+        'content-type': 'application/json',
+        persistent: 'true'
+    };
+}
+
 function connectParams(parsed, options) {
     return {
         host: parsed.hostname,
@@ -55,10 +69,7 @@ export default function stompSend(url, destination, payload, options = {}) {
                 reject(error);
                 return;
             }
-            const frame = client.send({
-                destination,
-                'content-type': 'application/json'
-            });
+            const frame = client.send(stompHeaders(destination));
             frame.end(body);
             client.disconnect((disconnectError) => {
                 if (disconnectError) {

package/src/stompSource.js

@@ -1,4 +1,5 @@
 import stompit from 'stompit';
+import follow from './follow.js';
 
 /**
  * Idle state for STOMP source.
@@ -90,41 +91,46 @@ async function deliver(ctx, body) {
 }
 
 /**
- * Ignores a rejected delivery promise when running non-serial.
+ * Reads one STOMP frame body as text.
  *
- * @param {Error} error - rejected reason
- * @returns {undefined}
+ * @param {object} message - STOMP frame
+ * @returns {Promise<string>} UTF-8 body
  */
-function ignoreDelivery(error) {
-  return error;
+function read(message) {
+  return new Promise((resolve, reject) => {
+    message.readString('utf-8', (error, body) => {
+      if (error) {
+        reject(error);
+        return;
+      }
+      resolve(body.toString());
+    });
+  });
 }
 
 /**
  * Subscribes a channel and routes frames to the collector.
+ * Arrivals stay in subscribe order through follow, even when persist overlaps.
  *
  * @param {object} channel - STOMP channel
  * @param {string} destination - queue destination
  * @param {object} collector - message collector
- * @param {boolean} serial - global serial delivery
+ * @param {boolean} serial - wait for persist before the next arrival
  * @param {boolean} manualAck - collector settles frames
  * @returns {undefined}
  */
 function bind(channel, destination, collector, serial, manualAck) {
   let pending = Promise.resolve();
   channel.subscribe({ destination, ack: 'client-individual' }, (err, message) => {
-    if (err) { return; }
-    message.readString('utf-8', (readErr, body) => {
-      if (readErr) { return; }
-      const ctx = { channel, message, destination, collector, manualAck };
-      function work() {
-        return deliver(ctx, body.toString());
-      }
-      if (serial) {
-        pending = pending.then(work, work);
-      } else {
-        void work().catch(ignoreDelivery);
-      }
-    });
+    if (err) {
+      return;
+    }
+    const ctx = { channel, message, destination, collector, manualAck };
+    pending = follow(pending, () => {
+      return read(message);
+    }, (body) => {
+      return deliver(ctx, body);
+    }, serial);
   });
 }
 
@@ -132,8 +138,8 @@ function bind(channel, destination, collector, serial, manualAck) {
  * STOMP subscription source for streaming messages to a collector.
  *
  * Messages are `{destination, payload, settle}`. Default ack-after-accept;
- * with `manualAck` the collector owns settle. With `serial: false`
- * deliveries are not globally sequenced.
+ * with `manualAck` the collector owns settle. Arrivals enqueue in
+ * subscribe order. With `serial: false` persist may overlap after enqueue.
  *
  * @param {string} url - STOMP broker URL
  * @param {string} destination - STOMP destination

package/test/test_follow.js

@@ -0,0 +1,35 @@
+import assert from 'node:assert';
+import { describe, it } from 'mocha';
+import follow from '../src/follow.js';
+
+describe('follow', () => {
+    it('accepts in subscribe order when the first body is slower', async () => {
+        const order = [];
+        const first = `ä-${Math.random().toString(16).slice(2)}`;
+        const second = `ö-${Math.random().toString(16).slice(2)}`;
+        const pause = (ms) => {
+            return new Promise((resolve) => {
+                setTimeout(resolve, ms);
+            });
+        };
+        const deliver = (body) => {
+            order.push(body);
+            return Promise.resolve();
+        };
+        let pending = Promise.resolve();
+        pending = follow(pending, () => {
+            return pause(40).then(() => {
+                return first;
+            });
+        }, deliver, false);
+        pending = follow(pending, () => {
+            return Promise.resolve(second);
+        }, deliver, false);
+        await pending;
+        assert.deepStrictEqual(
+            order,
+            [first, second],
+            'a slower first body reordered accept ahead of subscribe'
+        );
+    });
+});

package/test/test_postgresSink.js

@@ -122,6 +122,28 @@ describe('postgresSink', () => {
     assert.strictEqual(queries.length, 1, 'injected pool was not used for write');
   });
 
+  it('appends updateWhere to the conflict update', async () => {
+    const queries = [];
+    const pool = {
+      query(sql) {
+        queries.push(sql);
+        return Promise.resolve({ rowCount: 1 });
+      }
+    };
+    const sink = postgresSink('postgresql://localhost:5432/db', 'segments',
+      ['machine', 'duration'], {
+        pool,
+        conflict: ['machine'],
+        update: ['duration'],
+        updateWhere: 'segments.duration = 0 OR EXCLUDED.duration > 0'
+      });
+    await sink.write([{ machine: `ä-${Math.random()}`, duration: 0 }]);
+    assert.ok(
+      queries[0].includes('WHERE segments.duration = 0 OR EXCLUDED.duration > 0'),
+      'conflict update did not keep the monotonic duration guard'
+    );
+  });
+
   it('routes write through an explicit client when provided', async () => {
     const pool = { query() { return Promise.reject(new Error('pool must not be used')); } };
     const clientQueries = [];

package/test/test_stompSend.js

@@ -1,6 +1,6 @@
 import assert from 'node:assert';
 import { describe, it } from 'mocha';
-import stompSend from '../src/stompSend.js';
+import stompSend, { stompHeaders } from '../src/stompSend.js';
 
 describe('stompSend', () => {
     it('throws on missing url', () => {
@@ -23,6 +23,15 @@ describe('stompSend', () => {
         );
     });
 
+    it('marks SEND frames persistent', () => {
+        const destination = `/exchange/\u00e9${Math.random().toString(36).slice(2)}`;
+        assert.strictEqual(
+            stompHeaders(destination).persistent,
+            'true',
+            'stompSend did not mark the frame persistent'
+        );
+    });
+
     it('returns a promise', () => {
         const pending = stompSend(
             `stomp://\u00e9${Math.random()}:61613`,