@yarkivaev/source-to-sink 1.0.2 → 1.0.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/source-to-sink",
-  "version": "1.0.2",
+  "version": "1.0.4",
   "description": "Generic library for building data streaming pipelines",
   "repository": {
     "type": "git",

package/src/postgresSink.js

@@ -5,7 +5,8 @@ import pg from 'pg';
  *
  * Creates a pg.Pool internally and implements the Sink
  * interface for use with batch collectors. Supports optional
- * conflict resolution via ON CONFLICT DO NOTHING.
+ * conflict resolution via ON CONFLICT DO NOTHING or
+ * ON CONFLICT DO UPDATE SET.
  *
  * @example
  * const sink = postgresSink('postgresql://localhost:5432/db', 'metrics', ['ts', 'value']);
@@ -20,9 +21,55 @@ import pg from 'pg';
  * @param {string} table - Target table name
  * @param {Array<string>} columns - Column names for insertion
  * @param {object} [options] - Optional configuration
- * @param {Array<string>} [options.conflict] - Columns for ON CONFLICT DO NOTHING clause
+ * @param {Array<string>} [options.conflict] - Columns for ON CONFLICT clause
+ * @param {Array<string>} [options.update] - Columns for DO UPDATE SET clause
  * @returns {object} Sink with write(records) method
  */
+/**
+ * Removes duplicate records by conflict key within a batch.
+ *
+ * PostgreSQL rejects INSERT batches where ON CONFLICT would update
+ * the same row twice. This keeps the last occurrence for each
+ * unique combination of conflict column values.
+ *
+ * @example
+ *   deduplicate(
+ *     [{ machine: 'a', start: 1, name: 'pending' },
+ *      { machine: 'a', start: 1, name: 'completed' }],
+ *     ['machine', 'start']
+ *   );
+ *   // => [{ machine: 'a', start: 1, name: 'completed' }]
+ *
+ * @param {Array} records - Array of record objects
+ * @param {Array<string>} keys - Conflict column names
+ * @returns {Array} Deduplicated records
+ */
+export function deduplicate(records, keys) {
+  if (keys.length === 0) return records;
+  const seen = new Map();
+  for (const record of records) {
+    const key = keys.map(k => record[k]).join('\0');
+    seen.set(key, record);
+  }
+  return Array.from(seen.values());
+}
+
+/**
+ * Builds the ON CONFLICT SQL suffix from options.
+ *
+ * @param {object} options - Sink options with conflict and update arrays
+ * @returns {string} SQL suffix or empty string
+ */
+function buildSuffix(options) {
+  if (!Array.isArray(options.conflict) || options.conflict.length === 0) return '';
+  const cols = options.conflict.join(', ');
+  if (Array.isArray(options.update) && options.update.length > 0) {
+    const sets = options.update.map(c => `${c} = EXCLUDED.${c}`).join(', ');
+    return ` ON CONFLICT (${cols}) DO UPDATE SET ${sets}`;
+  }
+  return ` ON CONFLICT (${cols}) DO NOTHING`;
+}
+
 export default function postgresSink(url, table, columns, options = {}) {
   if (typeof url !== 'string' || url.length === 0) {
     throw new Error('URL must be a non-empty string');
@@ -34,9 +81,8 @@ export default function postgresSink(url, table, columns, options = {}) {
     throw new Error('Columns must be a non-empty array');
   }
   const pool = new pg.Pool({ connectionString: url });
-  const suffix = Array.isArray(options.conflict) && options.conflict.length > 0
-    ? ` ON CONFLICT (${options.conflict.join(', ')}) DO NOTHING`
-    : '';
+  const suffix = buildSuffix(options);
+  const conflict = Array.isArray(options.conflict) ? options.conflict : [];
   return {
     /**
      * Writes records to PostgreSQL table.
@@ -45,13 +91,14 @@ export default function postgresSink(url, table, columns, options = {}) {
      * @returns {Promise} Promise resolving when insert completes
      */
     write(records) {
-      const placeholders = records.map((_, i) => {
+      const unique = deduplicate(records, conflict);
+      const placeholders = unique.map((_, i) => {
         const offset = i * columns.length;
         const row = columns.map((__, j) => `$${offset + j + 1}`).join(', ');
         return `(${row})`;
       }).join(', ');
       const query = `INSERT INTO ${table} (${columns.join(', ')}) VALUES ${placeholders}${suffix}`;
-      const values = records.flatMap((record) => columns.map((col) => record[col]));
+      const values = unique.flatMap((record) => columns.map((col) => record[col]));
       return pool.query(query, values);
     }
   };

package/test/test_postgresSink.js

@@ -1,6 +1,6 @@
 import assert from 'node:assert';
 import { describe, it } from 'mocha';
-import postgresSink from '../src/postgresSink.js';
+import postgresSink, { deduplicate } from '../src/postgresSink.js';
 
 describe('postgresSink', () => {
   it('throws on missing url', () => {
@@ -73,4 +73,39 @@ describe('postgresSink', () => {
       ['ts', 'value'], {});
     assert.strictEqual(typeof sink.write, 'function', 'Should have write method without conflict key');
   });
+
+  it('returns sink with write method when conflict and update options are provided', () => {
+    const sink = postgresSink('postgresql://localhost:5432/db', 'segments',
+      ['machine', 'name', 'start_time', 'end_time', 'duration'],
+      { conflict: ['machine', 'start_time'], update: ['name', 'end_time', 'duration'] });
+    assert.strictEqual(typeof sink.write, 'function', 'Should have write method with update option');
+  });
+
+  it('deduplicates batch by conflict columns keeping last occurrence', () => {
+    const records = [
+      { machine: 'ü-1', name: 'pending', start_time: '2024-01-01' },
+      { machine: 'ü-1', name: 'completed', start_time: '2024-01-01' },
+      { machine: 'ö-2', name: 'pending', start_time: '2024-01-02' }
+    ];
+    const result = deduplicate(records, ['machine', 'start_time']);
+    assert.strictEqual(result.length, 2, 'Should remove duplicate conflict key');
+  });
+
+  it('keeps last record when conflict columns match', () => {
+    const records = [
+      { machine: 'ü-1', name: 'pending', start_time: '2024-01-01' },
+      { machine: 'ü-1', name: 'completed', start_time: '2024-01-01' }
+    ];
+    const result = deduplicate(records, ['machine', 'start_time']);
+    assert.strictEqual(result[0].name, 'completed', 'Should keep the last occurrence');
+  });
+
+  it('returns records unchanged when no conflict columns', () => {
+    const records = [
+      { machine: 'ä-1', name: 'à', start_time: '2024-01-01' },
+      { machine: 'ä-1', name: 'à', start_time: '2024-01-01' }
+    ];
+    const result = deduplicate(records, []);
+    assert.strictEqual(result.length, 2, 'Should not deduplicate without conflict columns');
+  });
 });