@yarkivaev/source-to-sink 1.0.3 → 1.0.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/source-to-sink",
-  "version": "1.0.3",
+  "version": "1.0.4",
   "description": "Generic library for building data streaming pipelines",
   "repository": {
     "type": "git",

package/src/postgresSink.js

@@ -25,6 +25,35 @@ import pg from 'pg';
  * @param {Array<string>} [options.update] - Columns for DO UPDATE SET clause
  * @returns {object} Sink with write(records) method
  */
+/**
+ * Removes duplicate records by conflict key within a batch.
+ *
+ * PostgreSQL rejects INSERT batches where ON CONFLICT would update
+ * the same row twice. This keeps the last occurrence for each
+ * unique combination of conflict column values.
+ *
+ * @example
+ *   deduplicate(
+ *     [{ machine: 'a', start: 1, name: 'pending' },
+ *      { machine: 'a', start: 1, name: 'completed' }],
+ *     ['machine', 'start']
+ *   );
+ *   // => [{ machine: 'a', start: 1, name: 'completed' }]
+ *
+ * @param {Array} records - Array of record objects
+ * @param {Array<string>} keys - Conflict column names
+ * @returns {Array} Deduplicated records
+ */
+export function deduplicate(records, keys) {
+  if (keys.length === 0) return records;
+  const seen = new Map();
+  for (const record of records) {
+    const key = keys.map(k => record[k]).join('\0');
+    seen.set(key, record);
+  }
+  return Array.from(seen.values());
+}
+
 /**
  * Builds the ON CONFLICT SQL suffix from options.
  *
@@ -53,6 +82,7 @@ export default function postgresSink(url, table, columns, options = {}) {
   }
   const pool = new pg.Pool({ connectionString: url });
   const suffix = buildSuffix(options);
+  const conflict = Array.isArray(options.conflict) ? options.conflict : [];
   return {
     /**
      * Writes records to PostgreSQL table.
@@ -61,13 +91,14 @@ export default function postgresSink(url, table, columns, options = {}) {
      * @returns {Promise} Promise resolving when insert completes
      */
     write(records) {
-      const placeholders = records.map((_, i) => {
+      const unique = deduplicate(records, conflict);
+      const placeholders = unique.map((_, i) => {
         const offset = i * columns.length;
         const row = columns.map((__, j) => `$${offset + j + 1}`).join(', ');
         return `(${row})`;
       }).join(', ');
       const query = `INSERT INTO ${table} (${columns.join(', ')}) VALUES ${placeholders}${suffix}`;
-      const values = records.flatMap((record) => columns.map((col) => record[col]));
+      const values = unique.flatMap((record) => columns.map((col) => record[col]));
       return pool.query(query, values);
     }
   };

package/test/test_postgresSink.js

@@ -1,6 +1,6 @@
 import assert from 'node:assert';
 import { describe, it } from 'mocha';
-import postgresSink from '../src/postgresSink.js';
+import postgresSink, { deduplicate } from '../src/postgresSink.js';
 
 describe('postgresSink', () => {
   it('throws on missing url', () => {
@@ -80,4 +80,32 @@ describe('postgresSink', () => {
       { conflict: ['machine', 'start_time'], update: ['name', 'end_time', 'duration'] });
     assert.strictEqual(typeof sink.write, 'function', 'Should have write method with update option');
   });
+
+  it('deduplicates batch by conflict columns keeping last occurrence', () => {
+    const records = [
+      { machine: 'ü-1', name: 'pending', start_time: '2024-01-01' },
+      { machine: 'ü-1', name: 'completed', start_time: '2024-01-01' },
+      { machine: 'ö-2', name: 'pending', start_time: '2024-01-02' }
+    ];
+    const result = deduplicate(records, ['machine', 'start_time']);
+    assert.strictEqual(result.length, 2, 'Should remove duplicate conflict key');
+  });
+
+  it('keeps last record when conflict columns match', () => {
+    const records = [
+      { machine: 'ü-1', name: 'pending', start_time: '2024-01-01' },
+      { machine: 'ü-1', name: 'completed', start_time: '2024-01-01' }
+    ];
+    const result = deduplicate(records, ['machine', 'start_time']);
+    assert.strictEqual(result[0].name, 'completed', 'Should keep the last occurrence');
+  });
+
+  it('returns records unchanged when no conflict columns', () => {
+    const records = [
+      { machine: 'ä-1', name: 'à', start_time: '2024-01-01' },
+      { machine: 'ä-1', name: 'à', start_time: '2024-01-01' }
+    ];
+    const result = deduplicate(records, []);
+    assert.strictEqual(result.length, 2, 'Should not deduplicate without conflict columns');
+  });
 });