ffc-pay-etl-framework 1.0.0 → 1.1.0-alpha.10

package/dist/cjs/destinations/postgresDestination.js

@@ -1,43 +1,52 @@
 const EventEmitter = require('node:events');
 const { Transform } = require('node:stream');
-const { Sequelize } = require('sequelize');
 const debug = require('debug')('destination');
 const DEFAULT_PORT = 5432;
 function isKeyWord(column) {
     return ['USER'].includes(column.toUpperCase());
 }
 function getMappingForColumn(mapping, column) {
+    if (mapping.length === 0) {
+        return;
+    }
     const [map] = mapping.filter(m => m.column === column);
     return map;
 }
+function hasReturningColumns(mapping) {
+    return mapping.filter(m => m.returning === true).length > 0;
+}
+function getReturningColumns(mapping) {
+    return mapping.filter(m => m.returning === true).map(m => m === null || m === void 0 ? void 0 : m.targetColumn);
+}
 function writeInsertStatement(columnMapping, table, chunk) {
     let statement = `INSERT INTO ${table} (${chunk._columns.map(column => {
         const mapping = getMappingForColumn(columnMapping, column);
-        return mapping.targetColumn
+        return (mapping === null || mapping === void 0 ? void 0 : mapping.targetColumn)
             ?
                 isKeyWord(mapping.targetColumn)
                     ? `"${mapping.targetColumn}"`
                     : mapping.targetColumn
             : isKeyWord(column)
-                ? `"${mapping.column}"`
-                : mapping.column;
+                ? `"${mapping === null || mapping === void 0 ? void 0 : mapping.column}"`
+                : mapping === null || mapping === void 0 ? void 0 : mapping.column;
     })
         .join(",")}) VALUES (${chunk._columns.map((column, index) => {
         const mapping = getMappingForColumn(columnMapping, column);
-        if (!mapping)
-            debug('Mapping not found for column %s', column);
-        if (mapping.targetType === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
+        if ((mapping === null || mapping === void 0 ? void 0 : mapping.targetType) === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
             debug('Source data is not a number');
             return 0;
         }
-        if (mapping.targetType === "varchar" || mapping.targetType === "char") {
+        if ((mapping === null || mapping === void 0 ? void 0 : mapping.targetType) === "varchar" || (mapping === null || mapping === void 0 ? void 0 : mapping.targetType) === "char") {
             return `'${chunk[index]}'`;
         }
-        if (mapping.targetType === "date") {
-            return `to_date('${chunk[index]}','${mapping.format}')`;
+        if ((mapping === null || mapping === void 0 ? void 0 : mapping.targetType) === "date") {
+            return `to_date('${chunk[index]}','${mapping === null || mapping === void 0 ? void 0 : mapping.format}')`;
         }
         return chunk[index] ? chunk[index] : 'null';
     })})`;
+    if (hasReturningColumns(columnMapping)) {
+        statement = statement + ` RETURNING ${getReturningColumns(columnMapping).join(',')}`;
+    }
     return statement;
 }
 /**
@@ -67,7 +76,7 @@ function PostgresDestination(options) {
         write(chunk, _, callback) {
             let insertStatement;
             // @ts-ignore
-            if (chunk.errors.length === 0 | options.includeErrors) {
+            if (chunk.errors.length === 0 || options.includeErrors) {
                 insertStatement = writeInsertStatement(mapping, table, chunk);
                 debug('Insert statement: [%s]', insertStatement);
                 // @ts-ignore
@@ -99,6 +108,7 @@ function PostgresDestination(options) {
         }
     });
     Object.assign(Transform.prototype, {
+        type: 'PostgresDestination',
         setConnection: function (connection) {
             this.connection = connection;
         }.bind(transform),
@@ -116,5 +126,7 @@ module.exports = {
     PostgresDestination,
     writeInsertStatement,
     isKeyWord,
-    getMappingForColumn
+    getMappingForColumn,
+    hasReturningColumns,
+    getReturningColumns
 };

package/dist/cjs/lib/index.js

@@ -3,6 +3,7 @@ const EventEmitter = require('node:events');
 const util = require('node:util');
 const { RowMetaData } = require("./rowMetaData");
 const { compose } = require("node:stream");
+const { Destinations } = require('..');
 /**
  * @typedef {Object} Etl
  * @function loader
@@ -18,7 +19,8 @@ const { compose } = require("node:stream");
  */
 function Etl() {
     EventEmitter.call(this);
-    let self = this;
+    const self = this;
+    self.store = [];
     self.beforeETLList = [];
     self.connectionList = [];
     self.validatorList = [];
@@ -46,6 +48,7 @@ function Etl() {
             throw new Error(`Connection with name ${connectionname} not found`);
         }
         pipelineTask.setConnection(connection);
+        pipelineTask.setETL(self);
         self.beforeETLList.push(pipelineTask);
         return self;
     };
@@ -60,10 +63,18 @@ function Etl() {
     this.destination = (destination, ...tasks) => {
         const connectionname = destination.getConnectionName();
         const connection = this.connectionList.filter(c => c.name === connectionname)[0];
-        if (connection) {
+        if (!connection && destination.type === 'PostgresDestination') {
+            throw new Error(`No connection could be found with name ${connectionname}`);
+        }
+        else {
             destination.setConnection(connection);
         }
-        tasks && destination.setTasks(tasks);
+        if (tasks) {
+            for (const task of tasks) {
+                task.setETL(self);
+            }
+            destination.setTasks(tasks);
+        }
         self.destinationList.push(destination);
         return self;
     };

package/dist/cjs/types/destinations/postgresDestination.d.ts

@@ -16,5 +16,7 @@ export function PostgresDestination(options: {
 export function writeInsertStatement(columnMapping: any, table: any, chunk: any): string;
 export function isKeyWord(column: any): boolean;
 export function getMappingForColumn(mapping: any, column: any): any;
+export function hasReturningColumns(mapping: any): boolean;
+export function getReturningColumns(mapping: any): any;
 import { Transform } from "stream";
 //# sourceMappingURL=postgresDestination.d.ts.map

package/dist/cjs/types/destinations/postgresDestination.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AA6CA;;;;;;;;GAQG;AACH;IAN2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;cA+DvC;AAnGD,yFA4BC;AArCD,gDAEC;AAED,oEAGC"}
+{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAuDA;;;;;;;;GAQG;AACH;IAN2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;cAgEvC;AAtGD,yFA8BC;AAhDD,gDAEC;AAED,oEAIC;AAED,2DAEC;AAED,uDAEC"}

package/dist/cjs/types/lib/index.d.ts

@@ -14,6 +14,7 @@ export type Etl = any;
  */
 export function Etl(): void;
 export class Etl {
+    store: any[];
     beforeETLList: any[];
     connectionList: any[];
     validatorList: any[];

package/dist/cjs/types/lib/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAMA;;;;;;;;GAQG;AAEH;;;GAGG;AACH,4BAwEC;;IArEG,qBAAuB;IACvB,sBAAwB;IACxB,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,6BAGC;IAED,gBAgBC;IAED,sCASC;IAED,qCAGC;IAED,mCAGC;IAED,wDAWC;IAED,mCAGC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAOA;;;;;;;;GAQG;AAEH;;;GAGG;AACH,4BAgFC;;IA7EG,aAAe;IACf,qBAAuB;IACvB,sBAAwB;IACxB,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,6BAGC;IAED,gBAgBC;IAED,sCAUC;IAED,qCAGC;IAED,mCAGC;IAED,wDAiBC;IAED,mCAGC"}

package/dist/esm/destinations/postgresDestination.js

@@ -2,44 +2,53 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 const EventEmitter = require('node:events');
 const { Transform } = require('node:stream');
-const { Sequelize } = require('sequelize');
 const debug = require('debug')('destination');
 const DEFAULT_PORT = 5432;
 function isKeyWord(column) {
     return ['USER'].includes(column.toUpperCase());
 }
 function getMappingForColumn(mapping, column) {
+    if (mapping.length === 0) {
+        return;
+    }
     const [map] = mapping.filter(m => m.column === column);
     return map;
 }
+function hasReturningColumns(mapping) {
+    return mapping.filter(m => m.returning === true).length > 0;
+}
+function getReturningColumns(mapping) {
+    return mapping.filter(m => m.returning === true).map(m => m?.targetColumn);
+}
 function writeInsertStatement(columnMapping, table, chunk) {
     let statement = `INSERT INTO ${table} (${chunk._columns.map(column => {
         const mapping = getMappingForColumn(columnMapping, column);
-        return mapping.targetColumn
+        return mapping?.targetColumn
             ?
                 isKeyWord(mapping.targetColumn)
                     ? `"${mapping.targetColumn}"`
                     : mapping.targetColumn
             : isKeyWord(column)
-                ? `"${mapping.column}"`
-                : mapping.column;
+                ? `"${mapping?.column}"`
+                : mapping?.column;
     })
         .join(",")}) VALUES (${chunk._columns.map((column, index) => {
         const mapping = getMappingForColumn(columnMapping, column);
-        if (!mapping)
-            debug('Mapping not found for column %s', column);
-        if (mapping.targetType === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
+        if (mapping?.targetType === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
             debug('Source data is not a number');
             return 0;
         }
-        if (mapping.targetType === "varchar" || mapping.targetType === "char") {
+        if (mapping?.targetType === "varchar" || mapping?.targetType === "char") {
             return `'${chunk[index]}'`;
         }
-        if (mapping.targetType === "date") {
-            return `to_date('${chunk[index]}','${mapping.format}')`;
+        if (mapping?.targetType === "date") {
+            return `to_date('${chunk[index]}','${mapping?.format}')`;
         }
         return chunk[index] ? chunk[index] : 'null';
     })})`;
+    if (hasReturningColumns(columnMapping)) {
+        statement = statement + ` RETURNING ${getReturningColumns(columnMapping).join(',')}`;
+    }
     return statement;
 }
 /**
@@ -69,7 +78,7 @@ function PostgresDestination(options) {
         write(chunk, _, callback) {
             let insertStatement;
             // @ts-ignore
-            if (chunk.errors.length === 0 | options.includeErrors) {
+            if (chunk.errors.length === 0 || options.includeErrors) {
                 insertStatement = writeInsertStatement(mapping, table, chunk);
                 debug('Insert statement: [%s]', insertStatement);
                 // @ts-ignore
@@ -100,6 +109,7 @@ function PostgresDestination(options) {
         }
     });
     Object.assign(Transform.prototype, {
+        type: 'PostgresDestination',
         setConnection: function (connection) {
             this.connection = connection;
         }.bind(transform),
@@ -116,5 +126,7 @@ module.exports = {
     PostgresDestination,
     writeInsertStatement,
     isKeyWord,
-    getMappingForColumn
+    getMappingForColumn,
+    hasReturningColumns,
+    getReturningColumns
 };

package/dist/esm/lib/index.js

@@ -5,6 +5,7 @@ const EventEmitter = require('node:events');
 const util = require('node:util');
 const { RowMetaData } = require("./rowMetaData");
 const { compose } = require("node:stream");
+const { Destinations } = require('..');
 /**
  * @typedef {Object} Etl
  * @function loader
@@ -20,7 +21,8 @@ const { compose } = require("node:stream");
  */
 function Etl() {
     EventEmitter.call(this);
-    let self = this;
+    const self = this;
+    self.store = [];
     self.beforeETLList = [];
     self.connectionList = [];
     self.validatorList = [];
@@ -48,6 +50,7 @@ function Etl() {
             throw new Error(`Connection with name ${connectionname} not found`);
         }
         pipelineTask.setConnection(connection);
+        pipelineTask.setETL(self);
         self.beforeETLList.push(pipelineTask);
         return self;
     };
@@ -62,10 +65,18 @@ function Etl() {
     this.destination = (destination, ...tasks) => {
         const connectionname = destination.getConnectionName();
         const connection = this.connectionList.filter(c => c.name === connectionname)[0];
-        if (connection) {
+        if (!connection && destination.type === 'PostgresDestination') {
+            throw new Error(`No connection could be found with name ${connectionname}`);
+        }
+        else {
             destination.setConnection(connection);
         }
-        tasks && destination.setTasks(tasks);
+        if (tasks) {
+            for (const task of tasks) {
+                task.setETL(self);
+            }
+            destination.setTasks(tasks);
+        }
         self.destinationList.push(destination);
         return self;
     };

package/dist/esm/types/destinations/postgresDestination.d.ts

@@ -16,5 +16,7 @@ export function PostgresDestination(options: {
 export function writeInsertStatement(columnMapping: any, table: any, chunk: any): string;
 export function isKeyWord(column: any): boolean;
 export function getMappingForColumn(mapping: any, column: any): any;
+export function hasReturningColumns(mapping: any): boolean;
+export function getReturningColumns(mapping: any): any;
 import { Transform } from "stream";
 //# sourceMappingURL=postgresDestination.d.ts.map

package/dist/esm/types/destinations/postgresDestination.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AA6CA;;;;;;;;GAQG;AACH;IAN2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;cA+DvC;AAnGD,yFA4BC;AArCD,gDAEC;AAED,oEAGC"}
+{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAuDA;;;;;;;;GAQG;AACH;IAN2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;cAgEvC;AAtGD,yFA8BC;AAhDD,gDAEC;AAED,oEAIC;AAED,2DAEC;AAED,uDAEC"}

package/dist/esm/types/lib/index.d.ts

@@ -14,6 +14,7 @@ export type Etl = any;
  */
 export function Etl(): void;
 export class Etl {
+    store: any[];
     beforeETLList: any[];
     connectionList: any[];
     validatorList: any[];

package/dist/esm/types/lib/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAMA;;;;;;;;GAQG;AAEH;;;GAGG;AACH,4BAwEC;;IArEG,qBAAuB;IACvB,sBAAwB;IACxB,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,6BAGC;IAED,gBAgBC;IAED,sCASC;IAED,qCAGC;IAED,mCAGC;IAED,wDAWC;IAED,mCAGC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAOA;;;;;;;;GAQG;AAEH;;;GAGG;AACH,4BAgFC;;IA7EG,aAAe;IACf,qBAAuB;IACvB,sBAAwB;IACxB,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,6BAGC;IAED,gBAgBC;IAED,sCAUC;IAED,qCAGC;IAED,mCAGC;IAED,wDAiBC;IAED,mCAGC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ffc-pay-etl-framework",
-  "version": "1.0.0",
+  "version": "1.1.0-alpha.10",
   "publisher": "Defra",
   "main": "dist/cjs/index.js",
   "private": false,
@@ -13,7 +13,7 @@
     "build:esm": "tsc -p ./tsconfig.esm.json && mv dist/esm/index.js dist/esm/index.mjs",
     "build:cjs": "tsc -p ./tsconfig.cjs.json",
     "prepack": "npm run build",
-    "example": "node ./src/examples/csv-multi-tool-validator.js"
+    "example": "node ./src/examples/sql-return-values.js"
   },
   "keywords": [
     "ETL"