ffc-pay-etl-framework 0.0.9-alpha.13 → 0.0.9

package/dist/cjs/destinations/postgresDestination.js

@@ -43,15 +43,29 @@ function PostgresDestination(options) {
         const [map] = mapping.filter(m => m.column === column);
         return map;
     }
+    function isKeyWord(column) {
+        return ['USER'].includes(column.toUpperCase());
+    }
     function writeInsertStatement(chunk) {
         let statement = `INSERT INTO ${table} (${chunk._columns.map(column => {
             let mapping = getMappingForColumn(column);
-            return mapping ? mapping.targetColumn : column;
+            return mapping
+                ?
+                    isKeyWord(mapping.targetColumn)
+                        ? `"${mapping.targetColumn}"`
+                        : mapping.targetColumn
+                : isKeyWord(column)
+                    ? `"${mapping.column}"`
+                    : mapping.column;
         })
             .join(",")}) VALUES (${chunk._columns.map((column, index) => {
             let mapping = getMappingForColumn(column);
             if (!mapping)
                 debug('Mapping not found for column %s', column);
+            if (mapping.targetType === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
+                debug('Source data is not a number');
+                return 0;
+            }
             if (mapping.targetType === "varchar" || mapping.targetType === "char")
                 return `'${chunk[index]}'`;
             return chunk[index] ? chunk[index] : 'null';

package/dist/cjs/lib/index.js

@@ -1,4 +1,6 @@
 // @ts-nocheck
+const EventEmitter = require('node:events');
+const util = require('node:util');
 const { RowMetaData } = require("./rowMetaData");
 const { compose } = require("node:stream");
 /**
@@ -14,6 +16,7 @@ const { compose } = require("node:stream");
  * @returns Etl
  */
 function Etl() {
+    EventEmitter.call(this);
     let self = this;
     self.validatorList = [];
     self.transformationList = [];
@@ -25,7 +28,10 @@ function Etl() {
     this.pump = () => {
         this.loader
             .pump(this.loader)
-            .pipe(compose(RowMetaData(), ...self.validatorList, ...self.transformationList, ...self.destinationList));
+            .pipe(compose(RowMetaData(), ...self.validatorList, ...self.transformationList, ...self.destinationList)
+        // @ts-ignore
+        ).on('finish', (data) => self.emit('finish', data));
+        return self;
     };
     this.validator = (validator) => {
         self.validatorList.push(validator);
@@ -41,6 +47,7 @@ function Etl() {
     };
     return;
 }
+util.inherits(Etl, EventEmitter);
 module.exports = {
     Etl
 };

package/dist/cjs/types/destinations/postgresDestination.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAIA;;;;;;;;;;;;GAYG;AACH,6CAVG;IAAwB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,aAAa;CACrC,aA+EF"}
+{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAIA;;;;;;;;;;;;GAYG;AACH,6CAVG;IAAwB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,aAAa;CACrC,aA6FF"}

package/dist/cjs/types/lib/index.d.ts

@@ -17,7 +17,7 @@ export class Etl {
     transformationList: any[];
     destinationList: any[];
     loader: (loader: any) => this;
-    pump: () => void;
+    pump: () => this;
     validator: (validator: any) => this;
     destination: (destination: any) => this;
     transform: (transform: any) => this;

package/dist/cjs/types/lib/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAIA;;;;;;;GAOG;AAEH;;;GAGG;AACH,4BAwCC;;IAtCG,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,8BAGC;IAED,iBAWC;IAED,oCAGC;IAED,wCAGC;IAED,oCAGC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAMA;;;;;;;GAOG;AAEH;;;GAGG;AACH,4BA2CC;;IAxCG,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,8BAGC;IAED,iBAaC;IAED,oCAGC;IAED,wCAGC;IAED,oCAGC"}

package/dist/esm/destinations/postgresDestination.js

@@ -44,15 +44,29 @@ function PostgresDestination(options) {
         const [map] = mapping.filter(m => m.column === column);
         return map;
     }
+    function isKeyWord(column) {
+        return ['USER'].includes(column.toUpperCase());
+    }
     function writeInsertStatement(chunk) {
         let statement = `INSERT INTO ${table} (${chunk._columns.map(column => {
             let mapping = getMappingForColumn(column);
-            return mapping ? mapping.targetColumn : column;
+            return mapping
+                ?
+                    isKeyWord(mapping.targetColumn)
+                        ? `"${mapping.targetColumn}"`
+                        : mapping.targetColumn
+                : isKeyWord(column)
+                    ? `"${mapping.column}"`
+                    : mapping.column;
         })
             .join(",")}) VALUES (${chunk._columns.map((column, index) => {
             let mapping = getMappingForColumn(column);
             if (!mapping)
                 debug('Mapping not found for column %s', column);
+            if (mapping.targetType === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
+                debug('Source data is not a number');
+                return 0;
+            }
             if (mapping.targetType === "varchar" || mapping.targetType === "char")
                 return `'${chunk[index]}'`;
             return chunk[index] ? chunk[index] : 'null';

package/dist/esm/lib/index.js

@@ -1,5 +1,7 @@
 "use strict";
 // @ts-nocheck
+const EventEmitter = require('node:events');
+const util = require('node:util');
 const { RowMetaData } = require("./rowMetaData");
 const { compose } = require("node:stream");
 /**
@@ -15,6 +17,7 @@ const { compose } = require("node:stream");
  * @returns Etl
  */
 function Etl() {
+    EventEmitter.call(this);
     let self = this;
     self.validatorList = [];
     self.transformationList = [];
@@ -26,7 +29,10 @@ function Etl() {
     this.pump = () => {
         this.loader
             .pump(this.loader)
-            .pipe(compose(RowMetaData(), ...self.validatorList, ...self.transformationList, ...self.destinationList));
+            .pipe(compose(RowMetaData(), ...self.validatorList, ...self.transformationList, ...self.destinationList)
+        // @ts-ignore
+        ).on('finish', (data) => self.emit('finish', data));
+        return self;
     };
     this.validator = (validator) => {
         self.validatorList.push(validator);
@@ -42,6 +48,7 @@ function Etl() {
     };
     return;
 }
+util.inherits(Etl, EventEmitter);
 module.exports = {
     Etl
 };

package/dist/esm/types/destinations/postgresDestination.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAIA;;;;;;;;;;;;GAYG;AACH,6CAVG;IAAwB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,aAAa;CACrC,aA+EF"}
+{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAIA;;;;;;;;;;;;GAYG;AACH,6CAVG;IAAwB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,aAAa;CACrC,aA6FF"}

package/dist/esm/types/lib/index.d.ts

@@ -17,7 +17,7 @@ export class Etl {
     transformationList: any[];
     destinationList: any[];
     loader: (loader: any) => this;
-    pump: () => void;
+    pump: () => this;
     validator: (validator: any) => this;
     destination: (destination: any) => this;
     transform: (transform: any) => this;

package/dist/esm/types/lib/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAIA;;;;;;;GAOG;AAEH;;;GAGG;AACH,4BAwCC;;IAtCG,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,8BAGC;IAED,iBAWC;IAED,oCAGC;IAED,wCAGC;IAED,oCAGC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAMA;;;;;;;GAOG;AAEH;;;GAGG;AACH,4BA2CC;;IAxCG,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,8BAGC;IAED,iBAaC;IAED,oCAGC;IAED,wCAGC;IAED,oCAGC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ffc-pay-etl-framework",
-  "version": "0.0.9-alpha.13",
+  "version": "0.0.9",
   "publisher": "Defra",
   "main": "dist/cjs/index.js",
   "private": false,
@@ -24,35 +24,6 @@
   "jest": {
     "setupFiles": [
       "<rootDir>/jest/setEnvVars.js"
-    ],
-    "collectCoverage": true,
-    "collectCoverageFrom": [
-      "**/*.js",
-      "!**/*.test.js"
-    ],
-    "coverageDirectory": "test-output",
-    "coverageReporters": [
-      "text-summary",
-      "lcov"
-    ],
-    "coveragePathIgnorePatterns": [
-      "<rootDir>/node_modules/",
-      "<rootDir>/test-output/",
-      "<rootDir>/test/"
-    ],
-    "modulePathIgnorePatterns": [
-      "node_modules"
-    ],
-    "reporters": [
-      "default",
-      [
-        "jest-junit",
-        {
-          "suiteName": "jest tests",
-          "outputDirectory": "test-output",
-          "outputName": "junit.xml"
-        }
-      ]
     ]
   },
   "devDependencies": {
@@ -66,7 +37,6 @@
   "dependencies": {
     "@faker-js/faker": "^8.4.1",
     "csv": "^6.3.9",
-    "jest-junit": "16.0.0",
     "mariadb": "^3.3.1",
     "mysql2": "^3.10.3",
     "oracledb": "^6.5.1",