ffc-pay-etl-framework 1.1.14-alpha.9 → 1.2.0

package/README.md

@@ -2,7 +2,7 @@
 
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
 
-[![All Contributors](https://img.shields.io/badge/all_contributors-2-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-3-orange.svg?style=flat-square)](#contributors-)
 
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
@@ -98,8 +98,8 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
   <tbody>
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/suityou01"><img src="https://avatars.githubusercontent.com/u/7422644?v=4?s=100" width="100px;" alt="Charlie Benger-Stevenson"/><br /><sub><b>Charlie Benger-Stevenson</b></sub></a><br /><a href="https://github.com/DEFRA/ffc-pay-etl-framework/commits?author=suityou01" title="Code">💻</a></td>
-      <td align="center" valign="top" width="14.28%"><a href="https://github.com/GodsonLeigh"><img src="https://avatars.githubusercontent.com/u/139965284?v=4?s=100" width="100px;" alt="GodsonLeigh"/><br /><sub><b>GodsonLeigh</b></sub></a><br /><a href="https://github.com/DEFRA/ffc-pay-etl-framework/commits?author=GodsonLeigh" title="Code">💻</a></td>
-    </tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/GodsonLeigh"><img src="https://avatars.githubusercontent.com/u/139965284?v=4?s=100" width="100px;" alt="Leigh Godson"/><br /><sub><b>Leigh Godson</b></sub></a><br /><a href="https://github.com/DEFRA/ffc-pay-etl-framework/commits?author=GodsonLeigh" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/samplackett"><img src="https://avatars.githubusercontent.com/u/60177449?v=4?s=100" width="100px;" alt="Sam Plackett"/><br /><sub><b>Sam Plackett</b></sub></a><br /><a href="https://github.com/DEFRA/ffc-pay-etl-framework/commits?author=samplackett" title="Code">💻</a></td>    </tr>
   </tbody>
   <tfoot>
     <tr>

package/dist/cjs/database_connections/index.js

@@ -1,4 +1,6 @@
 const { PostgresDatabaseConnection } = require("./postgresDatabaseConnection");
+const { ProvidedConnection } = require("./providedConnection");
 module.exports = {
-    PostgresDatabaseConnection
+    PostgresDatabaseConnection,
+    ProvidedConnection
 };

package/dist/cjs/database_connections/providedConnection.js

@@ -0,0 +1,29 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+/**
+ *
+ * @param {Object} options
+ * @param {Object} options.connectionname
+ * @param {Object} options.sequelize
+ * @returns Connection
+ */
+function ProvidedConnection(options) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const connectionname = options.connectionname;
+        const sequelize = options.sequelize;
+        return {
+            name: connectionname,
+            db: sequelize
+        };
+    });
+}
+module.exports = {
+    ProvidedConnection
+};

package/dist/cjs/destinations/consoleDestination.js

@@ -10,10 +10,11 @@ function ConsoleDestination(options) {
     const writable = new Writable({
         objectMode: true,
         write(chunk, _, callback) {
+            var _a;
             if (chunk.errors.length === 0 || includeErrors)
                 console.log(chunk);
             // @ts-ignore
-            this.tasks && this.tasks.forEach(task => task.write(chunk));
+            (_a = this.tasks) === null || _a === void 0 ? void 0 : _a.forEach(task => task.write(chunk));
             callback();
         }
     });
@@ -22,7 +23,8 @@ function ConsoleDestination(options) {
             this.connection = connection;
         }.bind(writable),
         getConnectionName: function () {
-            return this.connection && this.connection.name;
+            var _a;
+            return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.name;
         }.bind(writable),
         setTasks: function (tasks) {
             this.tasks = tasks;

package/dist/cjs/destinations/csvFileDestination.js

@@ -26,6 +26,7 @@ function CSVFileDestination(options) {
     const writable = new Writable({
         objectMode: true,
         write(chunk, _, callback) {
+            var _a;
             if (!headersWritten && headers) {
                 if (quotationMarks) {
                     fs.writeFileSync(fileHandle, `${chunk._columns.map(c => `"${c}"`).join(",")}\n`);
@@ -44,7 +45,7 @@ function CSVFileDestination(options) {
                 }
             }
             // @ts-ignore
-            this.tasks && this.tasks.forEach(task => task.write(chunk));
+            (_a = this.tasks) === null || _a === void 0 ? void 0 : _a.forEach(task => task.write(chunk));
             lastChunk = chunk;
             callback();
         },
@@ -58,7 +59,8 @@ function CSVFileDestination(options) {
             this.connection = connection;
         }.bind(writable),
         getConnectionName: function () {
-            return this.connection && this.connection.name;
+            var _a;
+            return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.name;
         }.bind(writable),
         setTasks: function (tasks) {
             this.tasks = tasks;

package/dist/cjs/destinations/postgresDestination.js

@@ -1,43 +1,54 @@
 const EventEmitter = require('node:events');
 const { Transform } = require('node:stream');
-const { Sequelize } = require('sequelize');
 const debug = require('debug')('destination');
 const DEFAULT_PORT = 5432;
 function isKeyWord(column) {
     return ['USER'].includes(column.toUpperCase());
 }
 function getMappingForColumn(mapping, column) {
+    if (mapping.length === 0) {
+        return {};
+    }
     const [map] = mapping.filter(m => m.column === column);
     return map;
 }
-function writeInsertStatement(columnMapping, table, chunk) {
-    let statement = `INSERT INTO ${table} (${chunk._columns.map(column => {
+function hasReturningColumns(mapping) {
+    return mapping.filter(m => m.returning === true).length > 0;
+}
+function getReturningColumns(mapping) {
+    return mapping.filter(m => m.returning === true).map(m => m === null || m === void 0 ? void 0 : m.targetColumn);
+}
+function writeInsertStatement(columnMapping, table, chunk, schema, ignoredColumns = []) {
+    const filteredColumns = chunk._columns.filter(column => {
         const mapping = getMappingForColumn(columnMapping, column);
-        return mapping.targetColumn
-            ?
-                isKeyWord(mapping.targetColumn)
-                    ? `"${mapping.targetColumn}"`
-                    : mapping.targetColumn
-            : isKeyWord(column)
-                ? `"${mapping.column}"`
-                : mapping.column;
+        const targetColumn = (mapping === null || mapping === void 0 ? void 0 : mapping.targetColumn) ? mapping === null || mapping === void 0 ? void 0 : mapping.targetColumn : mapping === null || mapping === void 0 ? void 0 : mapping.column;
+        return !ignoredColumns.includes(targetColumn);
+    });
+    let statement = `INSERT INTO ${schema !== null && schema !== void 0 ? schema : 'public'}."${table}" (${filteredColumns.map(column => {
+        const mapping = getMappingForColumn(columnMapping, column);
+        return (mapping === null || mapping === void 0 ? void 0 : mapping.targetColumn) ? `"${mapping.targetColumn}"` : `"${mapping === null || mapping === void 0 ? void 0 : mapping.column}"`;
     })
-        .join(",")}) VALUES (${chunk._columns.map((column, index) => {
+        .join(",")}) VALUES (${filteredColumns.map(column => {
+        const index = chunk._columns.indexOf(column);
         const mapping = getMappingForColumn(columnMapping, column);
-        if (!mapping)
-            debug('Mapping not found for column %s', column);
-        if (mapping.targetType === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
-            debug('Source data is not a number');
+        if ((mapping === null || mapping === void 0 ? void 0 : mapping.targetType) === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
+            debug('Source data is not a number.');
             return 0;
         }
-        if (mapping.targetType === "varchar" || mapping.targetType === "char") {
+        if ((mapping === null || mapping === void 0 ? void 0 : mapping.targetType) === "varchar" || (mapping === null || mapping === void 0 ? void 0 : mapping.targetType) === "char") {
             return `'${chunk[index]}'`;
         }
-        if (mapping.targetType === "date") {
-            return `to_date('${chunk[index]}','${mapping.format}')`;
+        if ((mapping === null || mapping === void 0 ? void 0 : mapping.targetType) === "date") {
+            if (!chunk[index]) {
+                return `''`;
+            }
+            return `to_timestamp('${chunk[index]}','${mapping === null || mapping === void 0 ? void 0 : mapping.format}')`;
         }
         return chunk[index] ? chunk[index] : 'null';
     })})`;
+    if (hasReturningColumns(columnMapping)) {
+        statement = statement + ` RETURNING ${getReturningColumns(columnMapping).join(',')}`;
+    }
     return statement;
 }
 /**
@@ -47,14 +58,19 @@ function writeInsertStatement(columnMapping, table, chunk) {
  * @param {Object} options.connectionname
  * @param {Object} options.mapping
  * @param {Object} options.includeErrors
+ * @param {String} [options.schema]
+ * @param {Array<String>} [options.ignoredColumns]
  * @returns Transform
  */
 function PostgresDestination(options) {
+    var _a;
     EventEmitter.call(this);
     const table = options.table;
     const connectionname = options.connectionname;
     const mapping = options.mapping;
     const includeErrors = options.includeErrors;
+    const schema = options.schema;
+    const ignoredColumns = (_a = options.ignoredColumns) !== null && _a !== void 0 ? _a : [];
     let lastChunk;
     const transform = new Transform({
         objectMode: true,
@@ -67,17 +83,18 @@ function PostgresDestination(options) {
         write(chunk, _, callback) {
             let insertStatement;
             // @ts-ignore
-            if (chunk.errors.length === 0 | options.includeErrors) {
-                insertStatement = writeInsertStatement(mapping, table, chunk);
+            if (chunk.errors.length === 0 || options.includeErrors) {
+                insertStatement = writeInsertStatement(mapping, table, chunk, schema, ignoredColumns);
                 debug('Insert statement: [%s]', insertStatement);
                 // @ts-ignore
                 this.connection.db.query(insertStatement)
                     .then(result => {
+                    var _a;
                     debug('result %o', result);
                     chunk._result = result;
                     lastChunk = chunk;
                     // @ts-ignore
-                    this.tasks && this.tasks.forEach(task => task.write(chunk));
+                    (_a = this.tasks) === null || _a === void 0 ? void 0 : _a.forEach(task => task.write(chunk));
                     // @ts-ignore
                     callback(null, chunk);
                 }).catch(error => {
@@ -98,11 +115,13 @@ function PostgresDestination(options) {
         }
     });
     Object.assign(Transform.prototype, {
+        type: 'PostgresDestination',
         setConnection: function (connection) {
             this.connection = connection;
         }.bind(transform),
         getConnectionName: function () {
-            return this.connection && this.connection.name;
+            var _a;
+            return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.name;
         }.bind(transform),
         setTasks: function (tasks) {
             this.tasks = tasks;
@@ -114,5 +133,7 @@ module.exports = {
     PostgresDestination,
     writeInsertStatement,
     isKeyWord,
-    getMappingForColumn
+    getMappingForColumn,
+    hasReturningColumns,
+    getReturningColumns
 };

package/dist/cjs/destinations/sqlFileDestination.js

@@ -64,7 +64,8 @@ function SQLFileDestination(options) {
             this.connection = connection;
         }.bind(writable),
         getConnectionName: function () {
-            return this.connection && this.connection.name;
+            var _a;
+            return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.name;
         }.bind(writable),
         setTasks: function (tasks) {
             this.tasks = tasks;

package/dist/cjs/lib/index.js

@@ -3,6 +3,7 @@ const EventEmitter = require('node:events');
 const util = require('node:util');
 const { RowMetaData } = require("./rowMetaData");
 const { compose } = require("node:stream");
+const { Destinations } = require('..');
 /**
  * @typedef {Object} Etl
  * @function loader
@@ -18,7 +19,8 @@ const { compose } = require("node:stream");
  */
 function Etl() {
     EventEmitter.call(this);
-    let self = this;
+    const self = this;
+    self.store = [];
     self.beforeETLList = [];
     self.connectionList = [];
     self.validatorList = [];
@@ -41,11 +43,12 @@ function Etl() {
     };
     this.beforeETL = (pipelineTask) => {
         const connectionname = pipelineTask.getConnectionName();
-        const connection = this.connectionList.filter(c => c.name === connectionname);
+        const connection = this.connectionList.filter(c => c.name === connectionname)[0];
         if (!connection) {
             throw new Error(`Connection with name ${connectionname} not found`);
         }
         pipelineTask.setConnection(connection);
+        pipelineTask.setETL(self);
         self.beforeETLList.push(pipelineTask);
         return self;
     };
@@ -60,14 +63,22 @@ function Etl() {
     this.destination = (destination, ...tasks) => {
         const connectionname = destination.getConnectionName();
         const connection = this.connectionList.filter(c => c.name === connectionname)[0];
-        if (connection) {
+        if (!connection && destination.type === 'PostgresDestination') {
+            throw new Error(`No connection could be found with name ${connectionname}`);
+        }
+        else {
             destination.setConnection(connection);
         }
-        tasks && destination.setTasks(tasks);
+        if (tasks) {
+            for (const task of tasks) {
+                task.setETL(self);
+            }
+            destination.setTasks(tasks);
+        }
         self.destinationList.push(destination);
         return self;
     };
-    this.transform = (transform, ...tasks) => {
+    this.transform = (transform) => {
         self.transformationList.push(transform);
         return self;
     };

package/dist/cjs/loaders/csvloader.js

@@ -2,21 +2,31 @@
 const fs = require("fs");
 const { Transform } = require("stream");
 const { parse } = require("csv-parse");
-const { stdout, stderr } = require("process");
 /**
  *
  * @param {Object} options
- * @param {String} options.path
+ * @param {String} [options.path]
+ * @param {ReadableStream} [options.stream]
+ * @param {Number} [options.startingLine]
+ * @param {Boolean} [options.relax]
  * @returns StreamReader
  */
 function CSVLoader(options) {
-    let csvLoader = fs.createReadStream(options.path);
+    var _a, _b;
+    let csvLoader;
+    if (options.path) {
+        csvLoader = fs.createReadStream(options.path);
+    }
+    else {
+        csvLoader = options.stream;
+    }
     let lineCount = 1;
+    const fromLine = (_a = options.startingLine) !== null && _a !== void 0 ? _a : 2;
+    const relaxQuotes = (_b = options.relax) !== null && _b !== void 0 ? _b : false;
     csvLoader._columns = options.columns;
     csvLoader.pump = (csvLoader) => {
-        return csvLoader
-            .pipe(parse({ delimiter: ",", from_line: 2 }))
-            .pipe(new Transform({
+        const parser = parse({ delimiter: ",", from_line: fromLine, relax_quotes: relaxQuotes });
+        const transformer = new Transform({
             readableObjectMode: true,
             writableObjectMode: true,
             emitClose: true,
@@ -24,9 +34,18 @@ function CSVLoader(options) {
                 chunk["_columns"] = options.columns;
                 chunk["_linecount"] = lineCount;
                 lineCount += 1;
+                // remove non-printable characters
+                options.columns.forEach((_column, index) => {
+                    if (chunk[index]) {
+                        chunk[index] = chunk[index].replace(/[\x00-\x1F\x7F-\x9F]/g, '');
+                    }
+                });
                 callback(null, chunk);
             }
-        }));
+        });
+        return csvLoader
+            .pipe(parser)
+            .pipe(transformer);
     };
     return csvLoader;
 }

package/dist/cjs/types/database_connections/index.d.ts

@@ -1,3 +1,4 @@
-export { PostgresDatabaseConnection };
 import { PostgresDatabaseConnection } from "./postgresDatabaseConnection";
+import { ProvidedConnection } from "./providedConnection";
+export { PostgresDatabaseConnection, ProvidedConnection };
 //# sourceMappingURL=index.d.ts.map

package/dist/cjs/types/database_connections/providedConnection.d.ts

@@ -0,0 +1,15 @@
+/**
+ *
+ * @param {Object} options
+ * @param {Object} options.connectionname
+ * @param {Object} options.sequelize
+ * @returns Connection
+ */
+export function ProvidedConnection(options: {
+    connectionname: any;
+    sequelize: any;
+}): Promise<{
+    name: any;
+    db: any;
+}>;
+//# sourceMappingURL=providedConnection.d.ts.map

package/dist/cjs/types/database_connections/providedConnection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"providedConnection.d.ts","sourceRoot":"","sources":["../../../../src/database_connections/providedConnection.js"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH;IAJ2B,cAAc;IACd,SAAS;;;;GAUnC"}

package/dist/cjs/types/destinations/postgresDestination.d.ts

@@ -5,6 +5,8 @@
  * @param {Object} options.connectionname
  * @param {Object} options.mapping
  * @param {Object} options.includeErrors
+ * @param {String} [options.schema]
+ * @param {Array<String>} [options.ignoredColumns]
  * @returns Transform
  */
 export function PostgresDestination(options: {
@@ -12,9 +14,13 @@ export function PostgresDestination(options: {
     connectionname: any;
     mapping: any;
     includeErrors: any;
+    schema?: string;
+    ignoredColumns?: Array<string>;
 }): Transform;
-export function writeInsertStatement(columnMapping: any, table: any, chunk: any): string;
+export function writeInsertStatement(columnMapping: any, table: any, chunk: any, schema: any, ignoredColumns?: any[]): string;
 export function isKeyWord(column: any): boolean;
 export function getMappingForColumn(mapping: any, column: any): any;
+export function hasReturningColumns(mapping: any): boolean;
+export function getReturningColumns(mapping: any): any;
 import { Transform } from "stream";
 //# sourceMappingURL=postgresDestination.d.ts.map

package/dist/cjs/types/destinations/postgresDestination.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AA6CA;;;;;;;;GAQG;AACH;IAN2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;cA+DvC;AAnGD,yFA4BC;AArCD,gDAEC;AAED,oEAGC"}
+{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAyDA;;;;;;;;;;GAUG;AACH;IAR2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;IACZ,MAAM;IACC,cAAc,GAAtC,aAAa;cAkEvB;AA5GD,8HAgCC;AAlDD,gDAEC;AAED,oEAIC;AAED,2DAEC;AAED,uDAEC"}

package/dist/cjs/types/lib/index.d.ts

@@ -14,6 +14,7 @@ export type Etl = any;
  */
 export function Etl(): void;
 export class Etl {
+    store: any[];
     beforeETLList: any[];
     connectionList: any[];
     validatorList: any[];
@@ -25,6 +26,6 @@ export class Etl {
     connection: (connection: any) => Etl;
     validator: (validator: any) => Etl;
     destination: (destination: any, ...tasks: any[]) => Etl;
-    transform: (transform: any, ...tasks: any[]) => Etl;
+    transform: (transform: any) => Etl;
 }
 //# sourceMappingURL=index.d.ts.map

package/dist/cjs/types/lib/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAMA;;;;;;;;GAQG;AAEH;;;GAGG;AACH,4BAwEC;;IArEG,qBAAuB;IACvB,sBAAwB;IACxB,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,6BAGC;IAED,gBAgBC;IAED,sCASC;IAED,qCAGC;IAED,mCAGC;IAED,wDAWC;IAED,oDAGC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAOA;;;;;;;;GAQG;AAEH;;;GAGG;AACH,4BAgFC;;IA7EG,aAAe;IACf,qBAAuB;IACvB,sBAAwB;IACxB,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,6BAGC;IAED,gBAgBC;IAED,sCAUC;IAED,qCAGC;IAED,mCAGC;IAED,wDAiBC;IAED,mCAGC"}

package/dist/cjs/types/loaders/csvloader.d.ts

@@ -2,11 +2,17 @@
 /**
  *
  * @param {Object} options
- * @param {String} options.path
+ * @param {String} [options.path]
+ * @param {ReadableStream} [options.stream]
+ * @param {Number} [options.startingLine]
+ * @param {Boolean} [options.relax]
  * @returns StreamReader
  */
 export function CSVLoader(options: {
-    path: string;
-}): fs.ReadStream;
+    path?: string;
+    stream?: ReadableStream;
+    startingLine?: number;
+    relax?: boolean;
+}): ReadableStream<any> | fs.ReadStream;
 import fs = require("fs");
 //# sourceMappingURL=csvloader.d.ts.map

package/dist/cjs/types/loaders/csvloader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csvloader.d.ts","sourceRoot":"","sources":["../../../../src/loaders/csvloader.js"],"names":[],"mappings":";AAMA;;;;;GAKG;AACH;IAH2B,IAAI;kBAwB9B"}
+{"version":3,"file":"csvloader.d.ts","sourceRoot":"","sources":["../../../../src/loaders/csvloader.js"],"names":[],"mappings":";AAKA;;;;;;;;GAQG;AACH;IAN4B,IAAI;IACI,MAAM,GAA/B,cAAc;IACG,YAAY;IACX,KAAK;wCAyCjC"}

package/dist/esm/database_connections/index.js

@@ -1,6 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const { PostgresDatabaseConnection } = require("./postgresDatabaseConnection");
+const { ProvidedConnection } = require("./providedConnection");
 module.exports = {
-    PostgresDatabaseConnection
+    PostgresDatabaseConnection,
+    ProvidedConnection
 };

package/dist/esm/database_connections/providedConnection.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/**
+ *
+ * @param {Object} options
+ * @param {Object} options.connectionname
+ * @param {Object} options.sequelize
+ * @returns Connection
+ */
+async function ProvidedConnection(options) {
+    const connectionname = options.connectionname;
+    const sequelize = options.sequelize;
+    return {
+        name: connectionname,
+        db: sequelize
+    };
+}
+module.exports = {
+    ProvidedConnection
+};

package/dist/esm/destinations/consoleDestination.js

@@ -15,7 +15,7 @@ function ConsoleDestination(options) {
             if (chunk.errors.length === 0 || includeErrors)
                 console.log(chunk);
             // @ts-ignore
-            this.tasks && this.tasks.forEach(task => task.write(chunk));
+            this.tasks?.forEach(task => task.write(chunk));
             callback();
         }
     });
@@ -24,7 +24,7 @@ function ConsoleDestination(options) {
             this.connection = connection;
         }.bind(writable),
         getConnectionName: function () {
-            return this.connection && this.connection.name;
+            return this.connection?.name;
         }.bind(writable),
         setTasks: function (tasks) {
             this.tasks = tasks;

package/dist/esm/destinations/csvFileDestination.js

@@ -46,7 +46,7 @@ function CSVFileDestination(options) {
                 }
             }
             // @ts-ignore
-            this.tasks && this.tasks.forEach(task => task.write(chunk));
+            this.tasks?.forEach(task => task.write(chunk));
             lastChunk = chunk;
             callback();
         },
@@ -60,7 +60,7 @@ function CSVFileDestination(options) {
             this.connection = connection;
         }.bind(writable),
         getConnectionName: function () {
-            return this.connection && this.connection.name;
+            return this.connection?.name;
         }.bind(writable),
         setTasks: function (tasks) {
             this.tasks = tasks;

package/dist/esm/destinations/postgresDestination.js

@@ -2,44 +2,55 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 const EventEmitter = require('node:events');
 const { Transform } = require('node:stream');
-const { Sequelize } = require('sequelize');
 const debug = require('debug')('destination');
 const DEFAULT_PORT = 5432;
 function isKeyWord(column) {
     return ['USER'].includes(column.toUpperCase());
 }
 function getMappingForColumn(mapping, column) {
+    if (mapping.length === 0) {
+        return {};
+    }
     const [map] = mapping.filter(m => m.column === column);
     return map;
 }
-function writeInsertStatement(columnMapping, table, chunk) {
-    let statement = `INSERT INTO ${table} (${chunk._columns.map(column => {
+function hasReturningColumns(mapping) {
+    return mapping.filter(m => m.returning === true).length > 0;
+}
+function getReturningColumns(mapping) {
+    return mapping.filter(m => m.returning === true).map(m => m?.targetColumn);
+}
+function writeInsertStatement(columnMapping, table, chunk, schema, ignoredColumns = []) {
+    const filteredColumns = chunk._columns.filter(column => {
         const mapping = getMappingForColumn(columnMapping, column);
-        return mapping.targetColumn
-            ?
-                isKeyWord(mapping.targetColumn)
-                    ? `"${mapping.targetColumn}"`
-                    : mapping.targetColumn
-            : isKeyWord(column)
-                ? `"${mapping.column}"`
-                : mapping.column;
+        const targetColumn = mapping?.targetColumn ? mapping?.targetColumn : mapping?.column;
+        return !ignoredColumns.includes(targetColumn);
+    });
+    let statement = `INSERT INTO ${schema ?? 'public'}."${table}" (${filteredColumns.map(column => {
+        const mapping = getMappingForColumn(columnMapping, column);
+        return mapping?.targetColumn ? `"${mapping.targetColumn}"` : `"${mapping?.column}"`;
     })
-        .join(",")}) VALUES (${chunk._columns.map((column, index) => {
+        .join(",")}) VALUES (${filteredColumns.map(column => {
+        const index = chunk._columns.indexOf(column);
         const mapping = getMappingForColumn(columnMapping, column);
-        if (!mapping)
-            debug('Mapping not found for column %s', column);
-        if (mapping.targetType === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
-            debug('Source data is not a number');
+        if (mapping?.targetType === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
+            debug('Source data is not a number.');
             return 0;
         }
-        if (mapping.targetType === "varchar" || mapping.targetType === "char") {
+        if (mapping?.targetType === "varchar" || mapping?.targetType === "char") {
             return `'${chunk[index]}'`;
         }
-        if (mapping.targetType === "date") {
-            return `to_date('${chunk[index]}','${mapping.format}')`;
+        if (mapping?.targetType === "date") {
+            if (!chunk[index]) {
+                return `''`;
+            }
+            return `to_timestamp('${chunk[index]}','${mapping?.format}')`;
         }
         return chunk[index] ? chunk[index] : 'null';
     })})`;
+    if (hasReturningColumns(columnMapping)) {
+        statement = statement + ` RETURNING ${getReturningColumns(columnMapping).join(',')}`;
+    }
     return statement;
 }
 /**
@@ -49,6 +60,8 @@ function writeInsertStatement(columnMapping, table, chunk) {
  * @param {Object} options.connectionname
  * @param {Object} options.mapping
  * @param {Object} options.includeErrors
+ * @param {String} [options.schema]
+ * @param {Array<String>} [options.ignoredColumns]
  * @returns Transform
  */
 function PostgresDestination(options) {
@@ -57,6 +70,8 @@ function PostgresDestination(options) {
     const connectionname = options.connectionname;
     const mapping = options.mapping;
     const includeErrors = options.includeErrors;
+    const schema = options.schema;
+    const ignoredColumns = options.ignoredColumns ?? [];
     let lastChunk;
     const transform = new Transform({
         objectMode: true,
@@ -69,8 +84,8 @@ function PostgresDestination(options) {
         write(chunk, _, callback) {
             let insertStatement;
             // @ts-ignore
-            if (chunk.errors.length === 0 | options.includeErrors) {
-                insertStatement = writeInsertStatement(mapping, table, chunk);
+            if (chunk.errors.length === 0 || options.includeErrors) {
+                insertStatement = writeInsertStatement(mapping, table, chunk, schema, ignoredColumns);
                 debug('Insert statement: [%s]', insertStatement);
                 // @ts-ignore
                 this.connection.db.query(insertStatement)
@@ -79,7 +94,7 @@ function PostgresDestination(options) {
                     chunk._result = result;
                     lastChunk = chunk;
                     // @ts-ignore
-                    this.tasks && this.tasks.forEach(task => task.write(chunk));
+                    this.tasks?.forEach(task => task.write(chunk));
                     // @ts-ignore
                     callback(null, chunk);
                 }).catch(error => {
@@ -100,11 +115,12 @@ function PostgresDestination(options) {
         }
     });
     Object.assign(Transform.prototype, {
+        type: 'PostgresDestination',
         setConnection: function (connection) {
             this.connection = connection;
         }.bind(transform),
         getConnectionName: function () {
-            return this.connection && this.connection.name;
+            return this.connection?.name;
         }.bind(transform),
         setTasks: function (tasks) {
             this.tasks = tasks;
@@ -116,5 +132,7 @@ module.exports = {
     PostgresDestination,
     writeInsertStatement,
     isKeyWord,
-    getMappingForColumn
+    getMappingForColumn,
+    hasReturningColumns,
+    getReturningColumns
 };

package/dist/esm/destinations/sqlFileDestination.js

@@ -66,7 +66,7 @@ function SQLFileDestination(options) {
             this.connection = connection;
         }.bind(writable),
         getConnectionName: function () {
-            return this.connection && this.connection.name;
+            return this.connection?.name;
         }.bind(writable),
         setTasks: function (tasks) {
             this.tasks = tasks;

package/dist/esm/lib/index.js

@@ -5,6 +5,7 @@ const EventEmitter = require('node:events');
 const util = require('node:util');
 const { RowMetaData } = require("./rowMetaData");
 const { compose } = require("node:stream");
+const { Destinations } = require('..');
 /**
  * @typedef {Object} Etl
  * @function loader
@@ -20,7 +21,8 @@ const { compose } = require("node:stream");
  */
 function Etl() {
     EventEmitter.call(this);
-    let self = this;
+    const self = this;
+    self.store = [];
     self.beforeETLList = [];
     self.connectionList = [];
     self.validatorList = [];
@@ -43,11 +45,12 @@ function Etl() {
     };
     this.beforeETL = (pipelineTask) => {
         const connectionname = pipelineTask.getConnectionName();
-        const connection = this.connectionList.filter(c => c.name === connectionname);
+        const connection = this.connectionList.filter(c => c.name === connectionname)[0];
         if (!connection) {
             throw new Error(`Connection with name ${connectionname} not found`);
         }
         pipelineTask.setConnection(connection);
+        pipelineTask.setETL(self);
         self.beforeETLList.push(pipelineTask);
         return self;
     };
@@ -62,14 +65,22 @@ function Etl() {
     this.destination = (destination, ...tasks) => {
         const connectionname = destination.getConnectionName();
         const connection = this.connectionList.filter(c => c.name === connectionname)[0];
-        if (connection) {
+        if (!connection && destination.type === 'PostgresDestination') {
+            throw new Error(`No connection could be found with name ${connectionname}`);
+        }
+        else {
             destination.setConnection(connection);
         }
-        tasks && destination.setTasks(tasks);
+        if (tasks) {
+            for (const task of tasks) {
+                task.setETL(self);
+            }
+            destination.setTasks(tasks);
+        }
         self.destinationList.push(destination);
         return self;
     };
-    this.transform = (transform, ...tasks) => {
+    this.transform = (transform) => {
         self.transformationList.push(transform);
         return self;
     };

package/dist/esm/loaders/csvloader.js

@@ -4,21 +4,30 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const fs = require("fs");
 const { Transform } = require("stream");
 const { parse } = require("csv-parse");
-const { stdout, stderr } = require("process");
 /**
  *
  * @param {Object} options
- * @param {String} options.path
+ * @param {String} [options.path]
+ * @param {ReadableStream} [options.stream]
+ * @param {Number} [options.startingLine]
+ * @param {Boolean} [options.relax]
  * @returns StreamReader
  */
 function CSVLoader(options) {
-    let csvLoader = fs.createReadStream(options.path);
+    let csvLoader;
+    if (options.path) {
+        csvLoader = fs.createReadStream(options.path);
+    }
+    else {
+        csvLoader = options.stream;
+    }
     let lineCount = 1;
+    const fromLine = options.startingLine ?? 2;
+    const relaxQuotes = options.relax ?? false;
     csvLoader._columns = options.columns;
     csvLoader.pump = (csvLoader) => {
-        return csvLoader
-            .pipe(parse({ delimiter: ",", from_line: 2 }))
-            .pipe(new Transform({
+        const parser = parse({ delimiter: ",", from_line: fromLine, relax_quotes: relaxQuotes });
+        const transformer = new Transform({
             readableObjectMode: true,
             writableObjectMode: true,
             emitClose: true,
@@ -26,9 +35,18 @@ function CSVLoader(options) {
                 chunk["_columns"] = options.columns;
                 chunk["_linecount"] = lineCount;
                 lineCount += 1;
+                // remove non-printable characters
+                options.columns.forEach((_column, index) => {
+                    if (chunk[index]) {
+                        chunk[index] = chunk[index].replace(/[\x00-\x1F\x7F-\x9F]/g, '');
+                    }
+                });
                 callback(null, chunk);
             }
-        }));
+        });
+        return csvLoader
+            .pipe(parser)
+            .pipe(transformer);
     };
     return csvLoader;
 }

package/dist/esm/types/database_connections/index.d.ts

@@ -1,3 +1,4 @@
-export { PostgresDatabaseConnection };
 import { PostgresDatabaseConnection } from "./postgresDatabaseConnection";
+import { ProvidedConnection } from "./providedConnection";
+export { PostgresDatabaseConnection, ProvidedConnection };
 //# sourceMappingURL=index.d.ts.map

package/dist/esm/types/database_connections/providedConnection.d.ts

@@ -0,0 +1,15 @@
+/**
+ *
+ * @param {Object} options
+ * @param {Object} options.connectionname
+ * @param {Object} options.sequelize
+ * @returns Connection
+ */
+export function ProvidedConnection(options: {
+    connectionname: any;
+    sequelize: any;
+}): Promise<{
+    name: any;
+    db: any;
+}>;
+//# sourceMappingURL=providedConnection.d.ts.map

package/dist/esm/types/database_connections/providedConnection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"providedConnection.d.ts","sourceRoot":"","sources":["../../../../src/database_connections/providedConnection.js"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH;IAJ2B,cAAc;IACd,SAAS;;;;GAUnC"}

package/dist/esm/types/destinations/postgresDestination.d.ts

@@ -5,6 +5,8 @@
  * @param {Object} options.connectionname
  * @param {Object} options.mapping
  * @param {Object} options.includeErrors
+ * @param {String} [options.schema]
+ * @param {Array<String>} [options.ignoredColumns]
  * @returns Transform
  */
 export function PostgresDestination(options: {
@@ -12,9 +14,13 @@ export function PostgresDestination(options: {
     connectionname: any;
     mapping: any;
     includeErrors: any;
+    schema?: string;
+    ignoredColumns?: Array<string>;
 }): Transform;
-export function writeInsertStatement(columnMapping: any, table: any, chunk: any): string;
+export function writeInsertStatement(columnMapping: any, table: any, chunk: any, schema: any, ignoredColumns?: any[]): string;
 export function isKeyWord(column: any): boolean;
 export function getMappingForColumn(mapping: any, column: any): any;
+export function hasReturningColumns(mapping: any): boolean;
+export function getReturningColumns(mapping: any): any;
 import { Transform } from "stream";
 //# sourceMappingURL=postgresDestination.d.ts.map

package/dist/esm/types/destinations/postgresDestination.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AA6CA;;;;;;;;GAQG;AACH;IAN2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;cA+DvC;AAnGD,yFA4BC;AArCD,gDAEC;AAED,oEAGC"}
+{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAyDA;;;;;;;;;;GAUG;AACH;IAR2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;IACZ,MAAM;IACC,cAAc,GAAtC,aAAa;cAkEvB;AA5GD,8HAgCC;AAlDD,gDAEC;AAED,oEAIC;AAED,2DAEC;AAED,uDAEC"}

package/dist/esm/types/lib/index.d.ts

@@ -14,6 +14,7 @@ export type Etl = any;
  */
 export function Etl(): void;
 export class Etl {
+    store: any[];
     beforeETLList: any[];
     connectionList: any[];
     validatorList: any[];
@@ -25,6 +26,6 @@ export class Etl {
     connection: (connection: any) => Etl;
     validator: (validator: any) => Etl;
     destination: (destination: any, ...tasks: any[]) => Etl;
-    transform: (transform: any, ...tasks: any[]) => Etl;
+    transform: (transform: any) => Etl;
 }
 //# sourceMappingURL=index.d.ts.map

package/dist/esm/types/lib/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAMA;;;;;;;;GAQG;AAEH;;;GAGG;AACH,4BAwEC;;IArEG,qBAAuB;IACvB,sBAAwB;IACxB,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,6BAGC;IAED,gBAgBC;IAED,sCASC;IAED,qCAGC;IAED,mCAGC;IAED,wDAWC;IAED,oDAGC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAOA;;;;;;;;GAQG;AAEH;;;GAGG;AACH,4BAgFC;;IA7EG,aAAe;IACf,qBAAuB;IACvB,sBAAwB;IACxB,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,6BAGC;IAED,gBAgBC;IAED,sCAUC;IAED,qCAGC;IAED,mCAGC;IAED,wDAiBC;IAED,mCAGC"}

package/dist/esm/types/loaders/csvloader.d.ts

@@ -2,11 +2,17 @@
 /**
  *
  * @param {Object} options
- * @param {String} options.path
+ * @param {String} [options.path]
+ * @param {ReadableStream} [options.stream]
+ * @param {Number} [options.startingLine]
+ * @param {Boolean} [options.relax]
  * @returns StreamReader
  */
 export function CSVLoader(options: {
-    path: string;
-}): fs.ReadStream;
+    path?: string;
+    stream?: ReadableStream;
+    startingLine?: number;
+    relax?: boolean;
+}): ReadableStream<any> | fs.ReadStream;
 import fs = require("fs");
 //# sourceMappingURL=csvloader.d.ts.map

package/dist/esm/types/loaders/csvloader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csvloader.d.ts","sourceRoot":"","sources":["../../../../src/loaders/csvloader.js"],"names":[],"mappings":";AAMA;;;;;GAKG;AACH;IAH2B,IAAI;kBAwB9B"}
+{"version":3,"file":"csvloader.d.ts","sourceRoot":"","sources":["../../../../src/loaders/csvloader.js"],"names":[],"mappings":";AAKA;;;;;;;;GAQG;AACH;IAN4B,IAAI;IACI,MAAM,GAA/B,cAAc;IACG,YAAY;IACX,KAAK;wCAyCjC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ffc-pay-etl-framework",
-  "version": "1.1.14-alpha.9",
+  "version": "1.2.0",
   "publisher": "Defra",
   "main": "dist/cjs/index.js",
   "private": false,
@@ -13,12 +13,17 @@
     "build:esm": "tsc -p ./tsconfig.esm.json && mv dist/esm/index.js dist/esm/index.mjs",
     "build:cjs": "tsc -p ./tsconfig.cjs.json",
     "prepack": "npm run build",
-    "example": "node ./src/examples/csv-multi-tool-validator.js"
+    "example": "node ./src/examples/sql-return-values.js"
   },
   "keywords": [
     "ETL"
   ],
-  "author": "Charlie Benger-Stevenson",
+  "author": "Defra",
+  "contributors": [
+    "Charlie Benger-Stevenson charlie.benger-stevenson.external@eviden.com",
+    "Leigh Godson leigh.godson@eviden.com",
+    "Sam Plackett samuel.plackett@eviden.com"
+  ],
   "license": "OGL-UK-3.0",
   "repository": "https://github.com/DEFRA/ffc-pay-etl-framework",
   "description": "A framework for creating ETL pipelines in node",