ffc-pay-etl-framework 0.0.1

package/LICENSE.md

@@ -0,0 +1,24 @@
+Defra - Copyright (c) 2024
+
+This software consists of voluntary contributions made by many individuals.
+For exact contribution history, see the revision history
+available at https://github.com/DEFRA/ffc-pay-etl-framework
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,55 @@
+## 🚀 Features
+- 💾 CSV Source and Destination
+- 👮 Validators
+- 🤖 Transformers
+- 🚨 Error Checking
+- 🐘 Write directly to Postgres
+- 👨‍⚕️ Intellisense 
+
+## 📦 Install
+```bash
+npm install --save-dev ffc-pay-etl-framework
+```
+
+## 🪄 Usage
+
+```js
+// ESM
+import { Etl, Loaders, Validators, Transformers, Destinations } from "ffc-pay-etl-framework"
+
+// CJS
+const { Etl, Loaders, Validators, Transformers, Destinations } = require("ffc-pay-etl-framework")
+
+let csvFile = `${process.cwd()}/test/fixtures/SoilType.csv`
+
+const etl = new Etl.Etl()
+
+etl
+.loader(Loaders.CSVLoader({path: csvFile, columns: columns}))
+.transform(Transformers.FakerTransformer({
+    columns: [{
+        name: "Dist Name",
+        faker: "location.city"
+    }]
+}))
+.destination(Destinations.CSVFileDestination({ 
+    fileName: "SoilType_Output.csv", 
+    headers: true, 
+    includeErrors: false, 
+    quotationMarks: true
+}))
+.pump()
+```
+
+## 📢 Shout outs
+
+[Faker JS](https://fakerjs.dev/guide/)
+
+[Validator](https://github.com/validatorjs/validator.js)
+
+[Kaggle](https://www.kaggle.com/datasets/anushkahedaoo/farming-factors)
+
+## ✨ Contributing
+
+Please make sure to read the [Contributing Guide](https://github.com/DEFRA/ffc-pay-etl-framework/blob/next/CONTRIBUTING.md) before making a pull request.
+

package/dist/cjs/destinations/consoleDestination.js

@@ -0,0 +1,21 @@
+const { Writable } = require("node:stream");
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.includeErrors
+ * @returns Writable
+ */
+function ConsoleDestination(options) {
+    const includeErrors = options.includeErrors;
+    return new Writable({
+        objectMode: true,
+        write(chunk, _, callback) {
+            if (chunk.errors.length === 0 || includeErrors)
+                console.log(chunk);
+            callback();
+        }
+    });
+}
+module.exports = {
+    ConsoleDestination
+};

package/dist/cjs/destinations/csvFileDestination.js

@@ -0,0 +1,48 @@
+const { Writable } = require("node:stream");
+const fs = require("fs");
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.fileName
+ * @param {Boolean} options.headers
+ * @param {Boolean} options.includeErrors
+ * @param {Boolean} options.quotationMarks
+ * @returns Writable
+ */
+function CSVFileDestination(options) {
+    const fileName = options.fileName;
+    const headers = options.headers;
+    const includeErrors = options.includeErrors;
+    const quotationMarks = options.quotationMarks;
+    let fileHandle;
+    fs.open(fileName, 'w+', ((error, fd) => {
+        fileHandle = fd;
+    }));
+    let headersWritten = false;
+    return new Writable({
+        objectMode: true,
+        write(chunk, _, callback) {
+            if (!headersWritten && headers) {
+                if (quotationMarks) {
+                    fs.writeFileSync(fileHandle, `${chunk._columns.map(c => `"${c}"`).join(",")}\n`);
+                }
+                else {
+                    fs.writeFileSync(fileHandle, `${chunk._columns.join(",")}\n`);
+                }
+                headersWritten = true;
+            }
+            if (chunk.errors.length === 0 || includeErrors) {
+                if (quotationMarks) {
+                    fs.writeFileSync(fileHandle, `${chunk.map(c => `"${c}"`).join(",")}\n`);
+                }
+                else {
+                    fs.writeFileSync(fileHandle, `${chunk.map(c => `"${c}"`).join(",")}\n`);
+                }
+            }
+            callback();
+        }
+    });
+}
+module.exports = {
+    CSVFileDestination
+};

package/dist/cjs/destinations/index.js

@@ -0,0 +1,11 @@
+const { ConsoleDestination } = require("./consoleDestination");
+const { SQLFileDestination, SQL_MODE } = require("./sqlFileDestination");
+const { CSVFileDestination } = require("./csvFileDestination");
+const { PostgresDestination } = require("./postgresDestination");
+module.exports = {
+    ConsoleDestination,
+    SQLFileDestination,
+    SQL_MODE,
+    CSVFileDestination,
+    PostgresDestination
+};

package/dist/cjs/destinations/postgresDestination.js

@@ -0,0 +1,67 @@
+const { Transform } = require("node:stream");
+const { Sequelize } = require('sequelize');
+/**
+ *
+ * @param {Object} options
+ * @param {Object} options.table
+ * @param {Object} options.username
+ * @param {Object} options.password
+ * @param {Object} options.database
+ * @param {Object} options.host
+ * @param {Object} options.mapping
+ * @param {Object} options.includeErrors
+ * @returns Transform
+ */
+function PostgresDestination(options) {
+    const table = options.table;
+    const username = options.username;
+    const password = options.password;
+    const database = options.database;
+    const host = options.host;
+    const mapping = options.mapping;
+    const includeErrors = options.includeErrors;
+    const sequelize = new Sequelize(database, username, password, {
+        host: host,
+        dialect: 'postgres',
+        logging: false
+    });
+    sequelize.authenticate();
+    function getMappingForColumn(column) {
+        const [map] = mapping.filter(m => m.column === column);
+        return map;
+    }
+    function writeInsertStatement(chunk) {
+        let statement = `INSERT INTO ${table} (${chunk._columns.map(column => {
+            let mapping = getMappingForColumn(column);
+            return mapping ? mapping.targetColumn : column;
+        })
+            .join(",")}) VALUES (${chunk._columns.map((column, index) => {
+            let mapping = getMappingForColumn(column);
+            if (mapping.targetType === "varchar" || mapping.targetType === "char")
+                return `'${chunk[index]}'`;
+            return chunk[index];
+        })})`;
+        return statement;
+    }
+    return new Transform({
+        objectMode: true,
+        transform(chunk, _, callback) {
+            let insertStatement;
+            // @ts-ignore
+            if (chunk.errors.length === 0 | options.includeErrors) {
+                insertStatement = writeInsertStatement(chunk);
+                sequelize.query(insertStatement)
+                    .then(result => {
+                    chunk._result = result;
+                    callback(null, chunk);
+                }).catch(error => {
+                    chunk.errors.push(error);
+                    callback(null, chunk);
+                });
+            }
+        }
+    });
+}
+module.exports = {
+    PostgresDestination
+};

package/dist/cjs/destinations/sqlFileDestination.js

@@ -0,0 +1,64 @@
+const { Writable } = require("node:stream");
+const fs = require("fs");
+/**
+ * @enum {number}
+ */
+const SQL_MODE = {
+    "INSERT_MODE": 1,
+    "UPDATE_MODE": 2
+};
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.fileName
+ * @param {SQL_MODE} options.mode
+ * @param {String} options.table
+ * @param {Array} options.mapping
+ * @param {Boolean} options.includeErrors
+ * @returns Transform
+ */
+function SQLFileDestination(options) {
+    const fileName = options.fileName;
+    const sqlMode = options.mode;
+    const table = options.table;
+    const mapping = options.mapping;
+    const includeErrors = options.includeErrors;
+    function writeInsertStatement(chunk) {
+        let statement = `INSERT INTO ${table} (${mapping.map(m => m.targetColumn)
+            .join(",")}) VALUES (${mapping.map((m) => {
+            const srcColumnIndex = chunk._columns.indexOf(m.column);
+            if (m.targetType === "string")
+                return `'${chunk[srcColumnIndex]}'`;
+            return chunk[srcColumnIndex];
+        })});\n`;
+        fs.writeFileSync(fileName, statement, {
+            encoding: "utf8",
+            flag: "a+"
+        });
+    }
+    function writeUpdateStatement(chunk) {
+    }
+    return new Writable({
+        objectMode: true,
+        write(chunk, _, callback) {
+            if (chunk.errors.length === 0)
+                if (sqlMode === SQL_MODE.INSERT_MODE) {
+                    // @ts-ignore
+                    if (chunk.errors.length === 0 | includeErrors) {
+                        writeInsertStatement(chunk);
+                    }
+                }
+                else if (sqlMode === SQL_MODE.UPDATE_MODE) {
+                    // @ts-ignore
+                    if (chunk.errors.length === 0 | includeErrors) {
+                        writeUpdateStatement(chunk);
+                    }
+                }
+            callback();
+        }
+    });
+}
+module.exports = {
+    SQLFileDestination,
+    SQL_MODE
+};

package/dist/cjs/index.js

@@ -0,0 +1,12 @@
+const Validators = require("./validators");
+const Transformers = require("./transformers");
+const Loaders = require("./loaders");
+const Destinations = require("./destinations");
+const Etl = require("./lib");
+module.exports = {
+    Etl,
+    Validators,
+    Transformers,
+    Loaders,
+    Destinations,
+};

package/dist/cjs/lib/index.js

@@ -0,0 +1,46 @@
+// @ts-nocheck
+const { RowMetaData } = require("./rowMetaData");
+const { compose } = require("node:stream");
+/**
+ * @typedef {Object} Etl
+ * @function loader
+ * @function pump
+ * @function validator
+ * @function destination
+ * @function transform
+ */
+/**
+ *
+ * @returns Etl
+ */
+function Etl() {
+    let self = this;
+    self.validatorList = [];
+    self.transformationList = [];
+    self.destinationList = [];
+    this.loader = (loader) => {
+        self.loader = loader;
+        return self;
+    };
+    this.pump = () => {
+        this.loader
+            .pump(this.loader)
+            .pipe(compose(RowMetaData, ...self.validatorList, ...self.transformationList, ...self.destinationList));
+    };
+    this.validator = (validator) => {
+        self.validatorList.push(validator);
+        return self;
+    };
+    this.destination = (destination) => {
+        self.destinationList.push(destination);
+        return self;
+    };
+    this.transform = (transform) => {
+        self.transformationList.push(transform);
+        return self;
+    };
+    return;
+}
+module.exports = {
+    Etl
+};

package/dist/cjs/lib/rowMetaData.js

@@ -0,0 +1,23 @@
+// @ts-nocheck
+const { Transform } = require("stream");
+const util = require("util");
+function RowMetaData(options) {
+    return new Transform({
+        readableObjectMode: true,
+        writableObjectMode: true,
+        decodeStrings: false,
+        construct(callback) {
+            this.rowId = 0;
+            callback();
+        },
+        transform(chunk, _, callback) {
+            chunk['_rowId'] = this.rowId;
+            chunk['errors'] = [];
+            this.rowId += 1;
+            callback(null, chunk);
+        }
+    });
+}
+module.exports = {
+    RowMetaData
+};

package/dist/cjs/loaders/csvloader.js

@@ -0,0 +1,31 @@
+// @ts-nocheck
+const fs = require("fs");
+const { Transform } = require("stream");
+const { parse } = require("csv-parse");
+const { stdout } = require("process");
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.path
+ * @returns StreamReader
+ */
+function CSVLoader(options) {
+    let csvLoader = fs.createReadStream(options.path);
+    csvLoader._columns = options.columns;
+    csvLoader.pump = (csvLoader) => {
+        return csvLoader
+            .pipe(parse({ delimiter: ",", from_line: 2 }))
+            .pipe(new Transform({
+            readableObjectMode: true,
+            writableObjectMode: true,
+            transform(chunk, _, callback) {
+                chunk["_columns"] = options.columns;
+                callback(null, chunk);
+            }
+        }));
+    };
+    return csvLoader;
+}
+module.exports = {
+    CSVLoader
+};

package/dist/cjs/loaders/index.js

@@ -0,0 +1,4 @@
+const { CSVLoader } = require("./csvloader");
+module.exports = {
+    CSVLoader
+};

package/dist/cjs/transformers/fakerTransformer.js

@@ -0,0 +1,39 @@
+const { Transform } = require("node:stream");
+const { faker } = require("@faker-js/faker");
+/**
+ *
+ * @param {Object} options
+ * @param {Array} options.columns {
+ *           columns: [
+ *               {
+ *                   name: "column-name",
+ *                   faker: "company.name"
+ *               }
+ *           ]
+ *       }
+ * @returns StreamReader
+ */
+function FakerTransformer(options) {
+    let self = this;
+    self.columns = options.columns;
+    function getFaker(fakerType) {
+        return fakerType.split('.').reduce((a, b) => {
+            return Object.keys(a).length === 0 ? faker[b] : a[b];
+        }, {});
+    }
+    return new Transform({
+        readableObjectMode: true,
+        writableObjectMode: true,
+        transform(chunk, _, callback) {
+            const { _columns } = chunk;
+            self.columns.forEach(column => {
+                const colIndex = _columns.indexOf(column.name);
+                chunk[colIndex] = getFaker(column.faker)();
+            });
+            callback(null, chunk);
+        }
+    });
+}
+module.exports = {
+    FakerTransformer
+};

package/dist/cjs/transformers/index.js

@@ -0,0 +1,6 @@
+const { ToUpperCaseTransformer } = require("./toUpperCaseTransformer");
+const { FakerTransformer } = require("./fakerTransformer");
+module.exports = {
+    ToUpperCaseTransformer,
+    FakerTransformer
+};

package/dist/cjs/transformers/toUpperCaseTransformer.js

@@ -0,0 +1,24 @@
+const { Transform } = require("node:stream");
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.column
+ * @returns Writable
+ */
+function ToUpperCaseTransformer(options) {
+    let self = this;
+    self.column = options.column;
+    return new Transform({
+        readableObjectMode: true,
+        writableObjectMode: true,
+        transform(chunk, _, callback) {
+            const { _columns } = chunk;
+            const colIndex = _columns.indexOf(self.column);
+            chunk[colIndex] = chunk[colIndex].toUpperCase();
+            callback(null, chunk);
+        }
+    });
+}
+module.exports = {
+    ToUpperCaseTransformer
+};

package/dist/cjs/types/destinations/consoleDestination.d.ts

@@ -0,0 +1,11 @@
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.includeErrors
+ * @returns Writable
+ */
+export function ConsoleDestination(options: {
+    includeErrors: string;
+}): Writable;
+import { Writable } from "stream";
+//# sourceMappingURL=consoleDestination.d.ts.map

package/dist/cjs/types/destinations/consoleDestination.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"consoleDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/consoleDestination.js"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,4CAHG;IAAwB,aAAa;CACrC,YAYF"}

package/dist/cjs/types/destinations/csvFileDestination.d.ts

@@ -0,0 +1,17 @@
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.fileName
+ * @param {Boolean} options.headers
+ * @param {Boolean} options.includeErrors
+ * @param {Boolean} options.quotationMarks
+ * @returns Writable
+ */
+export function CSVFileDestination(options: {
+    fileName: string;
+    headers: boolean;
+    includeErrors: boolean;
+    quotationMarks: boolean;
+}): Writable;
+import { Writable } from "stream";
+//# sourceMappingURL=csvFileDestination.d.ts.map

package/dist/cjs/types/destinations/csvFileDestination.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csvFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/csvFileDestination.js"],"names":[],"mappings":"AAGA;;;;;;;;GAQG;AACH,4CANG;IAAwB,QAAQ;IACP,OAAO;IACP,aAAa;IACb,cAAc;CACvC,YAkCF"}

package/dist/cjs/types/destinations/index.d.ts

@@ -0,0 +1,7 @@
+import { ConsoleDestination } from "./consoleDestination";
+import { SQLFileDestination } from "./sqlFileDestination";
+import { SQL_MODE } from "./sqlFileDestination";
+import { CSVFileDestination } from "./csvFileDestination";
+import { PostgresDestination } from "./postgresDestination";
+export { ConsoleDestination, SQLFileDestination, SQL_MODE, CSVFileDestination, PostgresDestination };
+//# sourceMappingURL=index.d.ts.map

package/dist/cjs/types/destinations/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/destinations/index.js"],"names":[],"mappings":""}

package/dist/cjs/types/destinations/postgresDestination.d.ts

@@ -0,0 +1,23 @@
+/**
+ *
+ * @param {Object} options
+ * @param {Object} options.table
+ * @param {Object} options.username
+ * @param {Object} options.password
+ * @param {Object} options.database
+ * @param {Object} options.host
+ * @param {Object} options.mapping
+ * @param {Object} options.includeErrors
+ * @returns Transform
+ */
+export function PostgresDestination(options: {
+    table: any;
+    username: any;
+    password: any;
+    database: any;
+    host: any;
+    mapping: any;
+    includeErrors: any;
+}): Transform;
+import { Transform } from "stream";
+//# sourceMappingURL=postgresDestination.d.ts.map

package/dist/cjs/types/destinations/postgresDestination.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAGA;;;;;;;;;;;GAWG;AACH,6CATG;IAAwB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,OAAO;IACP,aAAa;CACrC,aAuDF"}

package/dist/cjs/types/destinations/sqlFileDestination.d.ts

@@ -0,0 +1,24 @@
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.fileName
+ * @param {SQL_MODE} options.mode
+ * @param {String} options.table
+ * @param {Array} options.mapping
+ * @param {Boolean} options.includeErrors
+ * @returns Transform
+ */
+export function SQLFileDestination(options: {
+    fileName: string;
+    mode: SQL_MODE;
+    table: string;
+    mapping: any[];
+    includeErrors: boolean;
+}): Writable;
+export type SQL_MODE = number;
+export namespace SQL_MODE {
+    let INSERT_MODE: number;
+    let UPDATE_MODE: number;
+}
+import { Writable } from "stream";
+//# sourceMappingURL=sqlFileDestination.d.ts.map

package/dist/cjs/types/destinations/sqlFileDestination.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sqlFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/sqlFileDestination.js"],"names":[],"mappings":"AAWA;;;;;;;;;GASG;AACH,4CAPG;IAAwB,QAAQ;IACN,IAAI,EAAtB,QAAQ;IACQ,KAAK;IACN,OAAO;IACL,aAAa;CACtC,YA4CF;uBA3DS,MAAM"}

package/dist/cjs/types/index.d.ts

@@ -0,0 +1,7 @@
+import Etl = require("./lib");
+import Validators = require("./validators");
+import Transformers = require("./transformers");
+import Loaders = require("./loaders");
+import Destinations = require("./destinations");
+export { Etl, Validators, Transformers, Loaders, Destinations };
+//# sourceMappingURL=index.d.ts.map

package/dist/cjs/types/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/index.js"],"names":[],"mappings":""}

package/dist/cjs/types/lib/index.d.ts

@@ -0,0 +1,25 @@
+export type Etl = any;
+/**
+ * @typedef {Object} Etl
+ * @function loader
+ * @function pump
+ * @function validator
+ * @function destination
+ * @function transform
+ */
+/**
+ *
+ * @returns Etl
+ */
+export function Etl(): void;
+export class Etl {
+    validatorList: any[];
+    transformationList: any[];
+    destinationList: any[];
+    loader: (loader: any) => this;
+    pump: () => void;
+    validator: (validator: any) => this;
+    destination: (destination: any) => this;
+    transform: (transform: any) => this;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/cjs/types/lib/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAIA;;;;;;;GAOG;AAEH;;;GAGG;AACH,4BAwCC;;IAtCG,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,8BAGC;IAED,iBAWC;IAED,oCAGC;IAED,wCAGC;IAED,oCAGC"}

package/dist/cjs/types/lib/rowMetaData.d.ts

@@ -0,0 +1,3 @@
+export function RowMetaData(options: any): Transform;
+import { Transform } from "stream";
+//# sourceMappingURL=rowMetaData.d.ts.map

package/dist/cjs/types/lib/rowMetaData.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rowMetaData.d.ts","sourceRoot":"","sources":["../../../../src/lib/rowMetaData.js"],"names":[],"mappings":"AAIA,qDAgBC"}

package/dist/cjs/types/loaders/csvloader.d.ts

@@ -0,0 +1,11 @@
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.path
+ * @returns StreamReader
+ */
+export function CSVLoader(options: {
+    path: string;
+}): fs.ReadStream;
+import fs = require("fs");
+//# sourceMappingURL=csvloader.d.ts.map

package/dist/cjs/types/loaders/csvloader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csvloader.d.ts","sourceRoot":"","sources":["../../../../src/loaders/csvloader.js"],"names":[],"mappings":"AAMA;;;;;GAKG;AACH,mCAHG;IAAwB,IAAI;CAC5B,iBAmBF"}

package/dist/cjs/types/loaders/index.d.ts

@@ -0,0 +1,3 @@
+export { CSVLoader };
+import { CSVLoader } from "./csvloader";
+//# sourceMappingURL=index.d.ts.map