ffc-pay-etl-framework 0.0.1

package/dist/cjs/types/loaders/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/loaders/index.js"],"names":[],"mappings":""}

package/dist/cjs/types/transformers/fakerTransformer.d.ts

@@ -0,0 +1,37 @@
+/**
+ *
+ * @param {Object} options
+ * @param {Array} options.columns {
+ *           columns: [
+ *               {
+ *                   name: "column-name",
+ *                   faker: "company.name"
+ *               }
+ *           ]
+ *       }
+ * @returns StreamReader
+ */
+export function FakerTransformer(options: {
+    columns: any[];
+}): Transform;
+export class FakerTransformer {
+    /**
+     *
+     * @param {Object} options
+     * @param {Array} options.columns {
+     *           columns: [
+     *               {
+     *                   name: "column-name",
+     *                   faker: "company.name"
+     *               }
+     *           ]
+     *       }
+     * @returns StreamReader
+     */
+    constructor(options: {
+        columns: any[];
+    });
+    columns: any[];
+}
+import { Transform } from "stream";
+//# sourceMappingURL=fakerTransformer.d.ts.map

package/dist/cjs/types/transformers/fakerTransformer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fakerTransformer.d.ts","sourceRoot":"","sources":["../../../../src/transformers/fakerTransformer.js"],"names":[],"mappings":"AAGA;;;;;;;;;;;;GAYG;AACH,0CAVG;IAAuB,OAAO;CAQ9B,aAsBF;;IAjCD;;;;;;;;;;;;OAYG;IACH,qBAVG;QAAuB,OAAO;KAQ9B,EAsBF;IAlBG,eAA8B"}

package/dist/cjs/types/transformers/index.d.ts

@@ -0,0 +1,4 @@
+import { ToUpperCaseTransformer } from "./toUpperCaseTransformer";
+import { FakerTransformer } from "./fakerTransformer";
+export { ToUpperCaseTransformer, FakerTransformer };
+//# sourceMappingURL=index.d.ts.map

package/dist/cjs/types/transformers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/transformers/index.js"],"names":[],"mappings":""}

package/dist/cjs/types/transformers/toUpperCaseTransformer.d.ts

@@ -0,0 +1,23 @@
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.column
+ * @returns Writable
+ */
+export function ToUpperCaseTransformer(options: {
+    column: string;
+}): Transform;
+export class ToUpperCaseTransformer {
+    /**
+     *
+     * @param {Object} options
+     * @param {String} options.column
+     * @returns Writable
+     */
+    constructor(options: {
+        column: string;
+    });
+    column: string;
+}
+import { Transform } from "stream";
+//# sourceMappingURL=toUpperCaseTransformer.d.ts.map

package/dist/cjs/types/transformers/toUpperCaseTransformer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"toUpperCaseTransformer.d.ts","sourceRoot":"","sources":["../../../../src/transformers/toUpperCaseTransformer.js"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,gDAHG;IAAwB,MAAM;CAC9B,aAgBF;;IApBD;;;;;OAKG;IACH,qBAHG;QAAwB,MAAM;KAC9B,EAgBF;IAZG,eAA4B"}

package/dist/cjs/types/validators/index.d.ts

@@ -0,0 +1,5 @@
+import { RequiredValidator } from "./requiredValidator";
+import { UniqueValidator } from "./uniqueValidator";
+import { MultiToolValidator } from "./multiToolValidator";
+export { RequiredValidator, UniqueValidator, MultiToolValidator };
+//# sourceMappingURL=index.d.ts.map

package/dist/cjs/types/validators/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/validators/index.js"],"names":[],"mappings":""}

package/dist/cjs/types/validators/multiToolValidator.d.ts

@@ -0,0 +1,23 @@
+/**
+ *
+ * @param {Object} options
+ * @param {Array} options.columns { columns: [
+ *   {
+ *       name: "column-name",
+ *       validator: "isAlphanumeric",
+ *       args: {
+ *               locale: "en-GB",
+ *               options: {
+ *                   ignore: "-;%/ "
+ *               }
+ *       }
+ *   }
+ *  ]
+ * }
+ * @returns Writable
+ */
+export function MultiToolValidator(options: {
+    columns: any[];
+}): Transform;
+import { Transform } from "stream";
+//# sourceMappingURL=multiToolValidator.d.ts.map

package/dist/cjs/types/validators/multiToolValidator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"multiToolValidator.d.ts","sourceRoot":"","sources":["../../../../src/validators/multiToolValidator.js"],"names":[],"mappings":"AAIA;;;;;;;;;;;;;;;;;GAiBG;AACH,4CAfG;IAAuB,OAAO;CAa9B,aAyBF"}

package/dist/cjs/types/validators/requiredValidator.d.ts

@@ -0,0 +1,13 @@
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.message
+ * @param {Array} options.columns
+ * @returns Transform
+ */
+export function RequiredValidator(options: {
+    message: string;
+    columns: any[];
+}): Transform;
+import { Transform } from "stream";
+//# sourceMappingURL=requiredValidator.d.ts.map

package/dist/cjs/types/validators/requiredValidator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"requiredValidator.d.ts","sourceRoot":"","sources":["../../../../src/validators/requiredValidator.js"],"names":[],"mappings":"AAGA;;;;;;GAMG;AACH,2CAJG;IAAwB,OAAO;IACR,OAAO;CAC9B,aAsBF"}

package/dist/cjs/types/validators/uniqueValidator.d.ts

@@ -0,0 +1,29 @@
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.message
+ * @param {String} options.column
+ * @returns Transform
+ */
+export function UniqueValidator(options: {
+    message: string;
+    column: string;
+}): Transform;
+export class UniqueValidator {
+    /**
+     *
+     * @param {Object} options
+     * @param {String} options.message
+     * @param {String} options.column
+     * @returns Transform
+     */
+    constructor(options: {
+        message: string;
+        column: string;
+    });
+    column: string;
+    message: string;
+    uniqueColumnRows: any[];
+}
+import { Transform } from "stream";
+//# sourceMappingURL=uniqueValidator.d.ts.map

package/dist/cjs/types/validators/uniqueValidator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"uniqueValidator.d.ts","sourceRoot":"","sources":["../../../../src/validators/uniqueValidator.js"],"names":[],"mappings":"AAEA;;;;;;GAMG;AACH,yCAJG;IAAwB,OAAO;IACP,MAAM;CAC9B,aA6BF;;IAlCD;;;;;;OAMG;IACH,qBAJG;QAAwB,OAAO;QACP,MAAM;KAC9B,EA6BF;IAzBG,eAA4B;IAC5B,gBAAwF;IACxF,wBAA0B"}

package/dist/cjs/validators/index.js

@@ -0,0 +1,8 @@
+const { RequiredValidator } = require("./requiredValidator");
+const { UniqueValidator } = require("./uniqueValidator");
+const { MultiToolValidator } = require("./multiToolValidator");
+module.exports = {
+    RequiredValidator,
+    UniqueValidator,
+    MultiToolValidator
+};

package/dist/cjs/validators/multiToolValidator.js

@@ -0,0 +1,48 @@
+// @ts-nocheck
+const { Transform } = require("stream");
+const validator = require('validator');
+/**
+ *
+ * @param {Object} options
+ * @param {Array} options.columns { columns: [
+ *   {
+ *       name: "column-name",
+ *       validator: "isAlphanumeric",
+ *       args: {
+ *               locale: "en-GB",
+ *               options: {
+ *                   ignore: "-;%/ "
+ *               }
+ *       }
+ *   }
+ *  ]
+ * }
+ * @returns Writable
+ */
+function MultiToolValidator(options) {
+    function getValidator(validatorType) {
+        return validator[validatorType];
+    }
+    return new Transform({
+        readableObjectMode: true,
+        writableObjectMode: true,
+        decodeStrings: false,
+        construct(callback) {
+            this.columns = options.columns;
+            callback();
+        },
+        transform(chunk, _, callback) {
+            const { _columns } = chunk;
+            this.columns.forEach(column => {
+                const colIndex = _columns.indexOf(column.name);
+                const vtr = getValidator(column.validator);
+                if (!vtr(chunk[colIndex], ...Object.values(column.args)))
+                    chunk.errors.push(`${column.validator} returned invalid status in column ${colIndex}`);
+            });
+            callback(null, chunk);
+        }
+    });
+}
+module.exports = {
+    MultiToolValidator
+};

package/dist/cjs/validators/requiredValidator.js

@@ -0,0 +1,33 @@
+// @ts-nocheck
+const { Transform } = require("stream");
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.message
+ * @param {Array} options.columns
+ * @returns Transform
+ */
+function RequiredValidator(options) {
+    return new Transform({
+        readableObjectMode: true,
+        writableObjectMode: true,
+        decodeStrings: false,
+        construct(callback) {
+            this.message = options.message ? options.message : "Required value is null";
+            this.requiredColumns = options.columns;
+            callback();
+        },
+        transform(chunk, _, callback) {
+            this.requiredColumns.forEach(column => {
+                let requiredVal = chunk[column];
+                if (requiredVal === undefined || requiredVal === '') {
+                    chunk.errors.push(`${this.message} in column ${column}`);
+                }
+            });
+            callback(null, chunk);
+        }
+    });
+}
+module.exports = {
+    RequiredValidator
+};

package/dist/cjs/validators/uniqueValidator.js

@@ -0,0 +1,40 @@
+const { Transform } = require("node:stream");
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.message
+ * @param {String} options.column
+ * @returns Transform
+ */
+function UniqueValidator(options) {
+    let self = this;
+    self.column = options.column;
+    self.message = options.message ? options.message : "Expected unique value is not unique";
+    self.uniqueColumnRows = [];
+    function checkIsUnique(data) {
+        return self.uniqueColumnRows.indexOf(data) === -1;
+    }
+    return new Transform({
+        readableObjectMode: true,
+        writableObjectMode: true,
+        decodeStrings: false,
+        construct(callback) {
+            callback();
+        },
+        transform(chunk, _, callback) {
+            const { _columns } = chunk;
+            const colIndex = _columns.indexOf(self.column);
+            const uniqueValue = chunk[colIndex];
+            if (checkIsUnique(uniqueValue)) {
+                self.uniqueColumnRows.push(uniqueValue);
+            }
+            else {
+                chunk.errors.push(`${self.message} ${self.column}`);
+            }
+            callback(null, chunk);
+        }
+    });
+}
+module.exports = {
+    UniqueValidator
+};

package/dist/esm/destinations/consoleDestination.js

@@ -0,0 +1,22 @@
+"use strict";
+const { Writable } = require("node:stream");
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.includeErrors
+ * @returns Writable
+ */
+function ConsoleDestination(options) {
+    const includeErrors = options.includeErrors;
+    return new Writable({
+        objectMode: true,
+        write(chunk, _, callback) {
+            if (chunk.errors.length === 0 || includeErrors)
+                console.log(chunk);
+            callback();
+        }
+    });
+}
+module.exports = {
+    ConsoleDestination
+};

package/dist/esm/destinations/csvFileDestination.js

@@ -0,0 +1,49 @@
+"use strict";
+const { Writable } = require("node:stream");
+const fs = require("fs");
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.fileName
+ * @param {Boolean} options.headers
+ * @param {Boolean} options.includeErrors
+ * @param {Boolean} options.quotationMarks
+ * @returns Writable
+ */
+function CSVFileDestination(options) {
+    const fileName = options.fileName;
+    const headers = options.headers;
+    const includeErrors = options.includeErrors;
+    const quotationMarks = options.quotationMarks;
+    let fileHandle;
+    fs.open(fileName, 'w+', ((error, fd) => {
+        fileHandle = fd;
+    }));
+    let headersWritten = false;
+    return new Writable({
+        objectMode: true,
+        write(chunk, _, callback) {
+            if (!headersWritten && headers) {
+                if (quotationMarks) {
+                    fs.writeFileSync(fileHandle, `${chunk._columns.map(c => `"${c}"`).join(",")}\n`);
+                }
+                else {
+                    fs.writeFileSync(fileHandle, `${chunk._columns.join(",")}\n`);
+                }
+                headersWritten = true;
+            }
+            if (chunk.errors.length === 0 || includeErrors) {
+                if (quotationMarks) {
+                    fs.writeFileSync(fileHandle, `${chunk.map(c => `"${c}"`).join(",")}\n`);
+                }
+                else {
+                    fs.writeFileSync(fileHandle, `${chunk.map(c => `"${c}"`).join(",")}\n`);
+                }
+            }
+            callback();
+        }
+    });
+}
+module.exports = {
+    CSVFileDestination
+};

package/dist/esm/destinations/index.js

@@ -0,0 +1,12 @@
+"use strict";
+const { ConsoleDestination } = require("./consoleDestination");
+const { SQLFileDestination, SQL_MODE } = require("./sqlFileDestination");
+const { CSVFileDestination } = require("./csvFileDestination");
+const { PostgresDestination } = require("./postgresDestination");
+module.exports = {
+    ConsoleDestination,
+    SQLFileDestination,
+    SQL_MODE,
+    CSVFileDestination,
+    PostgresDestination
+};

package/dist/esm/destinations/postgresDestination.js

@@ -0,0 +1,68 @@
+"use strict";
+const { Transform } = require("node:stream");
+const { Sequelize } = require('sequelize');
+/**
+ *
+ * @param {Object} options
+ * @param {Object} options.table
+ * @param {Object} options.username
+ * @param {Object} options.password
+ * @param {Object} options.database
+ * @param {Object} options.host
+ * @param {Object} options.mapping
+ * @param {Object} options.includeErrors
+ * @returns Transform
+ */
+function PostgresDestination(options) {
+    const table = options.table;
+    const username = options.username;
+    const password = options.password;
+    const database = options.database;
+    const host = options.host;
+    const mapping = options.mapping;
+    const includeErrors = options.includeErrors;
+    const sequelize = new Sequelize(database, username, password, {
+        host: host,
+        dialect: 'postgres',
+        logging: false
+    });
+    sequelize.authenticate();
+    function getMappingForColumn(column) {
+        const [map] = mapping.filter(m => m.column === column);
+        return map;
+    }
+    function writeInsertStatement(chunk) {
+        let statement = `INSERT INTO ${table} (${chunk._columns.map(column => {
+            let mapping = getMappingForColumn(column);
+            return mapping ? mapping.targetColumn : column;
+        })
+            .join(",")}) VALUES (${chunk._columns.map((column, index) => {
+            let mapping = getMappingForColumn(column);
+            if (mapping.targetType === "varchar" || mapping.targetType === "char")
+                return `'${chunk[index]}'`;
+            return chunk[index];
+        })})`;
+        return statement;
+    }
+    return new Transform({
+        objectMode: true,
+        transform(chunk, _, callback) {
+            let insertStatement;
+            // @ts-ignore
+            if (chunk.errors.length === 0 | options.includeErrors) {
+                insertStatement = writeInsertStatement(chunk);
+                sequelize.query(insertStatement)
+                    .then(result => {
+                    chunk._result = result;
+                    callback(null, chunk);
+                }).catch(error => {
+                    chunk.errors.push(error);
+                    callback(null, chunk);
+                });
+            }
+        }
+    });
+}
+module.exports = {
+    PostgresDestination
+};

package/dist/esm/destinations/sqlFileDestination.js

@@ -0,0 +1,65 @@
+"use strict";
+const { Writable } = require("node:stream");
+const fs = require("fs");
+/**
+ * @enum {number}
+ */
+const SQL_MODE = {
+    "INSERT_MODE": 1,
+    "UPDATE_MODE": 2
+};
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.fileName
+ * @param {SQL_MODE} options.mode
+ * @param {String} options.table
+ * @param {Array} options.mapping
+ * @param {Boolean} options.includeErrors
+ * @returns Transform
+ */
+function SQLFileDestination(options) {
+    const fileName = options.fileName;
+    const sqlMode = options.mode;
+    const table = options.table;
+    const mapping = options.mapping;
+    const includeErrors = options.includeErrors;
+    function writeInsertStatement(chunk) {
+        let statement = `INSERT INTO ${table} (${mapping.map(m => m.targetColumn)
+            .join(",")}) VALUES (${mapping.map((m) => {
+            const srcColumnIndex = chunk._columns.indexOf(m.column);
+            if (m.targetType === "string")
+                return `'${chunk[srcColumnIndex]}'`;
+            return chunk[srcColumnIndex];
+        })});\n`;
+        fs.writeFileSync(fileName, statement, {
+            encoding: "utf8",
+            flag: "a+"
+        });
+    }
+    function writeUpdateStatement(chunk) {
+    }
+    return new Writable({
+        objectMode: true,
+        write(chunk, _, callback) {
+            if (chunk.errors.length === 0)
+                if (sqlMode === SQL_MODE.INSERT_MODE) {
+                    // @ts-ignore
+                    if (chunk.errors.length === 0 | includeErrors) {
+                        writeInsertStatement(chunk);
+                    }
+                }
+                else if (sqlMode === SQL_MODE.UPDATE_MODE) {
+                    // @ts-ignore
+                    if (chunk.errors.length === 0 | includeErrors) {
+                        writeUpdateStatement(chunk);
+                    }
+                }
+            callback();
+        }
+    });
+}
+module.exports = {
+    SQLFileDestination,
+    SQL_MODE
+};

package/dist/esm/index.mjs

@@ -0,0 +1,13 @@
+"use strict";
+const Validators = require("./validators");
+const Transformers = require("./transformers");
+const Loaders = require("./loaders");
+const Destinations = require("./destinations");
+const Etl = require("./lib");
+module.exports = {
+    Etl,
+    Validators,
+    Transformers,
+    Loaders,
+    Destinations,
+};

package/dist/esm/lib/index.js

@@ -0,0 +1,47 @@
+"use strict";
+// @ts-nocheck
+const { RowMetaData } = require("./rowMetaData");
+const { compose } = require("node:stream");
+/**
+ * @typedef {Object} Etl
+ * @function loader
+ * @function pump
+ * @function validator
+ * @function destination
+ * @function transform
+ */
+/**
+ *
+ * @returns Etl
+ */
+function Etl() {
+    let self = this;
+    self.validatorList = [];
+    self.transformationList = [];
+    self.destinationList = [];
+    this.loader = (loader) => {
+        self.loader = loader;
+        return self;
+    };
+    this.pump = () => {
+        this.loader
+            .pump(this.loader)
+            .pipe(compose(RowMetaData, ...self.validatorList, ...self.transformationList, ...self.destinationList));
+    };
+    this.validator = (validator) => {
+        self.validatorList.push(validator);
+        return self;
+    };
+    this.destination = (destination) => {
+        self.destinationList.push(destination);
+        return self;
+    };
+    this.transform = (transform) => {
+        self.transformationList.push(transform);
+        return self;
+    };
+    return;
+}
+module.exports = {
+    Etl
+};

package/dist/esm/lib/rowMetaData.js

@@ -0,0 +1,24 @@
+"use strict";
+// @ts-nocheck
+const { Transform } = require("stream");
+const util = require("util");
+function RowMetaData(options) {
+    return new Transform({
+        readableObjectMode: true,
+        writableObjectMode: true,
+        decodeStrings: false,
+        construct(callback) {
+            this.rowId = 0;
+            callback();
+        },
+        transform(chunk, _, callback) {
+            chunk['_rowId'] = this.rowId;
+            chunk['errors'] = [];
+            this.rowId += 1;
+            callback(null, chunk);
+        }
+    });
+}
+module.exports = {
+    RowMetaData
+};

package/dist/esm/loaders/csvloader.js

@@ -0,0 +1,32 @@
+"use strict";
+// @ts-nocheck
+const fs = require("fs");
+const { Transform } = require("stream");
+const { parse } = require("csv-parse");
+const { stdout } = require("process");
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.path
+ * @returns StreamReader
+ */
+function CSVLoader(options) {
+    let csvLoader = fs.createReadStream(options.path);
+    csvLoader._columns = options.columns;
+    csvLoader.pump = (csvLoader) => {
+        return csvLoader
+            .pipe(parse({ delimiter: ",", from_line: 2 }))
+            .pipe(new Transform({
+            readableObjectMode: true,
+            writableObjectMode: true,
+            transform(chunk, _, callback) {
+                chunk["_columns"] = options.columns;
+                callback(null, chunk);
+            }
+        }));
+    };
+    return csvLoader;
+}
+module.exports = {
+    CSVLoader
+};

package/dist/esm/loaders/index.js

@@ -0,0 +1,5 @@
+"use strict";
+const { CSVLoader } = require("./csvloader");
+module.exports = {
+    CSVLoader
+};