ffc-pay-etl-framework 0.0.1

package/dist/esm/transformers/fakerTransformer.js

@@ -0,0 +1,40 @@
+"use strict";
+const { Transform } = require("node:stream");
+const { faker } = require("@faker-js/faker");
+/**
+ *
+ * @param {Object} options
+ * @param {Array} options.columns {
+ *           columns: [
+ *               {
+ *                   name: "column-name",
+ *                   faker: "company.name"
+ *               }
+ *           ]
+ *       }
+ * @returns StreamReader
+ */
+function FakerTransformer(options) {
+    let self = this;
+    self.columns = options.columns;
+    function getFaker(fakerType) {
+        return fakerType.split('.').reduce((a, b) => {
+            return Object.keys(a).length === 0 ? faker[b] : a[b];
+        }, {});
+    }
+    return new Transform({
+        readableObjectMode: true,
+        writableObjectMode: true,
+        transform(chunk, _, callback) {
+            const { _columns } = chunk;
+            self.columns.forEach(column => {
+                const colIndex = _columns.indexOf(column.name);
+                chunk[colIndex] = getFaker(column.faker)();
+            });
+            callback(null, chunk);
+        }
+    });
+}
+module.exports = {
+    FakerTransformer
+};

package/dist/esm/transformers/index.js

@@ -0,0 +1,7 @@
+"use strict";
+const { ToUpperCaseTransformer } = require("./toUpperCaseTransformer");
+const { FakerTransformer } = require("./fakerTransformer");
+module.exports = {
+    ToUpperCaseTransformer,
+    FakerTransformer
+};

package/dist/esm/transformers/toUpperCaseTransformer.js

@@ -0,0 +1,25 @@
+"use strict";
+const { Transform } = require("node:stream");
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.column
+ * @returns Writable
+ */
+function ToUpperCaseTransformer(options) {
+    let self = this;
+    self.column = options.column;
+    return new Transform({
+        readableObjectMode: true,
+        writableObjectMode: true,
+        transform(chunk, _, callback) {
+            const { _columns } = chunk;
+            const colIndex = _columns.indexOf(self.column);
+            chunk[colIndex] = chunk[colIndex].toUpperCase();
+            callback(null, chunk);
+        }
+    });
+}
+module.exports = {
+    ToUpperCaseTransformer
+};

package/dist/esm/types/destinations/consoleDestination.d.ts

@@ -0,0 +1,11 @@
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.includeErrors
+ * @returns Writable
+ */
+export function ConsoleDestination(options: {
+    includeErrors: string;
+}): Writable;
+import { Writable } from "stream";
+//# sourceMappingURL=consoleDestination.d.ts.map

package/dist/esm/types/destinations/consoleDestination.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"consoleDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/consoleDestination.js"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,4CAHG;IAAwB,aAAa;CACrC,YAYF"}

package/dist/esm/types/destinations/csvFileDestination.d.ts

@@ -0,0 +1,17 @@
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.fileName
+ * @param {Boolean} options.headers
+ * @param {Boolean} options.includeErrors
+ * @param {Boolean} options.quotationMarks
+ * @returns Writable
+ */
+export function CSVFileDestination(options: {
+    fileName: string;
+    headers: boolean;
+    includeErrors: boolean;
+    quotationMarks: boolean;
+}): Writable;
+import { Writable } from "stream";
+//# sourceMappingURL=csvFileDestination.d.ts.map

package/dist/esm/types/destinations/csvFileDestination.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csvFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/csvFileDestination.js"],"names":[],"mappings":"AAGA;;;;;;;;GAQG;AACH,4CANG;IAAwB,QAAQ;IACP,OAAO;IACP,aAAa;IACb,cAAc;CACvC,YAkCF"}

package/dist/esm/types/destinations/index.d.ts

@@ -0,0 +1,7 @@
+import { ConsoleDestination } from "./consoleDestination";
+import { SQLFileDestination } from "./sqlFileDestination";
+import { SQL_MODE } from "./sqlFileDestination";
+import { CSVFileDestination } from "./csvFileDestination";
+import { PostgresDestination } from "./postgresDestination";
+export { ConsoleDestination, SQLFileDestination, SQL_MODE, CSVFileDestination, PostgresDestination };
+//# sourceMappingURL=index.d.ts.map

package/dist/esm/types/destinations/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/destinations/index.js"],"names":[],"mappings":""}

package/dist/esm/types/destinations/postgresDestination.d.ts

@@ -0,0 +1,23 @@
+/**
+ *
+ * @param {Object} options
+ * @param {Object} options.table
+ * @param {Object} options.username
+ * @param {Object} options.password
+ * @param {Object} options.database
+ * @param {Object} options.host
+ * @param {Object} options.mapping
+ * @param {Object} options.includeErrors
+ * @returns Transform
+ */
+export function PostgresDestination(options: {
+    table: any;
+    username: any;
+    password: any;
+    database: any;
+    host: any;
+    mapping: any;
+    includeErrors: any;
+}): Transform;
+import { Transform } from "stream";
+//# sourceMappingURL=postgresDestination.d.ts.map

package/dist/esm/types/destinations/postgresDestination.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAGA;;;;;;;;;;;GAWG;AACH,6CATG;IAAwB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,OAAO;IACP,aAAa;CACrC,aAuDF"}

package/dist/esm/types/destinations/sqlFileDestination.d.ts

@@ -0,0 +1,24 @@
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.fileName
+ * @param {SQL_MODE} options.mode
+ * @param {String} options.table
+ * @param {Array} options.mapping
+ * @param {Boolean} options.includeErrors
+ * @returns Transform
+ */
+export function SQLFileDestination(options: {
+    fileName: string;
+    mode: SQL_MODE;
+    table: string;
+    mapping: any[];
+    includeErrors: boolean;
+}): Writable;
+export type SQL_MODE = number;
+export namespace SQL_MODE {
+    let INSERT_MODE: number;
+    let UPDATE_MODE: number;
+}
+import { Writable } from "stream";
+//# sourceMappingURL=sqlFileDestination.d.ts.map

package/dist/esm/types/destinations/sqlFileDestination.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sqlFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/sqlFileDestination.js"],"names":[],"mappings":"AAWA;;;;;;;;;GASG;AACH,4CAPG;IAAwB,QAAQ;IACN,IAAI,EAAtB,QAAQ;IACQ,KAAK;IACN,OAAO;IACL,aAAa;CACtC,YA4CF;uBA3DS,MAAM"}

package/dist/esm/types/index.d.ts

@@ -0,0 +1,7 @@
+import Etl = require("./lib");
+import Validators = require("./validators");
+import Transformers = require("./transformers");
+import Loaders = require("./loaders");
+import Destinations = require("./destinations");
+export { Etl, Validators, Transformers, Loaders, Destinations };
+//# sourceMappingURL=index.d.ts.map

package/dist/esm/types/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/index.js"],"names":[],"mappings":""}

package/dist/esm/types/lib/index.d.ts

@@ -0,0 +1,25 @@
+export type Etl = any;
+/**
+ * @typedef {Object} Etl
+ * @function loader
+ * @function pump
+ * @function validator
+ * @function destination
+ * @function transform
+ */
+/**
+ *
+ * @returns Etl
+ */
+export function Etl(): void;
+export class Etl {
+    validatorList: any[];
+    transformationList: any[];
+    destinationList: any[];
+    loader: (loader: any) => this;
+    pump: () => void;
+    validator: (validator: any) => this;
+    destination: (destination: any) => this;
+    transform: (transform: any) => this;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/esm/types/lib/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAIA;;;;;;;GAOG;AAEH;;;GAGG;AACH,4BAwCC;;IAtCG,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,8BAGC;IAED,iBAWC;IAED,oCAGC;IAED,wCAGC;IAED,oCAGC"}

package/dist/esm/types/lib/rowMetaData.d.ts

@@ -0,0 +1,3 @@
+export function RowMetaData(options: any): Transform;
+import { Transform } from "stream";
+//# sourceMappingURL=rowMetaData.d.ts.map

package/dist/esm/types/lib/rowMetaData.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rowMetaData.d.ts","sourceRoot":"","sources":["../../../../src/lib/rowMetaData.js"],"names":[],"mappings":"AAIA,qDAgBC"}

package/dist/esm/types/loaders/csvloader.d.ts

@@ -0,0 +1,11 @@
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.path
+ * @returns StreamReader
+ */
+export function CSVLoader(options: {
+    path: string;
+}): fs.ReadStream;
+import fs = require("fs");
+//# sourceMappingURL=csvloader.d.ts.map

package/dist/esm/types/loaders/csvloader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csvloader.d.ts","sourceRoot":"","sources":["../../../../src/loaders/csvloader.js"],"names":[],"mappings":"AAMA;;;;;GAKG;AACH,mCAHG;IAAwB,IAAI;CAC5B,iBAmBF"}

package/dist/esm/types/loaders/index.d.ts

@@ -0,0 +1,3 @@
+export { CSVLoader };
+import { CSVLoader } from "./csvloader";
+//# sourceMappingURL=index.d.ts.map

package/dist/esm/types/loaders/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/loaders/index.js"],"names":[],"mappings":""}

package/dist/esm/types/transformers/fakerTransformer.d.ts

@@ -0,0 +1,37 @@
+/**
+ *
+ * @param {Object} options
+ * @param {Array} options.columns {
+ *           columns: [
+ *               {
+ *                   name: "column-name",
+ *                   faker: "company.name"
+ *               }
+ *           ]
+ *       }
+ * @returns StreamReader
+ */
+export function FakerTransformer(options: {
+    columns: any[];
+}): Transform;
+export class FakerTransformer {
+    /**
+     *
+     * @param {Object} options
+     * @param {Array} options.columns {
+     *           columns: [
+     *               {
+     *                   name: "column-name",
+     *                   faker: "company.name"
+     *               }
+     *           ]
+     *       }
+     * @returns StreamReader
+     */
+    constructor(options: {
+        columns: any[];
+    });
+    columns: any[];
+}
+import { Transform } from "stream";
+//# sourceMappingURL=fakerTransformer.d.ts.map

package/dist/esm/types/transformers/fakerTransformer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fakerTransformer.d.ts","sourceRoot":"","sources":["../../../../src/transformers/fakerTransformer.js"],"names":[],"mappings":"AAGA;;;;;;;;;;;;GAYG;AACH,0CAVG;IAAuB,OAAO;CAQ9B,aAsBF;;IAjCD;;;;;;;;;;;;OAYG;IACH,qBAVG;QAAuB,OAAO;KAQ9B,EAsBF;IAlBG,eAA8B"}

package/dist/esm/types/transformers/index.d.ts

@@ -0,0 +1,4 @@
+import { ToUpperCaseTransformer } from "./toUpperCaseTransformer";
+import { FakerTransformer } from "./fakerTransformer";
+export { ToUpperCaseTransformer, FakerTransformer };
+//# sourceMappingURL=index.d.ts.map

package/dist/esm/types/transformers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/transformers/index.js"],"names":[],"mappings":""}

package/dist/esm/types/transformers/toUpperCaseTransformer.d.ts

@@ -0,0 +1,23 @@
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.column
+ * @returns Writable
+ */
+export function ToUpperCaseTransformer(options: {
+    column: string;
+}): Transform;
+export class ToUpperCaseTransformer {
+    /**
+     *
+     * @param {Object} options
+     * @param {String} options.column
+     * @returns Writable
+     */
+    constructor(options: {
+        column: string;
+    });
+    column: string;
+}
+import { Transform } from "stream";
+//# sourceMappingURL=toUpperCaseTransformer.d.ts.map

package/dist/esm/types/transformers/toUpperCaseTransformer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"toUpperCaseTransformer.d.ts","sourceRoot":"","sources":["../../../../src/transformers/toUpperCaseTransformer.js"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,gDAHG;IAAwB,MAAM;CAC9B,aAgBF;;IApBD;;;;;OAKG;IACH,qBAHG;QAAwB,MAAM;KAC9B,EAgBF;IAZG,eAA4B"}

package/dist/esm/types/validators/index.d.ts

@@ -0,0 +1,5 @@
+import { RequiredValidator } from "./requiredValidator";
+import { UniqueValidator } from "./uniqueValidator";
+import { MultiToolValidator } from "./multiToolValidator";
+export { RequiredValidator, UniqueValidator, MultiToolValidator };
+//# sourceMappingURL=index.d.ts.map

package/dist/esm/types/validators/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/validators/index.js"],"names":[],"mappings":""}

package/dist/esm/types/validators/multiToolValidator.d.ts

@@ -0,0 +1,23 @@
+/**
+ *
+ * @param {Object} options
+ * @param {Array} options.columns { columns: [
+ *   {
+ *       name: "column-name",
+ *       validator: "isAlphanumeric",
+ *       args: {
+ *               locale: "en-GB",
+ *               options: {
+ *                   ignore: "-;%/ "
+ *               }
+ *       }
+ *   }
+ *  ]
+ * }
+ * @returns Writable
+ */
+export function MultiToolValidator(options: {
+    columns: any[];
+}): Transform;
+import { Transform } from "stream";
+//# sourceMappingURL=multiToolValidator.d.ts.map

package/dist/esm/types/validators/multiToolValidator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"multiToolValidator.d.ts","sourceRoot":"","sources":["../../../../src/validators/multiToolValidator.js"],"names":[],"mappings":"AAIA;;;;;;;;;;;;;;;;;GAiBG;AACH,4CAfG;IAAuB,OAAO;CAa9B,aAyBF"}

package/dist/esm/types/validators/requiredValidator.d.ts

@@ -0,0 +1,13 @@
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.message
+ * @param {Array} options.columns
+ * @returns Transform
+ */
+export function RequiredValidator(options: {
+    message: string;
+    columns: any[];
+}): Transform;
+import { Transform } from "stream";
+//# sourceMappingURL=requiredValidator.d.ts.map

package/dist/esm/types/validators/requiredValidator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"requiredValidator.d.ts","sourceRoot":"","sources":["../../../../src/validators/requiredValidator.js"],"names":[],"mappings":"AAGA;;;;;;GAMG;AACH,2CAJG;IAAwB,OAAO;IACR,OAAO;CAC9B,aAsBF"}

package/dist/esm/types/validators/uniqueValidator.d.ts

@@ -0,0 +1,29 @@
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.message
+ * @param {String} options.column
+ * @returns Transform
+ */
+export function UniqueValidator(options: {
+    message: string;
+    column: string;
+}): Transform;
+export class UniqueValidator {
+    /**
+     *
+     * @param {Object} options
+     * @param {String} options.message
+     * @param {String} options.column
+     * @returns Transform
+     */
+    constructor(options: {
+        message: string;
+        column: string;
+    });
+    column: string;
+    message: string;
+    uniqueColumnRows: any[];
+}
+import { Transform } from "stream";
+//# sourceMappingURL=uniqueValidator.d.ts.map

package/dist/esm/types/validators/uniqueValidator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"uniqueValidator.d.ts","sourceRoot":"","sources":["../../../../src/validators/uniqueValidator.js"],"names":[],"mappings":"AAEA;;;;;;GAMG;AACH,yCAJG;IAAwB,OAAO;IACP,MAAM;CAC9B,aA6BF;;IAlCD;;;;;;OAMG;IACH,qBAJG;QAAwB,OAAO;QACP,MAAM;KAC9B,EA6BF;IAzBG,eAA4B;IAC5B,gBAAwF;IACxF,wBAA0B"}

package/dist/esm/validators/index.js

@@ -0,0 +1,9 @@
+"use strict";
+const { RequiredValidator } = require("./requiredValidator");
+const { UniqueValidator } = require("./uniqueValidator");
+const { MultiToolValidator } = require("./multiToolValidator");
+module.exports = {
+    RequiredValidator,
+    UniqueValidator,
+    MultiToolValidator
+};

package/dist/esm/validators/multiToolValidator.js

@@ -0,0 +1,49 @@
+"use strict";
+// @ts-nocheck
+const { Transform } = require("stream");
+const validator = require('validator');
+/**
+ *
+ * @param {Object} options
+ * @param {Array} options.columns { columns: [
+ *   {
+ *       name: "column-name",
+ *       validator: "isAlphanumeric",
+ *       args: {
+ *               locale: "en-GB",
+ *               options: {
+ *                   ignore: "-;%/ "
+ *               }
+ *       }
+ *   }
+ *  ]
+ * }
+ * @returns Writable
+ */
+function MultiToolValidator(options) {
+    function getValidator(validatorType) {
+        return validator[validatorType];
+    }
+    return new Transform({
+        readableObjectMode: true,
+        writableObjectMode: true,
+        decodeStrings: false,
+        construct(callback) {
+            this.columns = options.columns;
+            callback();
+        },
+        transform(chunk, _, callback) {
+            const { _columns } = chunk;
+            this.columns.forEach(column => {
+                const colIndex = _columns.indexOf(column.name);
+                const vtr = getValidator(column.validator);
+                if (!vtr(chunk[colIndex], ...Object.values(column.args)))
+                    chunk.errors.push(`${column.validator} returned invalid status in column ${colIndex}`);
+            });
+            callback(null, chunk);
+        }
+    });
+}
+module.exports = {
+    MultiToolValidator
+};

package/dist/esm/validators/requiredValidator.js

@@ -0,0 +1,34 @@
+"use strict";
+// @ts-nocheck
+const { Transform } = require("stream");
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.message
+ * @param {Array} options.columns
+ * @returns Transform
+ */
+function RequiredValidator(options) {
+    return new Transform({
+        readableObjectMode: true,
+        writableObjectMode: true,
+        decodeStrings: false,
+        construct(callback) {
+            this.message = options.message ? options.message : "Required value is null";
+            this.requiredColumns = options.columns;
+            callback();
+        },
+        transform(chunk, _, callback) {
+            this.requiredColumns.forEach(column => {
+                let requiredVal = chunk[column];
+                if (requiredVal === undefined || requiredVal === '') {
+                    chunk.errors.push(`${this.message} in column ${column}`);
+                }
+            });
+            callback(null, chunk);
+        }
+    });
+}
+module.exports = {
+    RequiredValidator
+};

package/dist/esm/validators/uniqueValidator.js

@@ -0,0 +1,41 @@
+"use strict";
+const { Transform } = require("node:stream");
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.message
+ * @param {String} options.column
+ * @returns Transform
+ */
+function UniqueValidator(options) {
+    let self = this;
+    self.column = options.column;
+    self.message = options.message ? options.message : "Expected unique value is not unique";
+    self.uniqueColumnRows = [];
+    function checkIsUnique(data) {
+        return self.uniqueColumnRows.indexOf(data) === -1;
+    }
+    return new Transform({
+        readableObjectMode: true,
+        writableObjectMode: true,
+        decodeStrings: false,
+        construct(callback) {
+            callback();
+        },
+        transform(chunk, _, callback) {
+            const { _columns } = chunk;
+            const colIndex = _columns.indexOf(self.column);
+            const uniqueValue = chunk[colIndex];
+            if (checkIsUnique(uniqueValue)) {
+                self.uniqueColumnRows.push(uniqueValue);
+            }
+            else {
+                chunk.errors.push(`${self.message} ${self.column}`);
+            }
+            callback(null, chunk);
+        }
+    });
+}
+module.exports = {
+    UniqueValidator
+};

package/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "ffc-pay-etl-framework",
+  "version": "0.0.1",
+  "publisher": "Defra",
+  "main": "dist/cjs/index.js",
+  "private": false,
+  "scripts": {
+    "test": "jest",
+    "test:watch": "jest --watchAll",
+    "dev": "nodemon src/index.js",
+    "clean": "rm -rf ./dist",
+    "build": "npm run clean && npm run build:esm && npm run build:cjs",
+    "build:esm": "tsc -p ./tsconfig.esm.json && mv dist/esm/index.js dist/esm/index.mjs",
+    "build:cjs": "tsc -p ./tsconfig.cjs.json",
+    "prepack": "npm run build",
+    "example": "node ./src/examples/csv-multi-tool-validator.js"
+  },
+  "keywords": [
+    "ETL"
+  ],
+  "author": "Charlie Benger-Stevenson",
+  "license": "ISC",
+  "description": "A framework for creating ETL pipelines in node",
+  "devDependencies": {
+    "jest": "^29.7.0",
+    "nodemon": "^3.1.4",
+    "prettier": "3.3.3"
+  },
+  "dependencies": {
+    "@faker-js/faker": "^8.4.1",
+    "csv": "^6.3.9",
+    "mariadb": "^3.3.1",
+    "mysql2": "^3.10.3",
+    "oracledb": "^6.5.1",
+    "pg": "^8.12.0",
+    "pg-hstore": "^2.3.4",
+    "sequelize": "^6.37.3",
+    "sqlite3": "^5.1.7",
+    "tedious": "^18.2.4",
+    "typescript": "^5.5.3",
+    "validator": "^13.12.0"
+  },
+  "files": [
+    "/dist/**/*"
+  ],
+  "exports": {
+    ".": {
+      "import": {
+        "default": "./dist/esm/index.js"
+      },
+      "require": {
+        "default": "./dist/cjs/index.js"
+      }
+    }
+  }
+}