ffc-pay-etl-framework 0.0.10 → 0.0.11

package/README.md

@@ -1,15 +1,20 @@
 ## 🚀 Features
+
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
+
 [![All Contributors](https://img.shields.io/badge/all_contributors-2-orange.svg?style=flat-square)](#contributors-)
+
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
+
 - 💾 CSV Source and Destination
 - 👮 Validators
 - 🤖 Transformers
 - 🚨 Error Checking
 - 🐘 Write directly to Postgres
-- 👨‍⚕️ Intellisense 
+- 👨‍⚕️ Intellisense
 
 ## 📦 Install
+
 ```bash
 npm install --save-dev ffc-pay-etl-framework
 ```
@@ -18,30 +23,56 @@ npm install --save-dev ffc-pay-etl-framework
 
 ```js
 // ESM
-import { Etl, Loaders, Validators, Transformers, Destinations } from "ffc-pay-etl-framework"
+import {
+  Etl,
+  Loaders,
+  Validators,
+  Transformers,
+  Destinations,
+} from "ffc-pay-etl-framework";
 
 // CJS
-const { Etl, Loaders, Validators, Transformers, Destinations } = require("ffc-pay-etl-framework")
+const {
+  Etl,
+  Loaders,
+  Validators,
+  Transformers,
+  Destinations,
+} = require("ffc-pay-etl-framework");
 
-let csvFile = `${process.cwd()}/test/fixtures/SoilType.csv`
+let csvFile = `${process.cwd()}/test/fixtures/SoilType.csv`;
+const spinner = new pkg.Spinner().start("Running ETL Pipeline");
 
-const etl = new Etl.Etl()
+const etl = new Etl.Etl();
 
 etl
-.loader(Loaders.CSVLoader({path: csvFile, columns: columns}))
-.transform(Transformers.FakerTransformer({
-    columns: [{
-        name: "Dist Name",
-        faker: "location.city"
-    }]
-}))
-.destination(Destinations.CSVFileDestination({ 
-    fileName: "SoilType_Output.csv", 
-    headers: true, 
-    includeErrors: false, 
-    quotationMarks: true
-}))
-.pump()
+  .loader(Loaders.CSVLoader({ path: csvFile, columns: columns }))
+  .transform(
+    Transformers.FakerTransformer({
+      columns: [
+        {
+          name: "Dist Name",
+          faker: "location.city",
+        },
+      ],
+    })
+  )
+  .destination(
+    Destinations.CSVFileDestination({
+      fileName: "SoilType_Output.csv",
+      headers: true,
+      includeErrors: false,
+      quotationMarks: true,
+    })
+  )
+  .pump()
+  .on("finish", () => {
+    //Update spinner
+    spinner.succeed("ETL Pipeline - succeeded");
+  })
+  .on("result", (data) => {
+    console.log(data); // emits the last row with error information
+  });
 ```
 
 ## 📢 Shout outs
@@ -56,7 +87,6 @@ etl
 
 Please make sure to read the [Contributing Guide](https://github.com/DEFRA/ffc-pay-etl-framework/blob/next/CONTRIBUTING.md) before making a pull request.
 
-
 ## Contributors ✨
 
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
@@ -87,4 +117,4 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
 
 <!-- ALL-CONTRIBUTORS-LIST:END -->
 
-This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!
+This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!

package/dist/cjs/destinations/csvFileDestination.js

@@ -1,3 +1,5 @@
+const EventEmitter = require('node:events');
+const util = require('node:util');
 const { Writable } = require("node:stream");
 const fs = require("fs");
 /**
@@ -10,6 +12,8 @@ const fs = require("fs");
  * @returns Writable
  */
 function CSVFileDestination(options) {
+    EventEmitter.call(this);
+    let lastChunk;
     const fileName = options.fileName;
     const headers = options.headers;
     const includeErrors = options.includeErrors;
@@ -39,10 +43,16 @@ function CSVFileDestination(options) {
                     fs.writeFileSync(fileHandle, `${chunk.map(c => `"${c}"`).join(",")}\n`);
                 }
             }
+            lastChunk = chunk;
+            callback();
+        },
+        final(callback) {
+            this.emit('result', lastChunk);
             callback();
         }
     });
 }
+util.inherits(CSVFileDestination, EventEmitter);
 module.exports = {
     CSVFileDestination
 };

package/dist/cjs/destinations/postgresDestination.js

@@ -1,3 +1,5 @@
+const EventEmitter = require('node:events');
+const util = require('node:util');
 const { Transform } = require('node:stream');
 const { Sequelize } = require('sequelize');
 const debug = require('debug')('destination');
@@ -16,6 +18,7 @@ const DEFAULT_PORT = 5432;
  * @returns Transform
  */
 function PostgresDestination(options) {
+    EventEmitter.call(this);
     const table = options.table;
     const username = options.username;
     const password = options.password;
@@ -24,6 +27,7 @@ function PostgresDestination(options) {
     const port = options.port || DEFAULT_PORT;
     const mapping = options.mapping;
     const includeErrors = options.includeErrors;
+    let lastChunk;
     const sequelize = new Sequelize(database, username, password, {
         host: host,
         port: port,
@@ -91,11 +95,13 @@ function PostgresDestination(options) {
                     .then(result => {
                     debug('result %o', result);
                     chunk._result = result;
+                    lastChunk = chunk;
                     // @ts-ignore
                     callback(null, chunk);
                 }).catch(error => {
                     debug('error %o', error);
                     chunk.errors.push(error);
+                    lastChunk = chunk;
                     // @ts-ignore
                     callback(null, chunk);
                 });
@@ -103,6 +109,10 @@ function PostgresDestination(options) {
             else {
                 debug('Chunk has errors %o', chunk);
             }
+        },
+        final(callback) {
+            this.emit('result', lastChunk);
+            callback();
         }
     });
 }

package/dist/cjs/lib/index.js

@@ -28,7 +28,7 @@ function Etl() {
     this.pump = () => {
         this.loader
             .pump(this.loader)
-            .pipe(compose(RowMetaData(), ...self.validatorList, ...self.transformationList, ...self.destinationList)
+            .pipe(compose(RowMetaData(), ...self.validatorList, ...self.transformationList, ...self.destinationList.map(dl => dl.on('result', (data) => self.emit('result', data))))
         // @ts-ignore
         ).on('finish', (data) => self.emit('finish', data));
         return self;

package/dist/cjs/loaders/csvloader.js

@@ -2,7 +2,7 @@
 const fs = require("fs");
 const { Transform } = require("stream");
 const { parse } = require("csv-parse");
-const { stdout } = require("process");
+const { stdout, stderr } = require("process");
 /**
  *
  * @param {Object} options
@@ -11,6 +11,7 @@ const { stdout } = require("process");
  */
 function CSVLoader(options) {
     let csvLoader = fs.createReadStream(options.path);
+    let lineCount = 1;
     csvLoader._columns = options.columns;
     csvLoader.pump = (csvLoader) => {
         return csvLoader
@@ -21,6 +22,8 @@ function CSVLoader(options) {
             emitClose: true,
             transform(chunk, _, callback) {
                 chunk["_columns"] = options.columns;
+                chunk["_linecount"] = lineCount;
+                lineCount += 1;
                 callback(null, chunk);
             }
         }));

package/dist/cjs/types/destinations/csvFileDestination.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csvFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/csvFileDestination.js"],"names":[],"mappings":"AAGA;;;;;;;;GAQG;AACH,4CANG;IAAwB,QAAQ;IACP,OAAO;IACP,aAAa;IACb,cAAc;CACvC,YAkCF"}
+{"version":3,"file":"csvFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/csvFileDestination.js"],"names":[],"mappings":"AAKA;;;;;;;;GAQG;AACH,4CANG;IAAwB,QAAQ;IACP,OAAO;IACP,aAAa;IACb,cAAc;CACvC,YAwCF"}

package/dist/cjs/types/destinations/postgresDestination.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAIA;;;;;;;;;;;;GAYG;AACH,6CAVG;IAAwB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,aAAa;CACrC,aA6FF"}
+{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAMA;;;;;;;;;;;;GAYG;AACH,6CAVG;IAAwB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,aAAa;CACrC,aAoGF"}

package/dist/cjs/types/loaders/csvloader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csvloader.d.ts","sourceRoot":"","sources":["../../../../src/loaders/csvloader.js"],"names":[],"mappings":"AAMA;;;;;GAKG;AACH,mCAHG;IAAwB,IAAI;CAC5B,iBAoBF"}
+{"version":3,"file":"csvloader.d.ts","sourceRoot":"","sources":["../../../../src/loaders/csvloader.js"],"names":[],"mappings":"AAMA;;;;;GAKG;AACH,mCAHG;IAAwB,IAAI;CAC5B,iBAuBF"}

package/dist/esm/destinations/csvFileDestination.js

@@ -1,4 +1,6 @@
 "use strict";
+const EventEmitter = require('node:events');
+const util = require('node:util');
 const { Writable } = require("node:stream");
 const fs = require("fs");
 /**
@@ -11,6 +13,8 @@ const fs = require("fs");
  * @returns Writable
  */
 function CSVFileDestination(options) {
+    EventEmitter.call(this);
+    let lastChunk;
     const fileName = options.fileName;
     const headers = options.headers;
     const includeErrors = options.includeErrors;
@@ -40,10 +44,16 @@ function CSVFileDestination(options) {
                     fs.writeFileSync(fileHandle, `${chunk.map(c => `"${c}"`).join(",")}\n`);
                 }
             }
+            lastChunk = chunk;
+            callback();
+        },
+        final(callback) {
+            this.emit('result', lastChunk);
             callback();
         }
     });
 }
+util.inherits(CSVFileDestination, EventEmitter);
 module.exports = {
     CSVFileDestination
 };

package/dist/esm/destinations/postgresDestination.js

@@ -1,4 +1,6 @@
 "use strict";
+const EventEmitter = require('node:events');
+const util = require('node:util');
 const { Transform } = require('node:stream');
 const { Sequelize } = require('sequelize');
 const debug = require('debug')('destination');
@@ -17,6 +19,7 @@ const DEFAULT_PORT = 5432;
  * @returns Transform
  */
 function PostgresDestination(options) {
+    EventEmitter.call(this);
     const table = options.table;
     const username = options.username;
     const password = options.password;
@@ -25,6 +28,7 @@ function PostgresDestination(options) {
     const port = options.port || DEFAULT_PORT;
     const mapping = options.mapping;
     const includeErrors = options.includeErrors;
+    let lastChunk;
     const sequelize = new Sequelize(database, username, password, {
         host: host,
         port: port,
@@ -92,11 +96,13 @@ function PostgresDestination(options) {
                     .then(result => {
                     debug('result %o', result);
                     chunk._result = result;
+                    lastChunk = chunk;
                     // @ts-ignore
                     callback(null, chunk);
                 }).catch(error => {
                     debug('error %o', error);
                     chunk.errors.push(error);
+                    lastChunk = chunk;
                     // @ts-ignore
                     callback(null, chunk);
                 });
@@ -104,6 +110,10 @@ function PostgresDestination(options) {
             else {
                 debug('Chunk has errors %o', chunk);
             }
+        },
+        final(callback) {
+            this.emit('result', lastChunk);
+            callback();
         }
     });
 }

package/dist/esm/lib/index.js

@@ -29,7 +29,7 @@ function Etl() {
     this.pump = () => {
         this.loader
             .pump(this.loader)
-            .pipe(compose(RowMetaData(), ...self.validatorList, ...self.transformationList, ...self.destinationList)
+            .pipe(compose(RowMetaData(), ...self.validatorList, ...self.transformationList, ...self.destinationList.map(dl => dl.on('result', (data) => self.emit('result', data))))
         // @ts-ignore
         ).on('finish', (data) => self.emit('finish', data));
         return self;

package/dist/esm/loaders/csvloader.js

@@ -3,7 +3,7 @@
 const fs = require("fs");
 const { Transform } = require("stream");
 const { parse } = require("csv-parse");
-const { stdout } = require("process");
+const { stdout, stderr } = require("process");
 /**
  *
  * @param {Object} options
@@ -12,6 +12,7 @@ const { stdout } = require("process");
  */
 function CSVLoader(options) {
     let csvLoader = fs.createReadStream(options.path);
+    let lineCount = 1;
     csvLoader._columns = options.columns;
     csvLoader.pump = (csvLoader) => {
         return csvLoader
@@ -22,6 +23,8 @@ function CSVLoader(options) {
             emitClose: true,
             transform(chunk, _, callback) {
                 chunk["_columns"] = options.columns;
+                chunk["_linecount"] = lineCount;
+                lineCount += 1;
                 callback(null, chunk);
             }
         }));

package/dist/esm/types/destinations/csvFileDestination.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csvFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/csvFileDestination.js"],"names":[],"mappings":"AAGA;;;;;;;;GAQG;AACH,4CANG;IAAwB,QAAQ;IACP,OAAO;IACP,aAAa;IACb,cAAc;CACvC,YAkCF"}
+{"version":3,"file":"csvFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/csvFileDestination.js"],"names":[],"mappings":"AAKA;;;;;;;;GAQG;AACH,4CANG;IAAwB,QAAQ;IACP,OAAO;IACP,aAAa;IACb,cAAc;CACvC,YAwCF"}

package/dist/esm/types/destinations/postgresDestination.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAIA;;;;;;;;;;;;GAYG;AACH,6CAVG;IAAwB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,aAAa;CACrC,aA6FF"}
+{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAMA;;;;;;;;;;;;GAYG;AACH,6CAVG;IAAwB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,aAAa;CACrC,aAoGF"}

package/dist/esm/types/loaders/csvloader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csvloader.d.ts","sourceRoot":"","sources":["../../../../src/loaders/csvloader.js"],"names":[],"mappings":"AAMA;;;;;GAKG;AACH,mCAHG;IAAwB,IAAI;CAC5B,iBAoBF"}
+{"version":3,"file":"csvloader.d.ts","sourceRoot":"","sources":["../../../../src/loaders/csvloader.js"],"names":[],"mappings":"AAMA;;;;;GAKG;AACH,mCAHG;IAAwB,IAAI;CAC5B,iBAuBF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ffc-pay-etl-framework",
-  "version": "0.0.10",
+  "version": "0.0.11",
   "publisher": "Defra",
   "main": "dist/cjs/index.js",
   "private": false,