ffc-pay-etl-framework 1.4.1 → 1.4.2-beta.135

package/.npmignore

@@ -0,0 +1,26 @@
+# Ignore dotfiles explicitly by name
+/.all-contributorsrc
+/.dockerignore
+/.gitignore
+/.snyk
+/CODE_OF_CONDUCT.md
+/CONTRIBUTING.md
+/eslint.config.js
+/ISSUE_TEMPLATE.md
+/Jenkinsfile
+/LICENSE.md
+/PULL_REQUEST_TEMPLATE.md
+/sonar-project.properties
+/snyk_report.html
+/snyk-error.log
+/snyk-monitor-result.json
+/snyk-result.json
+
+# Ignore folders recursively
+__mocks__/
+.devcontainer/
+.github/
+jest/
+test/
+snyk-cli/
+test-output/

package/app/database-connections/index.js

@@ -0,0 +1,7 @@
+const { PostgresDatabaseConnection } = require('./postgres-database-connection')
+const { ProvidedConnection } = require('./provided-connection')
+
+module.exports = {
+  PostgresDatabaseConnection,
+  ProvidedConnection
+}

package/app/{database_connections/postgresDatabaseConnection.js → database-connections/postgres-database-connection.js}

@@ -13,7 +13,7 @@ const debug = require('debug')('connection')
  * @param {Object} options.port
  * @returns Connection
  */
-async function PostgresDatabaseConnection (options) {
+async function postgresDatabaseConnection (options) {
   const connectionname = options.connectionname
   const username = options.username
   const password = options.password
@@ -43,5 +43,5 @@ async function PostgresDatabaseConnection (options) {
 }
 
 module.exports = {
-  PostgresDatabaseConnection
+  PostgresDatabaseConnection: postgresDatabaseConnection
 }

package/app/{database_connections/providedConnection.js → database-connections/provided-connection.js}

@@ -5,7 +5,7 @@
  * @param {Object} options.sequelize
  * @returns Connection
  */
-async function ProvidedConnection (options) {
+async function providedConnection (options) {
   const connectionname = options.connectionname
   const sequelize = options.sequelize
   return {
@@ -15,5 +15,5 @@ async function ProvidedConnection (options) {
 }
 
 module.exports = {
-  ProvidedConnection
+  ProvidedConnection: providedConnection
 }

package/app/destinations/{consoleDestination.js → console-destination.js}

@@ -6,7 +6,7 @@ const { Writable } = require('node:stream')
  * @param {String} options.includeErrors
  * @returns Writable
  */
-function ConsoleDestination (options) {
+function consoleDestination (options) {
   const includeErrors = options.includeErrors
   const writable = new Writable({
     objectMode: true,
@@ -34,5 +34,5 @@ function ConsoleDestination (options) {
 }
 
 module.exports = {
-  ConsoleDestination
+  ConsoleDestination: consoleDestination
 }

package/app/destinations/csv-file-destination.js

@@ -0,0 +1,94 @@
+const EventEmitter = require('node:events')
+const util = require('node:util')
+const { Writable } = require('node:stream')
+const fs = require('node:fs')
+
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.fileName
+ * @param {Boolean} options.headers
+ * @param {Boolean} options.includeErrors
+ * @param {Boolean} options.quotationMarks
+ * @returns Writable
+ */
+function csvFileDestination (options) {
+  EventEmitter.call(this)
+  let lastChunk
+  const fileName = options.fileName
+  const headers = options.headers
+  const includeErrors = options.includeErrors
+  const quotationMarks = options.quotationMarks
+
+  let fileHandle
+  try {
+    fileHandle = fs.openSync(fileName, 'w+')
+  } catch (error) {
+    throw new Error(`Failed to open file ${fileName}: ${error.message}`)
+  }
+
+  let headersWritten = false
+
+  const writable = new Writable({
+    objectMode: true,
+    write (chunk, _, callback) {
+      try {
+        if (!headersWritten && headers) {
+          if (quotationMarks) {
+            const quotedColumns = chunk._columns.map(c => `"${c}"`).join(',')
+            fs.writeSync(fileHandle, quotedColumns + '\n')
+          } else {
+            fs.writeSync(fileHandle, `${chunk._columns.join(',')}\n`)
+          }
+          headersWritten = true
+        }
+
+        const errors = Array.isArray(chunk.errors) ? chunk.errors : []
+
+        if (errors.length === 0 || includeErrors) {
+          const line = chunk.map(c => `"${c}"`).join(',') + '\n'
+          fs.writeSync(fileHandle, line)
+        }
+
+        if (this.tasks && Array.isArray(this.tasks)) {
+          this.tasks.forEach(task => task.write(chunk))
+        }
+
+        lastChunk = chunk
+        callback()
+      } catch (err) {
+        callback(err)
+      }
+    },
+    final (callback) {
+      if (fileHandle) {
+        fs.closeSync(fileHandle)
+      }
+
+      this.emit('result', lastChunk)
+      callback()
+    }
+  })
+
+  Object.assign(writable, {
+    setConnection: function (connection) {
+      this.connection = connection
+    }.bind(writable),
+
+    getConnectionName: function () {
+      return this.connection?.name
+    }.bind(writable),
+
+    setTasks: function (tasks) {
+      this.tasks = tasks
+    }.bind(writable)
+  })
+
+  return writable
+}
+
+util.inherits(csvFileDestination, EventEmitter)
+
+module.exports = {
+  CSVFileDestination: csvFileDestination
+}

package/app/destinations/index.js

@@ -1,7 +1,7 @@
-const { ConsoleDestination } = require('./consoleDestination')
-const { SQLFileDestination, SQL_MODE } = require('./sqlFileDestination')
-const { CSVFileDestination } = require('./csvFileDestination')
-const { PostgresDestination } = require('./postgresDestination')
+const { ConsoleDestination } = require('./console-destination')
+const { SQLFileDestination, SQL_MODE } = require('./sql-file-destination')
+const { CSVFileDestination } = require('./csv-file-destination')
+const { PostgresDestination } = require('./postgres-destination')
 
 module.exports = {
   ConsoleDestination,

package/app/destinations/{postgresDestination.js → postgres-destination.js}

@@ -7,13 +7,15 @@ function isKeyWord (column) {
 }
 
 function getMappingForColumn (mapping, column) {
-  if (mapping.length === 0) { return {} }
-  const [map] = mapping.filter(m => m.column === column)
-  return map
+  if (mapping.length === 0) {
+    return {}
+  }
+  const map = mapping.find(m => m.column === column)
+  return map || {}
 }
 
 function hasReturningColumns (mapping) {
-  return mapping.filter(m => m.returning === true).length > 0
+  return mapping.some(m => m.returning === true)
 }
 
 function getReturningColumns (mapping) {
@@ -33,9 +35,9 @@ function writeInsertStatement (columnMapping, table, chunk, schema, ignoredColum
         .join(',')}) VALUES (${filteredColumns.map(column => {
         const index = chunk._columns.indexOf(column)
         const mapping = getMappingForColumn(columnMapping, column)
-        if (mapping?.targetType === 'number' && (isNaN(chunk[index]) || chunk[index] === '')) {
-            debug('Source data is not a number.')
-            return 0
+        if (mapping?.targetType === 'number' && (Number.isNaN(Number(chunk[index])) || chunk[index] === '')) {
+          debug('Source data is not a number.')
+          return 0
         }
         if (mapping?.targetType === 'varchar' || mapping?.targetType === 'char') {
             return `'${chunk[index]}'`
@@ -65,7 +67,7 @@ function writeInsertStatement (columnMapping, table, chunk, schema, ignoredColum
  * @param {Array<String>} [options.ignoredColumns]
  * @returns Transform
  */
-function PostgresDestination (options) {
+function postgresDestination (options) {
   EventEmitter.call(this)
   const table = options.table
   const connectionname = options.connectionname
@@ -130,7 +132,7 @@ function PostgresDestination (options) {
 }
 
 module.exports = {
-  PostgresDestination,
+  PostgresDestination: postgresDestination,
   writeInsertStatement,
   isKeyWord,
   getMappingForColumn,

package/app/destinations/{sqlFileDestination.js → sql-file-destination.js}

@@ -1,5 +1,5 @@
 const { Writable } = require('node:stream')
-const fs = require('fs')
+const fs = require('node:fs')
 
 /**
  * @enum {number}
@@ -19,12 +19,11 @@ const SQL_MODE = {
  * @param {Boolean} options.includeErrors
  * @returns Transform
  */
-function SQLFileDestination (options) {
+function sqlFileDestination (options) {
   const fileName = options.fileName
   const sqlMode = options.mode
   const table = options.table
   const mapping = options.mapping
-  const includeErrors = options.includeErrors
 
   function writeInsertStatement (chunk) {
     const statement = `INSERT INTO ${table} (${mapping.map(m => m.targetColumn)
@@ -41,21 +40,9 @@ function SQLFileDestination (options) {
   const writable = new Writable({
     objectMode: true,
     write (chunk, _, callback) {
-      if (chunk.errors.length === 0) {
-        if (sqlMode === SQL_MODE.INSERT_MODE) {
-        // @ts-ignore
-          if (chunk.errors.length === 0 | includeErrors) {
-            writeInsertStatement(chunk)
-          }
-        }
+      if (chunk.errors.length === 0 && sqlMode === SQL_MODE.INSERT_MODE) {
+        writeInsertStatement(chunk)
       }
-      // TODO - Disabled because Sonar
-      // } else if (sqlMode === SQL_MODE.UPDATE_MODE){
-      //     // @ts-ignore
-      //     if(chunk.errors.length === 0 | includeErrors){
-      //         writeUpdateStatement(chunk)
-      //     }
-      // }
       callback()
     }
   })
@@ -76,6 +63,6 @@ function SQLFileDestination (options) {
 }
 
 module.exports = {
-  SQLFileDestination,
+  SQLFileDestination: sqlFileDestination,
   SQL_MODE
 }

package/app/lib/index.js

@@ -1,7 +1,5 @@
-// @ts-nocheck
 const EventEmitter = require('node:events')
-const util = require('node:util')
-const { RowMetaData } = require('./rowMetaData')
+const { RowMetaData } = require('./row-meta-data')
 const { compose } = require('node:stream')
 
 /**
@@ -18,89 +16,88 @@ const { compose } = require('node:stream')
  *
  * @returns Etl
  */
-function Etl () {
-  EventEmitter.call(this)
-  const self = this
-  self.store = []
-  self.beforeETLList = []
-  self.connectionList = []
-  self.validatorList = []
-  self.transformationList = []
-  self.destinationList = []
+class Etl extends EventEmitter {
+  constructor () {
+    super()
+    this.store = []
+    this.beforeETLList = []
+    this.connectionList = []
+    this.validatorList = []
+    this.transformationList = []
+    this.destinationList = []
+  }
 
-  this.loader = (loader) => {
-    self.loader = loader
-    return self
+  loader (loader) {
+    this.loader = loader
+    return this
   }
 
-  this.pump = () => {
+  pump () {
     this.beforeETLList.forEach(task => {
       task.write({})
     })
+
     this.loader
       .pump(this.loader)
       .pipe(
         compose(
           RowMetaData(),
-          ...self.validatorList,
-          ...self.transformationList,
-          ...self.destinationList.map(dl => dl.on('result', (data) => self.emit('result', data)))
+          ...this.validatorList,
+          ...this.transformationList,
+          ...this.destinationList.map(dl => dl.on('result', data => this.emit('result', data)))
         )
-        // @ts-ignore
       )
-      .on('error', (err) => self.emit('error', err))
-    return self
+      .on('error', err => this.emit('error', err))
+    return this
   }
 
-  this.beforeETL = (pipelineTask) => {
+  beforeETL (pipelineTask) {
     const connectionname = pipelineTask.getConnectionName()
-    const connection = this.connectionList.filter(c => c.name === connectionname)[0]
+    const connection = this.connectionList.find(c => c.name === connectionname)
     if (!connection) {
       throw new Error(`Connection with name ${connectionname} not found`)
     }
     pipelineTask.setConnection(connection)
-    pipelineTask.setETL(self)
-    self.beforeETLList.push(pipelineTask)
-    return self
+    pipelineTask.setETL(this)
+    this.beforeETLList.push(pipelineTask)
+    return this
   }
 
-  this.connection = (connection) => {
-    self.connectionList.push(connection)
-    return self
+  connection (connection) {
+    this.connectionList.push(connection)
+    return this
   }
 
-  this.validator = (validator) => {
-    self.validatorList.push(validator)
-    return self
+  validator (validator) {
+    this.validatorList.push(validator)
+    return this
   }
 
-  this.destination = (destination, ...tasks) => {
+  destination (destination, ...tasks) {
     const connectionname = destination.getConnectionName()
-    const connection = this.connectionList.filter(c => c.name === connectionname)[0]
+    const connection = this.connectionList.find(c => c.name === connectionname)
     if (!connection && destination.type === 'PostgresDestination') {
       throw new Error(`No connection could be found with name ${connectionname}`)
     } else {
       destination.setConnection(connection)
     }
 
-    if (tasks) {
+    if (tasks && tasks.length > 0) {
       for (const task of tasks) {
-        task.setETL(self)
+        task.setETL(this)
       }
       destination.setTasks(tasks)
     }
-    self.destinationList.push(destination)
-    return self
+    this.destinationList.push(destination)
+    return this
   }
 
-  this.transform = (transform) => {
-    self.transformationList.push(transform)
-    return self
+  transform (transform) {
+    this.transformationList.push(transform)
+    return this
   }
 }
 
-util.inherits(Etl, EventEmitter)
-
 module.exports = {
   Etl
 }

package/app/lib/{rowMetaData.js → row-meta-data.js}

@@ -1,7 +1,7 @@
 // @ts-nocheck
-const { Transform } = require('stream')
+const { Transform } = require('node:stream')
 
-function RowMetaData (options) {
+function rowMetaData () {
   return new Transform({
     readableObjectMode: true,
     writableObjectMode: true,
@@ -20,5 +20,5 @@ function RowMetaData (options) {
 }
 
 module.exports = {
-  RowMetaData
+  RowMetaData: rowMetaData
 }

package/app/loaders/{csvloader.js → csv-loader.js}

@@ -1,6 +1,6 @@
 // @ts-nocheck
-const fs = require('fs')
-const { Transform } = require('stream')
+const fs = require('node:fs')
+const { Transform } = require('node:stream')
 const { parse } = require('csv-parse')
 
 /**
@@ -12,18 +12,18 @@ const { parse } = require('csv-parse')
  * @param {Boolean} [options.relax]
  * @returns StreamReader
  */
-function CSVLoader (options) {
-  let csvLoader
+function csvLoader (options) {
+  let loader
   if (options.path) {
-    csvLoader = fs.createReadStream(options.path)
+    loader = fs.createReadStream(options.path)
   } else {
-    csvLoader = options.stream
+    loader = options.stream
   }
   let lineCount = 1
   const fromLine = options.startingLine ?? 2
   const relaxQuotes = options.relax ?? false
-  csvLoader._columns = options.columns
-  csvLoader.pump = (csvLoader) => {
+  loader._columns = options.columns
+  loader.pump = () => {
     const parser = parse({ delimiter: ',', from_line: fromLine, relax_quotes: relaxQuotes })
     const transformer = new Transform({
       readableObjectMode: true,
@@ -38,21 +38,21 @@ function CSVLoader (options) {
         options.columns.forEach((_column, index) => {
           if (chunk[index]) {
             // eslint-disable-next-line no-control-regex
-            chunk[index] = chunk[index].replace(/[\x00-\x1F\x7F-\x9F]/g, '')
+            chunk[index] = chunk[index].replaceAll(/[\x00-\x1F\x7F-\x9F]/g, '')
           }
         })
 
         callback(null, chunk)
       }
     })
-    return csvLoader
+    return loader
       .pipe(parser)
       .pipe(transformer)
   }
 
-  return csvLoader
+  return loader
 }
 
 module.exports = {
-  CSVLoader
+  CSVLoader: csvLoader
 }

package/app/loaders/index.js

@@ -1,4 +1,4 @@
-const { CSVLoader } = require('./csvloader')
+const { CSVLoader } = require('./csv-loader')
 
 module.exports = {
   CSVLoader

package/app/misc/index.js

@@ -0,0 +1,5 @@
+const { PostgresSQLTask } = require('./postgres-sql-task')
+
+module.exports = {
+  PostgresSQLTask
+}

package/app/misc/{postgresSQLTask.js → postgres-sql-task.js}

@@ -1,5 +1,5 @@
 // @ts-nocheck
-const { PassThrough } = require('stream')
+const { PassThrough } = require('node:stream')
 const startPosOffset = 3
 const endPosOffset = 1
 
@@ -23,7 +23,7 @@ function doPlaceHolderValueInterpolations (chunk, sql, placeholders) {
   return sql
 }
 
-function PostgresSQLTask (options) {
+function postgresSQLTask (options) {
   const passthrough = new PassThrough({
     readableObjectMode: true,
     writableObjectMode: true,
@@ -39,17 +39,11 @@ function PostgresSQLTask (options) {
         this.connection.db.query(this.sql)
       } else {
         const interpolatedSql = doPlaceHolderValueInterpolations(chunk, this.sql, placeholders)
-        // TODO add more interpolation mechanisms to specify return values
-        // e.g. 'myReturnVal = SELECT MAX ID FROM TABLE;'
-        // and write to etl.store.myReturnVal or
-        // e.g. 'chunk.myReturnVal = SELECT MAX ID FROM TABLE;'
-        // and write to the chunk in say chunk.store.myReturnVal
         this.connection.db.query(interpolatedSql)
       }
       callback(null, chunk)
     }
   })
-  // Should definately split this out into a mixin
   Object.assign(passthrough, {
     setConnection: function (connection) {
       this.connection = connection
@@ -65,7 +59,7 @@ function PostgresSQLTask (options) {
 }
 
 module.exports = {
-  PostgresSQLTask,
+  PostgresSQLTask: postgresSQLTask,
   getPlaceHolders,
   doPlaceHolderValueInterpolations
 }

package/app/transformers/{fakerTransformer.js → faker-transformer.js}

@@ -14,9 +14,8 @@ const { Transform } = require('node:stream')
  * @param {String} options.locale
  * @returns StreamReader
  */
-function FakerTransformer (options) {
-  const self = this
-  self.columns = options.columns
+function fakerTransformer (options) {
+  const columns = options.columns
   let faker
   if (options.locale) {
     faker = require(`@faker-js/faker/locale/${options.locale}`).faker
@@ -29,12 +28,13 @@ function FakerTransformer (options) {
       return Object.keys(a).length === 0 ? faker[b] : a[b]
     }, {})
   }
+
   return new Transform({
     readableObjectMode: true,
     writableObjectMode: true,
     transform (chunk, _, callback) {
       const { _columns } = chunk
-      self.columns.forEach(column => {
+      columns.forEach(column => {
         const colIndex = _columns.indexOf(column.name)
         chunk[colIndex] = getFaker(column.faker)()
       })
@@ -44,5 +44,5 @@ function FakerTransformer (options) {
 }
 
 module.exports = {
-  FakerTransformer
+  FakerTransformer: fakerTransformer
 }

package/app/transformers/index.js

@@ -1,6 +1,6 @@
-const { ToUpperCaseTransformer } = require('./toUpperCaseTransformer')
-const { FakerTransformer } = require('./fakerTransformer')
-const { StringReplaceTransformer } = require('./stringReplaceTransformer')
+const { ToUpperCaseTransformer } = require('./to-upper-case-transformer')
+const { FakerTransformer } = require('./faker-transformer')
+const { StringReplaceTransformer } = require('./string-replace-transformer')
 
 module.exports = {
   StringReplaceTransformer,

package/app/transformers/{stringReplaceTransformer.js → string-replace-transformer.js}

@@ -6,9 +6,8 @@ const { Transform } = require('node:stream')
  * @param {String} options.column
  * @returns Writable
  */
-function StringReplaceTransformer (options) {
-  const self = this
-  self.replacements = options
+function stringReplaceTransformer (options) {
+  const replacements = options
 
   return new Transform({
     readableObjectMode: true,
@@ -16,8 +15,12 @@ function StringReplaceTransformer (options) {
     transform (chunk, _, callback) {
       const { _columns } = chunk
       // @ts-ignore
-      self.replacements.forEach(r => {
+      replacements.forEach(r => {
         const colIndex = _columns.indexOf(r.column)
+        if (colIndex === -1) {
+          // column not found, skip
+          return
+        }
         if (r.all) {
           chunk[colIndex] = chunk[colIndex].replaceAll(r.find, r.replace)
         } else {
@@ -31,5 +34,5 @@ function StringReplaceTransformer (options) {
 }
 
 module.exports = {
-  StringReplaceTransformer
+  StringReplaceTransformer: stringReplaceTransformer
 }

package/app/transformers/{toUpperCaseTransformer.js → to-upper-case-transformer.js}

@@ -6,22 +6,24 @@ const { Transform } = require('node:stream')
  * @param {String} options.column
  * @returns Writable
  */
-function ToUpperCaseTransformer (options) {
-  const self = this
-  self.column = options.column
+function toUpperCaseTransformer (options) {
+  const column = options.column
 
   return new Transform({
     readableObjectMode: true,
     writableObjectMode: true,
     transform (chunk, _, callback) {
       const { _columns } = chunk
-      const colIndex = _columns.indexOf(self.column)
-      chunk[colIndex] = chunk[colIndex].toUpperCase()
+      const colIndex = _columns.indexOf(column)
+
+      if (colIndex !== -1 && chunk[colIndex] != null) {
+        chunk[colIndex] = chunk[colIndex].toUpperCase()
+      }
       callback(null, chunk)
     }
   })
 }
 
 module.exports = {
-  ToUpperCaseTransformer
+  ToUpperCaseTransformer: toUpperCaseTransformer
 }