ffc-pay-etl-framework 1.4.1-alpha.5 → 1.4.1

package/app/database_connections/index.js

@@ -0,0 +1,7 @@
+const { PostgresDatabaseConnection } = require('./postgresDatabaseConnection')
+const { ProvidedConnection } = require('./providedConnection')
+
+module.exports = {
+  PostgresDatabaseConnection,
+  ProvidedConnection
+}

package/app/database_connections/postgresDatabaseConnection.js

@@ -0,0 +1,47 @@
+const DEFAULT_PORT = 5432
+const { Sequelize } = require('sequelize')
+const debug = require('debug')('connection')
+
+/**
+ *
+ * @param {Object} options
+ * @param {Object} options.connectionname
+ * @param {Object} options.username
+ * @param {Object} options.password
+ * @param {Object} options.database
+ * @param {Object} options.host
+ * @param {Object} options.port
+ * @returns Connection
+ */
+async function PostgresDatabaseConnection (options) {
+  const connectionname = options.connectionname
+  const username = options.username
+  const password = options.password
+  const database = options.database
+  const host = options.host
+  const port = options.port || DEFAULT_PORT
+
+  const sequelize = new Sequelize(database, username, password, {
+    host,
+    port,
+    dialect: 'postgres',
+    logging: false
+  })
+
+  try {
+    await sequelize.authenticate()
+    debug('sequelize.authenticate succeeded')
+    return {
+      name: connectionname,
+      db: sequelize
+    }
+  } catch (e) {
+    debug('sequelize.authenticate failed')
+    debug(e)
+    throw e
+  }
+}
+
+module.exports = {
+  PostgresDatabaseConnection
+}

package/app/database_connections/providedConnection.js

@@ -0,0 +1,19 @@
+/**
+ *
+ * @param {Object} options
+ * @param {Object} options.connectionname
+ * @param {Object} options.sequelize
+ * @returns Connection
+ */
+async function ProvidedConnection (options) {
+  const connectionname = options.connectionname
+  const sequelize = options.sequelize
+  return {
+    name: connectionname,
+    db: sequelize
+  }
+}
+
+module.exports = {
+  ProvidedConnection
+}

package/app/destinations/README.md

@@ -0,0 +1,107 @@
+# Destinations
+
+Destinations are expected to write out to an output. This could be a file, a database, an endpoint, whatever.
+
+Currently supported destinations are :
+
+- CSVFileDestination
+- PostgresDestination
+- SQLFileDestination
+
+## CSVFileDestination
+
+### Options
+
+| option         | description                                                   |
+| -------------- | ------------------------------------------------------------- |
+| fileName       | absolute path to the csv file to be written                   |
+| headers        | treat the first line of the file as column header information |
+| includeErrors  | write out any rows that contain errors                        |
+| quotationMarks | wrap values in quotation marks                                |
+
+### Usage
+
+```js
+CSVFileDestination({
+  fileName: "...",
+  headers: true,
+  includeErrors: false,
+  quotationMarks: true,
+})
+```
+
+### Example
+
+examples/csv-to-csv.js
+
+## PostgresDestination
+
+### Options
+
+| option        | description                                            |
+| ------------- | ------------------------------------------------------ |
+| table         | the name of the table to be inserted into              |
+| username      | the username for the database connection               |
+| password      | the password for the database connection               |
+| database      | the name of the database for the connection            |
+| host          | the host name for the database connection              |
+| port          | the port for the database connection. Defaults to 5432 |
+| includeErrors | write out any rows that contain errors                 |
+| mapping       | field mappings from source to target                   |
+
+### Usage
+
+```js
+PostgresDestination({
+  username: "postgres",
+  password: "ppp",
+  database: "etl_db",
+  host: "postgres",
+  table: "target",
+  includeErrors: false,
+  mapping: [
+    {
+      column: "column1",
+      targetColumn: "target_column1",
+      targetType: "varchar",
+    },
+  ],
+})
+```
+
+### Example
+
+examples/csv-to-postgres.js
+
+## SQLFileDestination
+
+### options
+
+| option   | description                                 |
+| -------- | ------------------------------------------- |
+| fileName | absolute path to the sql file to be written |
+| mode     | insert or update mode                       |
+| table    | the table for the statements                |
+| mapping  | field mappings from source to target        |
+
+### Usage
+
+```js
+SQLFileDestination({
+  filename: "...",
+  mode: SQL_MODE.INSERT_MODE,
+  table: "target",
+  includeErrors: false,
+  mapping: [
+    {
+      column: "column1",
+      targetColumn: "target_column1",
+      targetType: "varchar",
+    },
+  ],
+})
+```
+
+### Example
+
+examples/csv-to-sqlfile.js

package/app/destinations/consoleDestination.js

@@ -0,0 +1,38 @@
+const { Writable } = require('node:stream')
+
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.includeErrors
+ * @returns Writable
+ */
+function ConsoleDestination (options) {
+  const includeErrors = options.includeErrors
+  const writable = new Writable({
+    objectMode: true,
+    write (chunk, _, callback) {
+      if (chunk.errors.length === 0 || includeErrors) { console.log(chunk) }
+      // @ts-ignore
+      this.tasks?.forEach(task => task.write(chunk))
+      callback()
+    }
+  })
+
+  Object.assign(writable, {
+    setConnection: function (connection) {
+      this.connection = connection
+    }.bind(writable),
+    getConnectionName: function () {
+      return this.connection?.name
+    }.bind(writable),
+    setTasks: function (tasks) {
+      this.tasks = tasks
+    }.bind(writable)
+  })
+
+  return writable
+}
+
+module.exports = {
+  ConsoleDestination
+}

package/app/destinations/csvFileDestination.js

@@ -0,0 +1,75 @@
+const EventEmitter = require('node:events')
+const util = require('node:util')
+const { Writable } = require('node:stream')
+const fs = require('fs')
+
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.fileName
+ * @param {Boolean} options.headers
+ * @param {Boolean} options.includeErrors
+ * @param {Boolean} options.quotationMarks
+ * @returns Writable
+ */
+function CSVFileDestination (options) {
+  EventEmitter.call(this)
+  let lastChunk
+  const fileName = options.fileName
+  const headers = options.headers
+  const includeErrors = options.includeErrors
+  const quotationMarks = options.quotationMarks
+  let fileHandle
+  fs.open(fileName, 'w+', (_error, fd) => {
+    fileHandle = fd
+  })
+  let headersWritten = false
+  const writable = new Writable({
+    objectMode: true,
+    write (chunk, _, callback) {
+      if (!headersWritten && headers) {
+        if (quotationMarks) {
+          fs.writeFileSync(fileHandle, `${chunk._columns.map(c => `"${c}"`).join(',')}\n`)
+        } else {
+          fs.writeFileSync(fileHandle, `${chunk._columns.join(',')}\n`)
+        }
+        headersWritten = true
+      }
+      if (chunk.errors.length === 0 || includeErrors) {
+        if (quotationMarks) {
+          fs.writeFileSync(fileHandle, `${chunk.map(c => `"${c}"`).join(',')}\n`)
+        } else {
+          fs.writeFileSync(fileHandle, `${chunk.map(c => `"${c}"`).join(',')}\n`)
+        }
+      }
+      // @ts-ignore
+      this.tasks?.forEach(task => task.write(chunk))
+      lastChunk = chunk
+      callback()
+    },
+    final (callback) {
+      this.emit('result', lastChunk)
+      callback()
+    }
+  })
+
+  Object.assign(writable, {
+    setConnection: function (connection) {
+      this.connection = connection
+    }.bind(writable),
+    getConnectionName: function () {
+      return this.connection?.name
+    }.bind(writable),
+    setTasks: function (tasks) {
+      this.tasks = tasks
+    }.bind(writable)
+  })
+
+  return writable
+}
+
+util.inherits(CSVFileDestination, EventEmitter)
+
+module.exports = {
+  CSVFileDestination
+}

package/app/destinations/index.js

@@ -0,0 +1,12 @@
+const { ConsoleDestination } = require('./consoleDestination')
+const { SQLFileDestination, SQL_MODE } = require('./sqlFileDestination')
+const { CSVFileDestination } = require('./csvFileDestination')
+const { PostgresDestination } = require('./postgresDestination')
+
+module.exports = {
+  ConsoleDestination,
+  SQLFileDestination,
+  SQL_MODE,
+  CSVFileDestination,
+  PostgresDestination
+}

package/app/destinations/postgresDestination.js

@@ -0,0 +1,139 @@
+const EventEmitter = require('node:events')
+const { Transform } = require('node:stream')
+const debug = require('debug')('destination')
+
+function isKeyWord (column) {
+  return ['USER'].includes(column.toUpperCase())
+}
+
+function getMappingForColumn (mapping, column) {
+  if (mapping.length === 0) { return {} }
+  const [map] = mapping.filter(m => m.column === column)
+  return map
+}
+
+function hasReturningColumns (mapping) {
+  return mapping.filter(m => m.returning === true).length > 0
+}
+
+function getReturningColumns (mapping) {
+  return mapping.filter(m => m.returning === true).map(m => m?.targetColumn)
+}
+
+function writeInsertStatement (columnMapping, table, chunk, schema, ignoredColumns = []) {
+  const filteredColumns = chunk._columns.filter(column => {
+    const mapping = getMappingForColumn(columnMapping, column)
+    const targetColumn = mapping?.targetColumn ? mapping?.targetColumn : mapping?.column
+    return !ignoredColumns.includes(targetColumn)
+  })
+  let statement = `INSERT INTO ${schema ?? 'public'}."${table}" (${filteredColumns.map(column => {
+        const mapping = getMappingForColumn(columnMapping, column)
+        return mapping?.targetColumn ? `"${mapping.targetColumn}"` : `"${mapping?.column}"`
+    })
+        .join(',')}) VALUES (${filteredColumns.map(column => {
+        const index = chunk._columns.indexOf(column)
+        const mapping = getMappingForColumn(columnMapping, column)
+        if (mapping?.targetType === 'number' && (isNaN(chunk[index]) || chunk[index] === '')) {
+            debug('Source data is not a number.')
+            return 0
+        }
+        if (mapping?.targetType === 'varchar' || mapping?.targetType === 'char') {
+            return `'${chunk[index]}'`
+        }
+        if (mapping?.targetType === 'date') {
+            if (!chunk[index]) {
+                return '\'\''
+            }
+            return `to_timestamp('${chunk[index]}','${mapping?.format}')`
+        }
+        return chunk[index] ? chunk[index] : 'null'
+    })})`
+  if (hasReturningColumns(columnMapping)) {
+    statement = statement + ` RETURNING ${getReturningColumns(columnMapping).join(',')}`
+  }
+  return statement
+}
+
+/**
+ *
+ * @param {Object} options
+ * @param {Object} options.table
+ * @param {Object} options.connectionname
+ * @param {Object} options.mapping
+ * @param {Object} options.includeErrors
+ * @param {String} [options.schema]
+ * @param {Array<String>} [options.ignoredColumns]
+ * @returns Transform
+ */
+function PostgresDestination (options) {
+  EventEmitter.call(this)
+  const table = options.table
+  const connectionname = options.connectionname
+  const mapping = options.mapping
+  const schema = options.schema
+  const ignoredColumns = options.ignoredColumns ?? []
+  let lastChunk
+
+  const transform = new Transform({
+    objectMode: true,
+    emitClose: true,
+    construct (callback) {
+      // @ts-ignore
+      this.connectionname = connectionname
+      callback()
+    },
+    write (chunk, _, callback) {
+      let insertStatement
+      // @ts-ignore
+      if (chunk.errors.length === 0 || options.includeErrors) {
+        insertStatement = writeInsertStatement(mapping, table, chunk, schema, ignoredColumns)
+        debug('Insert statement: [%s]', insertStatement)
+        // @ts-ignore
+        this.connection.db.query(insertStatement)
+          .then(result => {
+            debug('result %o', result)
+            chunk._result = result
+            lastChunk = chunk
+            // @ts-ignore
+            this.tasks?.forEach(task => task.write(chunk))
+            // @ts-ignore
+            callback(null, chunk)
+          }).catch(error => {
+            debug('error %o', error)
+            chunk.errors.push(error)
+            lastChunk = chunk
+            // @ts-ignore
+            callback(error, chunk)
+          })
+      } else {
+        debug('Chunk has errors %o', chunk)
+      }
+    },
+    final (callback) {
+      this.emit('result', lastChunk)
+      callback()
+    }
+  })
+  Object.assign(Transform.prototype, {
+    type: 'PostgresDestination',
+    setConnection: function (connection) {
+      this.connection = connection
+    }.bind(transform),
+    getConnectionName: function () {
+      return this.connection?.name
+    }.bind(transform),
+    setTasks: function (tasks) {
+      this.tasks = tasks
+    }.bind(transform)
+  })
+  return transform
+}
+
+module.exports = {
+  PostgresDestination,
+  writeInsertStatement,
+  isKeyWord,
+  getMappingForColumn,
+  hasReturningColumns,
+  getReturningColumns
+}

package/app/destinations/sqlFileDestination.js

@@ -0,0 +1,81 @@
+const { Writable } = require('node:stream')
+const fs = require('fs')
+
+/**
+ * @enum {number}
+ */
+const SQL_MODE = {
+  INSERT_MODE: 1,
+  UPDATE_MODE: 2
+}
+
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.fileName
+ * @param {SQL_MODE} options.mode
+ * @param {String} options.table
+ * @param {Array} options.mapping
+ * @param {Boolean} options.includeErrors
+ * @returns Transform
+ */
+function SQLFileDestination (options) {
+  const fileName = options.fileName
+  const sqlMode = options.mode
+  const table = options.table
+  const mapping = options.mapping
+  const includeErrors = options.includeErrors
+
+  function writeInsertStatement (chunk) {
+    const statement = `INSERT INTO ${table} (${mapping.map(m => m.targetColumn)
+        .join(',')}) VALUES (${mapping.map((m) => {
+            const srcColumnIndex = chunk._columns.indexOf(m.column)
+            if (m.targetType === 'string') { return `'${chunk[srcColumnIndex]}'` }
+            return chunk[srcColumnIndex]
+        })});\n`
+    fs.writeFileSync(fileName, statement, {
+      encoding: 'utf8',
+      flag: 'a+'
+    })
+  }
+  const writable = new Writable({
+    objectMode: true,
+    write (chunk, _, callback) {
+      if (chunk.errors.length === 0) {
+        if (sqlMode === SQL_MODE.INSERT_MODE) {
+        // @ts-ignore
+          if (chunk.errors.length === 0 | includeErrors) {
+            writeInsertStatement(chunk)
+          }
+        }
+      }
+      // TODO - Disabled because Sonar
+      // } else if (sqlMode === SQL_MODE.UPDATE_MODE){
+      //     // @ts-ignore
+      //     if(chunk.errors.length === 0 | includeErrors){
+      //         writeUpdateStatement(chunk)
+      //     }
+      // }
+      callback()
+    }
+  })
+
+  Object.assign(writable, {
+    setConnection: function (connection) {
+      this.connection = connection
+    }.bind(writable),
+    getConnectionName: function () {
+      return this.connection?.name
+    }.bind(writable),
+    setTasks: function (tasks) {
+      this.tasks = tasks
+    }.bind(writable)
+  })
+
+  return writable
+}
+
+module.exports = {
+  SQLFileDestination,
+  SQL_MODE
+}

package/app/lib/index.js

@@ -0,0 +1,106 @@
+// @ts-nocheck
+const EventEmitter = require('node:events')
+const util = require('node:util')
+const { RowMetaData } = require('./rowMetaData')
+const { compose } = require('node:stream')
+
+/**
+ * @typedef {Object} Etl
+ * @function loader
+ * @function pump
+ * @function connection
+ * @function validator
+ * @function destination
+ * @function transform
+ */
+
+/**
+ *
+ * @returns Etl
+ */
+function Etl () {
+  EventEmitter.call(this)
+  const self = this
+  self.store = []
+  self.beforeETLList = []
+  self.connectionList = []
+  self.validatorList = []
+  self.transformationList = []
+  self.destinationList = []
+
+  this.loader = (loader) => {
+    self.loader = loader
+    return self
+  }
+
+  this.pump = () => {
+    this.beforeETLList.forEach(task => {
+      task.write({})
+    })
+    this.loader
+      .pump(this.loader)
+      .pipe(
+        compose(
+          RowMetaData(),
+          ...self.validatorList,
+          ...self.transformationList,
+          ...self.destinationList.map(dl => dl.on('result', (data) => self.emit('result', data)))
+        )
+        // @ts-ignore
+      )
+      .on('error', (err) => self.emit('error', err))
+    return self
+  }
+
+  this.beforeETL = (pipelineTask) => {
+    const connectionname = pipelineTask.getConnectionName()
+    const connection = this.connectionList.filter(c => c.name === connectionname)[0]
+    if (!connection) {
+      throw new Error(`Connection with name ${connectionname} not found`)
+    }
+    pipelineTask.setConnection(connection)
+    pipelineTask.setETL(self)
+    self.beforeETLList.push(pipelineTask)
+    return self
+  }
+
+  this.connection = (connection) => {
+    self.connectionList.push(connection)
+    return self
+  }
+
+  this.validator = (validator) => {
+    self.validatorList.push(validator)
+    return self
+  }
+
+  this.destination = (destination, ...tasks) => {
+    const connectionname = destination.getConnectionName()
+    const connection = this.connectionList.filter(c => c.name === connectionname)[0]
+    if (!connection && destination.type === 'PostgresDestination') {
+      throw new Error(`No connection could be found with name ${connectionname}`)
+    } else {
+      destination.setConnection(connection)
+    }
+
+    if (tasks) {
+      for (const task of tasks) {
+        task.setETL(self)
+      }
+      destination.setTasks(tasks)
+    }
+    self.destinationList.push(destination)
+    return self
+  }
+
+  this.transform = (transform) => {
+    self.transformationList.push(transform)
+    return self
+  }
+}
+
+util.inherits(Etl, EventEmitter)
+
+module.exports = {
+  Etl
+}

package/app/lib/rowMetaData.js

@@ -0,0 +1,24 @@
+// @ts-nocheck
+const { Transform } = require('stream')
+
+function RowMetaData (options) {
+  return new Transform({
+    readableObjectMode: true,
+    writableObjectMode: true,
+    decodeStrings: false,
+    construct (callback) {
+      this.rowId = 0
+      callback()
+    },
+    transform (chunk, _, callback) {
+      chunk['_rowId'] = this.rowId
+      chunk['errors'] = []
+      this.rowId += 1
+      callback(null, chunk)
+    }
+  })
+}
+
+module.exports = {
+  RowMetaData
+}