ffc-pay-etl-framework 1.4.2-alpha.6 → 1.4.2

package/.npmignore

@@ -0,0 +1,26 @@
+# Ignore dotfiles explicitly by name
+/.all-contributorsrc
+/.dockerignore
+/.gitignore
+/.snyk
+/CODE_OF_CONDUCT.md
+/CONTRIBUTING.md
+/eslint.config.js
+/ISSUE_TEMPLATE.md
+/Jenkinsfile
+/LICENSE.md
+/PULL_REQUEST_TEMPLATE.md
+/sonar-project.properties
+/snyk_report.html
+/snyk-error.log
+/snyk-monitor-result.json
+/snyk-result.json
+
+# Ignore folders recursively
+__mocks__/
+.devcontainer/
+.github/
+jest/
+test/
+snyk-cli/
+test-output/

package/app/database-connections/index.js

@@ -0,0 +1,7 @@
+const { PostgresDatabaseConnection } = require('./postgres-database-connection')
+const { ProvidedConnection } = require('./provided-connection')
+
+module.exports = {
+  PostgresDatabaseConnection,
+  ProvidedConnection
+}

package/app/database-connections/postgres-database-connection.js

@@ -0,0 +1,47 @@
+const DEFAULT_PORT = 5432
+const { Sequelize } = require('sequelize')
+const debug = require('debug')('connection')
+
+/**
+ *
+ * @param {Object} options
+ * @param {Object} options.connectionname
+ * @param {Object} options.username
+ * @param {Object} options.password
+ * @param {Object} options.database
+ * @param {Object} options.host
+ * @param {Object} options.port
+ * @returns Connection
+ */
+async function postgresDatabaseConnection (options) {
+  const connectionname = options.connectionname
+  const username = options.username
+  const password = options.password
+  const database = options.database
+  const host = options.host
+  const port = options.port || DEFAULT_PORT
+
+  const sequelize = new Sequelize(database, username, password, {
+    host,
+    port,
+    dialect: 'postgres',
+    logging: false
+  })
+
+  try {
+    await sequelize.authenticate()
+    debug('sequelize.authenticate succeeded')
+    return {
+      name: connectionname,
+      db: sequelize
+    }
+  } catch (e) {
+    debug('sequelize.authenticate failed')
+    debug(e)
+    throw e
+  }
+}
+
+module.exports = {
+  PostgresDatabaseConnection: postgresDatabaseConnection
+}

package/app/database-connections/provided-connection.js

@@ -0,0 +1,19 @@
+/**
+ *
+ * @param {Object} options
+ * @param {Object} options.connectionname
+ * @param {Object} options.sequelize
+ * @returns Connection
+ */
+async function providedConnection (options) {
+  const connectionname = options.connectionname
+  const sequelize = options.sequelize
+  return {
+    name: connectionname,
+    db: sequelize
+  }
+}
+
+module.exports = {
+  ProvidedConnection: providedConnection
+}

package/app/destinations/README.md

@@ -0,0 +1,107 @@
+# Destinations
+
+Destinations are expected to write out to an output. This could be a file, a database, an endpoint, whatever.
+
+Currently supported destinations are :
+
+- CSVFileDestination
+- PostgresDestination
+- SQLFileDestination
+
+## CSVFileDestination
+
+### Options
+
+| option         | description                                                   |
+| -------------- | ------------------------------------------------------------- |
+| fileName       | absolute path to the csv file to be written                   |
+| headers        | treat the first line of the file as column header information |
+| includeErrors  | write out any rows that contain errors                        |
+| quotationMarks | wrap values in quotation marks                                |
+
+### Usage
+
+```js
+CSVFileDestination({
+  fileName: "...",
+  headers: true,
+  includeErrors: false,
+  quotationMarks: true,
+})
+```
+
+### Example
+
+examples/csv-to-csv.js
+
+## PostgresDestination
+
+### Options
+
+| option        | description                                            |
+| ------------- | ------------------------------------------------------ |
+| table         | the name of the table to be inserted into              |
+| username      | the username for the database connection               |
+| password      | the password for the database connection               |
+| database      | the name of the database for the connection            |
+| host          | the host name for the database connection              |
+| port          | the port for the database connection. Defaults to 5432 |
+| includeErrors | write out any rows that contain errors                 |
+| mapping       | field mappings from source to target                   |
+
+### Usage
+
+```js
+PostgresDestination({
+  username: "postgres",
+  password: "ppp",
+  database: "etl_db",
+  host: "postgres",
+  table: "target",
+  includeErrors: false,
+  mapping: [
+    {
+      column: "column1",
+      targetColumn: "target_column1",
+      targetType: "varchar",
+    },
+  ],
+})
+```
+
+### Example
+
+examples/csv-to-postgres.js
+
+## SQLFileDestination
+
+### options
+
+| option   | description                                 |
+| -------- | ------------------------------------------- |
+| fileName | absolute path to the sql file to be written |
+| mode     | insert or update mode                       |
+| table    | the table for the statements                |
+| mapping  | field mappings from source to target        |
+
+### Usage
+
+```js
+SQLFileDestination({
+  filename: "...",
+  mode: SQL_MODE.INSERT_MODE,
+  table: "target",
+  includeErrors: false,
+  mapping: [
+    {
+      column: "column1",
+      targetColumn: "target_column1",
+      targetType: "varchar",
+    },
+  ],
+})
+```
+
+### Example
+
+examples/csv-to-sqlfile.js

package/app/destinations/console-destination.js

@@ -0,0 +1,38 @@
+const { Writable } = require('node:stream')
+
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.includeErrors
+ * @returns Writable
+ */
+function consoleDestination (options) {
+  const includeErrors = options.includeErrors
+  const writable = new Writable({
+    objectMode: true,
+    write (chunk, _, callback) {
+      if (chunk.errors.length === 0 || includeErrors) { console.log(chunk) }
+      // @ts-ignore
+      this.tasks?.forEach(task => task.write(chunk))
+      callback()
+    }
+  })
+
+  Object.assign(writable, {
+    setConnection: function (connection) {
+      this.connection = connection
+    }.bind(writable),
+    getConnectionName: function () {
+      return this.connection?.name
+    }.bind(writable),
+    setTasks: function (tasks) {
+      this.tasks = tasks
+    }.bind(writable)
+  })
+
+  return writable
+}
+
+module.exports = {
+  ConsoleDestination: consoleDestination
+}

package/app/destinations/csv-file-destination.js

@@ -0,0 +1,94 @@
+const EventEmitter = require('node:events')
+const util = require('node:util')
+const { Writable } = require('node:stream')
+const fs = require('node:fs')
+
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.fileName
+ * @param {Boolean} options.headers
+ * @param {Boolean} options.includeErrors
+ * @param {Boolean} options.quotationMarks
+ * @returns Writable
+ */
+function csvFileDestination (options) {
+  EventEmitter.call(this)
+  let lastChunk
+  const fileName = options.fileName
+  const headers = options.headers
+  const includeErrors = options.includeErrors
+  const quotationMarks = options.quotationMarks
+
+  let fileHandle
+  try {
+    fileHandle = fs.openSync(fileName, 'w+')
+  } catch (error) {
+    throw new Error(`Failed to open file ${fileName}: ${error.message}`)
+  }
+
+  let headersWritten = false
+
+  const writable = new Writable({
+    objectMode: true,
+    write (chunk, _, callback) {
+      try {
+        if (!headersWritten && headers) {
+          if (quotationMarks) {
+            const quotedColumns = chunk._columns.map(c => `"${c}"`).join(',')
+            fs.writeSync(fileHandle, quotedColumns + '\n')
+          } else {
+            fs.writeSync(fileHandle, `${chunk._columns.join(',')}\n`)
+          }
+          headersWritten = true
+        }
+
+        const errors = Array.isArray(chunk.errors) ? chunk.errors : []
+
+        if (errors.length === 0 || includeErrors) {
+          const line = chunk.map(c => `"${c}"`).join(',') + '\n'
+          fs.writeSync(fileHandle, line)
+        }
+
+        if (this.tasks && Array.isArray(this.tasks)) {
+          this.tasks.forEach(task => task.write(chunk))
+        }
+
+        lastChunk = chunk
+        callback()
+      } catch (err) {
+        callback(err)
+      }
+    },
+    final (callback) {
+      if (fileHandle) {
+        fs.closeSync(fileHandle)
+      }
+
+      this.emit('result', lastChunk)
+      callback()
+    }
+  })
+
+  Object.assign(writable, {
+    setConnection: function (connection) {
+      this.connection = connection
+    }.bind(writable),
+
+    getConnectionName: function () {
+      return this.connection?.name
+    }.bind(writable),
+
+    setTasks: function (tasks) {
+      this.tasks = tasks
+    }.bind(writable)
+  })
+
+  return writable
+}
+
+util.inherits(csvFileDestination, EventEmitter)
+
+module.exports = {
+  CSVFileDestination: csvFileDestination
+}

package/app/destinations/index.js

@@ -0,0 +1,12 @@
+const { ConsoleDestination } = require('./console-destination')
+const { SQLFileDestination, SQL_MODE } = require('./sql-file-destination')
+const { CSVFileDestination } = require('./csv-file-destination')
+const { PostgresDestination } = require('./postgres-destination')
+
+module.exports = {
+  ConsoleDestination,
+  SQLFileDestination,
+  SQL_MODE,
+  CSVFileDestination,
+  PostgresDestination
+}

package/app/destinations/postgres-destination.js

@@ -0,0 +1,141 @@
+const EventEmitter = require('node:events')
+const { Transform } = require('node:stream')
+const debug = require('debug')('destination')
+
+function isKeyWord (column) {
+  return ['USER'].includes(column.toUpperCase())
+}
+
+function getMappingForColumn (mapping, column) {
+  if (mapping.length === 0) {
+    return {}
+  }
+  const map = mapping.find(m => m.column === column)
+  return map || {}
+}
+
+function hasReturningColumns (mapping) {
+  return mapping.some(m => m.returning === true)
+}
+
+function getReturningColumns (mapping) {
+  return mapping.filter(m => m.returning === true).map(m => m?.targetColumn)
+}
+
+function writeInsertStatement (columnMapping, table, chunk, schema, ignoredColumns = []) {
+  const filteredColumns = chunk._columns.filter(column => {
+    const mapping = getMappingForColumn(columnMapping, column)
+    const targetColumn = mapping?.targetColumn ? mapping?.targetColumn : mapping?.column
+    return !ignoredColumns.includes(targetColumn)
+  })
+  let statement = `INSERT INTO ${schema ?? 'public'}."${table}" (${filteredColumns.map(column => {
+        const mapping = getMappingForColumn(columnMapping, column)
+        return mapping?.targetColumn ? `"${mapping.targetColumn}"` : `"${mapping?.column}"`
+    })
+        .join(',')}) VALUES (${filteredColumns.map(column => {
+        const index = chunk._columns.indexOf(column)
+        const mapping = getMappingForColumn(columnMapping, column)
+        if (mapping?.targetType === 'number' && (Number.isNaN(Number(chunk[index])) || chunk[index] === '')) {
+          debug('Source data is not a number.')
+          return 0
+        }
+        if (mapping?.targetType === 'varchar' || mapping?.targetType === 'char') {
+            return `'${chunk[index]}'`
+        }
+        if (mapping?.targetType === 'date') {
+            if (!chunk[index]) {
+                return '\'\''
+            }
+            return `to_timestamp('${chunk[index]}','${mapping?.format}')`
+        }
+        return chunk[index] ? chunk[index] : 'null'
+    })})`
+  if (hasReturningColumns(columnMapping)) {
+    statement = statement + ` RETURNING ${getReturningColumns(columnMapping).join(',')}`
+  }
+  return statement
+}
+
+/**
+ *
+ * @param {Object} options
+ * @param {Object} options.table
+ * @param {Object} options.connectionname
+ * @param {Object} options.mapping
+ * @param {Object} options.includeErrors
+ * @param {String} [options.schema]
+ * @param {Array<String>} [options.ignoredColumns]
+ * @returns Transform
+ */
+function postgresDestination (options) {
+  EventEmitter.call(this)
+  const table = options.table
+  const connectionname = options.connectionname
+  const mapping = options.mapping
+  const schema = options.schema
+  const ignoredColumns = options.ignoredColumns ?? []
+  let lastChunk
+
+  const transform = new Transform({
+    objectMode: true,
+    emitClose: true,
+    construct (callback) {
+      // @ts-ignore
+      this.connectionname = connectionname
+      callback()
+    },
+    write (chunk, _, callback) {
+      let insertStatement
+      // @ts-ignore
+      if (chunk.errors.length === 0 || options.includeErrors) {
+        insertStatement = writeInsertStatement(mapping, table, chunk, schema, ignoredColumns)
+        debug('Insert statement: [%s]', insertStatement)
+        // @ts-ignore
+        this.connection.db.query(insertStatement)
+          .then(result => {
+            debug('result %o', result)
+            chunk._result = result
+            lastChunk = chunk
+            // @ts-ignore
+            this.tasks?.forEach(task => task.write(chunk))
+            // @ts-ignore
+            callback(null, chunk)
+          }).catch(error => {
+            debug('error %o', error)
+            chunk.errors.push(error)
+            lastChunk = chunk
+            // @ts-ignore
+            callback(error, chunk)
+          })
+      } else {
+        debug('Chunk has errors %o', chunk)
+      }
+    },
+    final (callback) {
+      this.emit('result', lastChunk)
+      callback()
+    }
+  })
+  Object.assign(Transform.prototype, {
+    type: 'PostgresDestination',
+    setConnection: function (connection) {
+      this.connection = connection
+    }.bind(transform),
+    getConnectionName: function () {
+      return this.connection?.name
+    }.bind(transform),
+    setTasks: function (tasks) {
+      this.tasks = tasks
+    }.bind(transform)
+  })
+  return transform
+}
+
+module.exports = {
+  PostgresDestination: postgresDestination,
+  writeInsertStatement,
+  isKeyWord,
+  getMappingForColumn,
+  hasReturningColumns,
+  getReturningColumns
+}

package/app/destinations/sql-file-destination.js

@@ -0,0 +1,68 @@
+const { Writable } = require('node:stream')
+const fs = require('node:fs')
+
+/**
+ * @enum {number}
+ */
+const SQL_MODE = {
+  INSERT_MODE: 1,
+  UPDATE_MODE: 2
+}
+
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.fileName
+ * @param {SQL_MODE} options.mode
+ * @param {String} options.table
+ * @param {Array} options.mapping
+ * @param {Boolean} options.includeErrors
+ * @returns Transform
+ */
+function sqlFileDestination (options) {
+  const fileName = options.fileName
+  const sqlMode = options.mode
+  const table = options.table
+  const mapping = options.mapping
+
+  function writeInsertStatement (chunk) {
+    const statement = `INSERT INTO ${table} (${mapping.map(m => m.targetColumn)
+        .join(',')}) VALUES (${mapping.map((m) => {
+            const srcColumnIndex = chunk._columns.indexOf(m.column)
+            if (m.targetType === 'string') { return `'${chunk[srcColumnIndex]}'` }
+            return chunk[srcColumnIndex]
+        })});\n`
+    fs.writeFileSync(fileName, statement, {
+      encoding: 'utf8',
+      flag: 'a+'
+    })
+  }
+  const writable = new Writable({
+    objectMode: true,
+    write (chunk, _, callback) {
+      if (chunk.errors.length === 0 && sqlMode === SQL_MODE.INSERT_MODE) {
+        writeInsertStatement(chunk)
+      }
+      callback()
+    }
+  })
+
+  Object.assign(writable, {
+    setConnection: function (connection) {
+      this.connection = connection
+    }.bind(writable),
+    getConnectionName: function () {
+      return this.connection?.name
+    }.bind(writable),
+    setTasks: function (tasks) {
+      this.tasks = tasks
+    }.bind(writable)
+  })
+
+  return writable
+}
+
+module.exports = {
+  SQLFileDestination: sqlFileDestination,
+  SQL_MODE
+}

package/app/lib/index.js

@@ -0,0 +1,103 @@
+const EventEmitter = require('node:events')
+const { RowMetaData } = require('./row-meta-data')
+const { compose } = require('node:stream')
+
+/**
+ * @typedef {Object} Etl
+ * @function loader
+ * @function pump
+ * @function connection
+ * @function validator
+ * @function destination
+ * @function transform
+ */
+
+/**
+ *
+ * @returns Etl
+ */
+class Etl extends EventEmitter {
+  constructor () {
+    super()
+    this.store = []
+    this.beforeETLList = []
+    this.connectionList = []
+    this.validatorList = []
+    this.transformationList = []
+    this.destinationList = []
+  }
+
+  loader (loader) {
+    this.loader = loader
+    return this
+  }
+
+  pump () {
+    this.beforeETLList.forEach(task => {
+      task.write({})
+    })
+
+    this.loader
+      .pump(this.loader)
+      .pipe(
+        compose(
+          RowMetaData(),
+          ...this.validatorList,
+          ...this.transformationList,
+          ...this.destinationList.map(dl => dl.on('result', data => this.emit('result', data)))
+        )
+      )
+      .on('error', err => this.emit('error', err))
+    return this
+  }
+
+  beforeETL (pipelineTask) {
+    const connectionname = pipelineTask.getConnectionName()
+    const connection = this.connectionList.find(c => c.name === connectionname)
+    if (!connection) {
+      throw new Error(`Connection with name ${connectionname} not found`)
+    }
+    pipelineTask.setConnection(connection)
+    pipelineTask.setETL(this)
+    this.beforeETLList.push(pipelineTask)
+    return this
+  }
+
+  connection (connection) {
+    this.connectionList.push(connection)
+    return this
+  }
+
+  validator (validator) {
+    this.validatorList.push(validator)
+    return this
+  }
+
+  destination (destination, ...tasks) {
+    const connectionname = destination.getConnectionName()
+    const connection = this.connectionList.find(c => c.name === connectionname)
+    if (!connection && destination.type === 'PostgresDestination') {
+      throw new Error(`No connection could be found with name ${connectionname}`)
+    } else {
+      destination.setConnection(connection)
+    }
+
+    if (tasks && tasks.length > 0) {
+      for (const task of tasks) {
+        task.setETL(this)
+      }
+      destination.setTasks(tasks)
+    }
+    this.destinationList.push(destination)
+    return this
+  }
+
+  transform (transform) {
+    this.transformationList.push(transform)
+    return this
+  }
+}
+
+module.exports = {
+  Etl
+}

package/app/lib/row-meta-data.js

@@ -0,0 +1,24 @@
+// @ts-nocheck
+const { Transform } = require('node:stream')
+
+function rowMetaData () {
+  return new Transform({
+    readableObjectMode: true,
+    writableObjectMode: true,
+    decodeStrings: false,
+    construct (callback) {
+      this.rowId = 0
+      callback()
+    },
+    transform (chunk, _, callback) {
+      chunk['_rowId'] = this.rowId
+      chunk['errors'] = []
+      this.rowId += 1
+      callback(null, chunk)
+    }
+  })
+}
+
+module.exports = {
+  RowMetaData: rowMetaData
+}