ffc-pay-etl-framework 1.4.0 → 1.4.1-alpha.13

package/app/loaders/csvloader.js

@@ -0,0 +1,58 @@
+// @ts-nocheck
+const fs = require('fs')
+const { Transform } = require('stream')
+const { parse } = require('csv-parse')
+
+/**
+ *
+ * @param {Object} options
+ * @param {String} [options.path]
+ * @param {ReadableStream} [options.stream]
+ * @param {Number} [options.startingLine]
+ * @param {Boolean} [options.relax]
+ * @returns StreamReader
+ */
+function CSVLoader (options) {
+  let csvLoader
+  if (options.path) {
+    csvLoader = fs.createReadStream(options.path)
+  } else {
+    csvLoader = options.stream
+  }
+  let lineCount = 1
+  const fromLine = options.startingLine ?? 2
+  const relaxQuotes = options.relax ?? false
+  csvLoader._columns = options.columns
+  csvLoader.pump = (csvLoader) => {
+    const parser = parse({ delimiter: ',', from_line: fromLine, relax_quotes: relaxQuotes })
+    const transformer = new Transform({
+      readableObjectMode: true,
+      writableObjectMode: true,
+      emitClose: true,
+      transform (chunk, _, callback) {
+        chunk['_columns'] = options.columns
+        chunk['_linecount'] = lineCount
+        lineCount += 1
+
+        // remove non-printable characters
+        options.columns.forEach((_column, index) => {
+          if (chunk[index]) {
+            // eslint-disable-next-line no-control-regex
+            chunk[index] = chunk[index].replace(/[\x00-\x1F\x7F-\x9F]/g, '')
+          }
+        })
+
+        callback(null, chunk)
+      }
+    })
+    return csvLoader
+      .pipe(parser)
+      .pipe(transformer)
+  }
+
+  return csvLoader
+}
+
+module.exports = {
+  CSVLoader
+}

package/app/loaders/index.js

@@ -0,0 +1,5 @@
+const { CSVLoader } = require('./csvloader')
+
+module.exports = {
+  CSVLoader
+}

package/app/misc/README.md

@@ -0,0 +1,55 @@
+# Miscellanous pipeline items
+
+## PostgresSQLTask
+
+Executes an arbitrary sql task either as part of another transformation or as an event
+
+### Example
+
+```js
+const {
+  Connections,
+  Destinations,
+  Etl,
+  Loaders,
+  Misc,
+  Validators,
+  Transformers,
+} = require("ffc-pay-etl-framework");
+
+const etl = new Etl.Etl();
+
+etl
+  .connections(
+    new Connections.PostgresSQLConnection({
+      connectionname: "MyConnection",
+      database: "MyDB",
+      username: "MyUserName",
+      password: "MyPassword",
+      host: "localhost",
+      port: 5432,
+    })
+  )
+  .beforeETL(
+    new Misc.PostgresSQLTask({
+      connection: "MyConnection",
+      sql: "INSERT INTO MyTable (column1, column2) VALUES ('apples','oranges');",
+    })
+  )
+  .loader(new Loaders.CSVLoader({ path: csvFile, columns: columns }))
+  .destination(
+    new Destinations.PostgresDestination({
+      connection: "MyConnection",
+      table: "target",
+      mapping: [
+        {
+          column: "Dist Code",
+          targetColumn: "dist_code",
+          targetType: "string",
+        },
+      ],
+      includeErrors: false,
+    })
+  )
+  .pump();
+```

package/app/misc/postgresSQLTask.js

@@ -0,0 +1,71 @@
+// @ts-nocheck
+const { PassThrough } = require('stream')
+const startPosOffset = 3
+const endPosOffset = 1
+
+function getPlaceHolders (sql) {
+  const columnNamePlaceholderRegex = /(?<=\$\{)(.*?)(?=})/g
+  const matches = Array.from(sql.matchAll(columnNamePlaceholderRegex))
+  return matches.map(match => ({
+    match: match[0],
+    value: match[0].split('.')[1],
+    collection: match[0].split('.')[0],
+    startPos: match.index - startPosOffset,
+    endPos: match.index + match[0].length - endPosOffset
+  }))
+}
+
+function doPlaceHolderValueInterpolations (chunk, sql, placeholders) {
+  placeholders.forEach(placeholder => {
+    sql = sql.replace(new RegExp(`\\$\{${placeholder.match}}`),
+      chunk[chunk[`_${placeholder.collection}`].indexOf(placeholder.value)])
+  })
+  return sql
+}
+
+function PostgresSQLTask (options) {
+  const passthrough = new PassThrough({
+    readableObjectMode: true,
+    writableObjectMode: true,
+    decodeStrings: false,
+    construct (callback) {
+      this.sql = options.sql
+      this.connectionname = options.connectionname
+      callback()
+    },
+    transform (chunk, _, callback) {
+      const placeholders = getPlaceHolders(this.sql)
+      if (placeholders.length === 0) {
+        this.connection.db.query(this.sql)
+      } else {
+        const interpolatedSql = doPlaceHolderValueInterpolations(chunk, this.sql, placeholders)
+        // TODO add more interpolation mechanisms to specify return values
+        // e.g. 'myReturnVal = SELECT MAX ID FROM TABLE;'
+        // and write to etl.store.myReturnVal or
+        // e.g. 'chunk.myReturnVal = SELECT MAX ID FROM TABLE;'
+        // and write to the chunk in say chunk.store.myReturnVal
+        this.connection.db.query(interpolatedSql)
+      }
+      callback(null, chunk)
+    }
+  })
+  // Should definately split this out into a mixin
+  Object.assign(passthrough, {
+    setConnection: function (connection) {
+      this.connection = connection
+    }.bind(passthrough),
+    getConnectionName: function () {
+      return this.connection.name
+    }.bind(passthrough),
+    setETL: function (etl) {
+      this.etl = etl
+    }.bind(passthrough)
+  })
+  return passthrough
+}
+
+module.exports = {
+  PostgresSQLTask,
+  getPlaceHolders,
+  doPlaceHolderValueInterpolations
+}

package/app/transformers/README.md

@@ -0,0 +1,102 @@
+# Transformers
+
+Transformers manipulate row data and either replace the existing values or add new columns to each row depending on the type of transformer and the configuration.
+
+Currently supported transformers are :
+
+- ToUpperCaseTransformer
+- FakerTransformer
+
+## FakerTransformer
+
+### Options
+
+| option  | description                                                  |
+| ------- | ------------------------------------------------------------ |
+| columns | Array of column transformations                              |
+| locale  | See https://fakerjs.dev/guide/localization for valid locales |
+
+### Usage
+
+```js
+FakerTransformer({
+  columns: [
+    {
+      name: "column2",
+      faker: "company.name",
+    },
+  ],
+});
+```
+
+faker is a string value that can be any of the currently supported [apis](https://fakerjs.dev/api/)
+
+e.g.
+
+- finance.accountName
+- person.firstName
+- randomizer.next
+
+To create UK post code
+
+```js
+FakerTransformer({
+  columns: [
+    {
+      name: "column2",
+      faker: "location.zipCode",
+    },
+  ],
+  locale: "en_GB",
+});
+```
+
+### Example
+
+examples/csv-faker.js
+
+## ToUpperCaseTransformer
+
+### Options
+
+| option | description                     |
+| ------ | ------------------------------- |
+| column | source column to be transformed |
+
+### Usage
+
+```js
+ToUpperCaseTransformer({
+  column: "column2",
+});
+```
+
+### Example
+
+examples/csv-to-upper-case.js
+
+## StringReplaceTransformer
+
+### Options
+
+| option  | description                             |
+| ------- | --------------------------------------- |
+| column  | source column to be transformed         |
+| find    | search string to be replaced            |
+| replace | value to replace the search string with |
+
+### Usage
+
+```js
+StringReplaceTransformer([
+  {
+    column: "column2",
+    find: "'",
+    replace: "''",
+  },
+]);
+```
+
+### Example
+
+examples/csv-find-replace.js

package/app/transformers/fakerTransformer.js

@@ -0,0 +1,48 @@
+const { Transform } = require('node:stream')
+
+/**
+ *
+ * @param {Object} options
+ * @param {Array} options.columns {
+ *           columns: [
+ *               {
+ *                   name: "column-name",
+ *                   faker: "company.name"
+ *               }
+ *           ]
+ *       }
+ * @param {String} options.locale
+ * @returns StreamReader
+ */
+function FakerTransformer (options) {
+  const self = this
+  self.columns = options.columns
+  let faker
+  if (options.locale) {
+    faker = require(`@faker-js/faker/locale/${options.locale}`).faker
+  } else {
+    faker = require('@faker-js/faker').faker
+  }
+
+  function getFaker (fakerType) {
+    return fakerType.split('.').reduce((a, b) => {
+      return Object.keys(a).length === 0 ? faker[b] : a[b]
+    }, {})
+  }
+  return new Transform({
+    readableObjectMode: true,
+    writableObjectMode: true,
+    transform (chunk, _, callback) {
+      const { _columns } = chunk
+      self.columns.forEach(column => {
+        const colIndex = _columns.indexOf(column.name)
+        chunk[colIndex] = getFaker(column.faker)()
+      })
+      callback(null, chunk)
+    }
+  })
+}
+
+module.exports = {
+  FakerTransformer
+}

package/app/transformers/index.js

@@ -0,0 +1,9 @@
+const { ToUpperCaseTransformer } = require('./toUpperCaseTransformer')
+const { FakerTransformer } = require('./fakerTransformer')
+const { StringReplaceTransformer } = require('./stringReplaceTransformer')
+
+module.exports = {
+  StringReplaceTransformer,
+  ToUpperCaseTransformer,
+  FakerTransformer
+}

package/app/transformers/stringReplaceTransformer.js

@@ -0,0 +1,35 @@
+const { Transform } = require('node:stream')
+
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.column
+ * @returns Writable
+ */
+function StringReplaceTransformer (options) {
+  const self = this
+  self.replacements = options
+
+  return new Transform({
+    readableObjectMode: true,
+    writableObjectMode: true,
+    transform (chunk, _, callback) {
+      const { _columns } = chunk
+      // @ts-ignore
+      self.replacements.forEach(r => {
+        const colIndex = _columns.indexOf(r.column)
+        if (r.all) {
+          chunk[colIndex] = chunk[colIndex].replaceAll(r.find, r.replace)
+        } else {
+          chunk[colIndex] = chunk[colIndex].replace(r.find, r.replace)
+        }
+      })
+
+      callback(null, chunk)
+    }
+  })
+}
+
+module.exports = {
+  StringReplaceTransformer
+}

package/app/transformers/toUpperCaseTransformer.js

@@ -0,0 +1,27 @@
+const { Transform } = require('node:stream')
+
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.column
+ * @returns Writable
+ */
+function ToUpperCaseTransformer (options) {
+  const self = this
+  self.column = options.column
+
+  return new Transform({
+    readableObjectMode: true,
+    writableObjectMode: true,
+    transform (chunk, _, callback) {
+      const { _columns } = chunk
+      const colIndex = _columns.indexOf(self.column)
+      chunk[colIndex] = chunk[colIndex].toUpperCase()
+      callback(null, chunk)
+    }
+  })
+}
+
+module.exports = {
+  ToUpperCaseTransformer
+}

package/app/validators/README.md

@@ -0,0 +1,81 @@
+# Validators
+
+Validators are expected to check specific columns within each row against specific validation rules
+
+Currently supported destinations are :
+
+- RequiredValidator
+- UniqueValidator
+- MultiToolValidator
+
+## RequiredValidator
+
+### Options
+
+| option         | description                                                   |
+| -------------- | ------------------------------------------------------------- |
+| columns        | Array of column indexes                                       |
+
+### Usage
+
+```js
+Validators.RequiredValidator({
+    columns: [0] 
+})
+```
+
+### Example
+
+examples/csv-required-validator.js
+
+## UniqueValidator
+
+### Options
+
+| option         | description                                                   |
+| -------------- | ------------------------------------------------------------- |
+| column         | Column to be tested for uniqueness                            |
+
+### Usage
+
+```js
+Validators.UniqueValidator(
+    { column: "field_one" }
+)
+```
+
+### Example
+
+examples/csv-unique-validator.js
+
+## MultiToolValidator
+
+### Options
+
+| option         | description                                                   |
+| -------------- | ------------------------------------------------------------- |
+| column         | Column to be tested for validity and validator specification  |
+
+NB Any validators specified [here](https://github.com/validatorjs/validator.js) can be used
+
+### Usage
+
+```js
+Validators.MultiToolValidator( { columns: [
+    {
+        name: "SOIL TYPE PERCENT (Percent)",
+        validator: "isAlphanumeric",
+        args: {
+                locale: "en-GB",
+                options: {
+                    ignore: "-;%/ "
+                }
+        }
+    }
+]
+})
+```
+
+### Example
+
+examples/csv-multi-tool-validator.js

package/app/validators/index.js

@@ -0,0 +1,9 @@
+const { RequiredValidator } = require('./requiredValidator')
+const { UniqueValidator } = require('./uniqueValidator')
+const { MultiToolValidator } = require('./multiToolValidator')
+
+module.exports = {
+  RequiredValidator,
+  UniqueValidator,
+  MultiToolValidator
+}

package/app/validators/multiToolValidator.js

@@ -0,0 +1,49 @@
+// @ts-nocheck
+const { Transform } = require('stream')
+const validator = require('validator')
+
+/**
+ *
+ * @param {Object} options
+ * @param {Array} options.columns { columns: [
+ *   {
+ *       name: "column-name",
+ *       validator: "isAlphanumeric",
+ *       args: {
+ *               locale: "en-GB",
+ *               options: {
+ *                   ignore: "-;%/ "
+ *               }
+ *       }
+ *   }
+ *  ]
+ * }
+ * @returns Writable
+ */
+function MultiToolValidator (options) {
+  function getValidator (validatorType) {
+    return validator[validatorType]
+  }
+  return new Transform({
+    readableObjectMode: true,
+    writableObjectMode: true,
+    decodeStrings: false,
+    construct (callback) {
+      this.columns = options.columns
+      callback()
+    },
+    transform (chunk, _, callback) {
+      const { _columns } = chunk
+      this.columns.forEach(column => {
+        const colIndex = _columns.indexOf(column.name)
+        const vtr = getValidator(column.validator)
+        if (!vtr(chunk[colIndex], ...Object.values(column.args))) { chunk.errors.push(`${column.validator} returned invalid status in column ${colIndex}`) }
+      })
+      callback(null, chunk)
+    }
+  })
+}
+
+module.exports = {
+  MultiToolValidator
+}

package/app/validators/requiredValidator.js

@@ -0,0 +1,35 @@
+// @ts-nocheck
+const { Transform } = require('stream')
+
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.message
+ * @param {Array} options.columns
+ * @returns Transform
+ */
+function RequiredValidator (options) {
+  return new Transform({
+    readableObjectMode: true,
+    writableObjectMode: true,
+    decodeStrings: false,
+    construct (callback) {
+      this.message = options.message ? options.message : 'Required value is null'
+      this.requiredColumns = options.columns
+      callback()
+    },
+    transform (chunk, _, callback) {
+      this.requiredColumns.forEach(column => {
+        const requiredVal = chunk[column]
+        if (requiredVal === undefined || requiredVal === '') {
+          chunk.errors.push(`${this.message} in column ${column}`)
+        }
+      })
+      callback(null, chunk)
+    }
+  })
+}
+
+module.exports = {
+  RequiredValidator
+}

package/app/validators/uniqueValidator.js

@@ -0,0 +1,41 @@
+const { Transform } = require('node:stream')
+
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.message
+ * @param {String} options.column
+ * @returns Transform
+ */
+function UniqueValidator (options) {
+  const self = this
+  self.column = options.column
+  self.message = options.message ? options.message : 'Expected unique value is not unique'
+  self.uniqueColumnRows = []
+  function checkIsUnique (data) {
+    return self.uniqueColumnRows.indexOf(data) === -1
+  }
+  return new Transform({
+    readableObjectMode: true,
+    writableObjectMode: true,
+    decodeStrings: false,
+    construct (callback) {
+      callback()
+    },
+    transform (chunk, _, callback) {
+      const { _columns } = chunk
+      const colIndex = _columns.indexOf(self.column)
+      const uniqueValue = chunk[colIndex]
+      if (checkIsUnique(uniqueValue)) {
+        self.uniqueColumnRows.push(uniqueValue)
+      } else {
+        chunk.errors.push(`${self.message} ${self.column}`)
+      }
+      callback(null, chunk)
+    }
+  })
+}
+
+module.exports = {
+  UniqueValidator
+}