ffc-pay-etl-framework 1.4.2-alpha.6 → 1.4.2

package/app/loaders/csv-loader.js

@@ -0,0 +1,58 @@
+// @ts-nocheck
+const fs = require('node:fs')
+const { Transform } = require('node:stream')
+const { parse } = require('csv-parse')
+
+/**
+ *
+ * @param {Object} options
+ * @param {String} [options.path]
+ * @param {ReadableStream} [options.stream]
+ * @param {Number} [options.startingLine]
+ * @param {Boolean} [options.relax]
+ * @returns StreamReader
+ */
+function csvLoader (options) {
+  let loader
+  if (options.path) {
+    loader = fs.createReadStream(options.path)
+  } else {
+    loader = options.stream
+  }
+  let lineCount = 1
+  const fromLine = options.startingLine ?? 2
+  const relaxQuotes = options.relax ?? false
+  loader._columns = options.columns
+  loader.pump = () => {
+    const parser = parse({ delimiter: ',', from_line: fromLine, relax_quotes: relaxQuotes })
+    const transformer = new Transform({
+      readableObjectMode: true,
+      writableObjectMode: true,
+      emitClose: true,
+      transform (chunk, _, callback) {
+        chunk['_columns'] = options.columns
+        chunk['_linecount'] = lineCount
+        lineCount += 1
+
+        // remove non-printable characters
+        options.columns.forEach((_column, index) => {
+          if (chunk[index]) {
+            // eslint-disable-next-line no-control-regex
+            chunk[index] = chunk[index].replaceAll(/[\x00-\x1F\x7F-\x9F]/g, '')
+          }
+        })
+
+        callback(null, chunk)
+      }
+    })
+    return loader
+      .pipe(parser)
+      .pipe(transformer)
+  }
+
+  return loader
+}
+
+module.exports = {
+  CSVLoader: csvLoader
+}

package/app/loaders/index.js

@@ -0,0 +1,5 @@
+const { CSVLoader } = require('./csv-loader')
+
+module.exports = {
+  CSVLoader
+}

package/app/misc/README.md

@@ -0,0 +1,55 @@
+# Miscellanous pipeline items
+
+## PostgresSQLTask
+
+Executes an arbitrary sql task either as part of another transformation or as an event
+
+### Example
+
+```js
+const {
+  Connections,
+  Destinations,
+  Etl,
+  Loaders,
+  Misc,
+  Validators,
+  Transformers,
+} = require("ffc-pay-etl-framework");
+
+const etl = new Etl.Etl();
+
+etl
+  .connections(
+    new Connections.PostgresSQLConnection({
+      connectionname: "MyConnection",
+      database: "MyDB",
+      username: "MyUserName",
+      password: "MyPassword",
+      host: "localhost",
+      port: 5432,
+    })
+  )
+  .beforeETL(
+    new Misc.PostgresSQLTask({
+      connection: "MyConnection",
+      sql: "INSERT INTO MyTable (column1, column2) VALUES ('apples','oranges');",
+    })
+  )
+  .loader(new Loaders.CSVLoader({ path: csvFile, columns: columns }))
+  .destination(
+    new Destinations.PostgresDestination({
+      connection: "MyConnection",
+      table: "target",
+      mapping: [
+        {
+          column: "Dist Code",
+          targetColumn: "dist_code",
+          targetType: "string",
+        },
+      ],
+      includeErrors: false,
+    })
+  )
+  .pump();
+```

package/app/misc/index.js

@@ -0,0 +1,5 @@
+const { PostgresSQLTask } = require('./postgres-sql-task')
+
+module.exports = {
+  PostgresSQLTask
+}

package/app/misc/postgres-sql-task.js

@@ -0,0 +1,65 @@
+// @ts-nocheck
+const { PassThrough } = require('node:stream')
+const startPosOffset = 3
+const endPosOffset = 1
+
+function getPlaceHolders (sql) {
+  const columnNamePlaceholderRegex = /(?<=\$\{)(.*?)(?=})/g
+  const matches = Array.from(sql.matchAll(columnNamePlaceholderRegex))
+  return matches.map(match => ({
+    match: match[0],
+    value: match[0].split('.')[1],
+    collection: match[0].split('.')[0],
+    startPos: match.index - startPosOffset,
+    endPos: match.index + match[0].length - endPosOffset
+  }))
+}
+
+function doPlaceHolderValueInterpolations (chunk, sql, placeholders) {
+  placeholders.forEach(placeholder => {
+    sql = sql.replace(new RegExp(`\\$\{${placeholder.match}}`),
+      chunk[chunk[`_${placeholder.collection}`].indexOf(placeholder.value)])
+  })
+  return sql
+}
+
+function postgresSQLTask (options) {
+  const passthrough = new PassThrough({
+    readableObjectMode: true,
+    writableObjectMode: true,
+    decodeStrings: false,
+    construct (callback) {
+      this.sql = options.sql
+      this.connectionname = options.connectionname
+      callback()
+    },
+    transform (chunk, _, callback) {
+      const placeholders = getPlaceHolders(this.sql)
+      if (placeholders.length === 0) {
+        this.connection.db.query(this.sql)
+      } else {
+        const interpolatedSql = doPlaceHolderValueInterpolations(chunk, this.sql, placeholders)
+        this.connection.db.query(interpolatedSql)
+      }
+      callback(null, chunk)
+    }
+  })
+  Object.assign(passthrough, {
+    setConnection: function (connection) {
+      this.connection = connection
+    }.bind(passthrough),
+    getConnectionName: function () {
+      return this.connection.name
+    }.bind(passthrough),
+    setETL: function (etl) {
+      this.etl = etl
+    }.bind(passthrough)
+  })
+  return passthrough
+}
+
+module.exports = {
+  PostgresSQLTask: postgresSQLTask,
+  getPlaceHolders,
+  doPlaceHolderValueInterpolations
+}

package/app/transformers/README.md

@@ -0,0 +1,102 @@
+# Transformers
+
+Transformers manipulate row data and either replace the existing values or add new columns to each row depending on the type of transformer and the configuration.
+
+Currently supported transformers are :
+
+- ToUpperCaseTransformer
+- FakerTransformer
+
+## FakerTransformer
+
+### Options
+
+| option  | description                                                  |
+| ------- | ------------------------------------------------------------ |
+| columns | Array of column transformations                              |
+| locale  | See https://fakerjs.dev/guide/localization for valid locales |
+
+### Usage
+
+```js
+FakerTransformer({
+  columns: [
+    {
+      name: "column2",
+      faker: "company.name",
+    },
+  ],
+});
+```
+
+faker is a string value that can be any of the currently supported [apis](https://fakerjs.dev/api/)
+
+e.g.
+
+- finance.accountName
+- person.firstName
+- randomizer.next
+
+To create UK post code
+
+```js
+FakerTransformer({
+  columns: [
+    {
+      name: "column2",
+      faker: "location.zipCode",
+    },
+  ],
+  locale: "en_GB",
+});
+```
+
+### Example
+
+examples/csv-faker.js
+
+## ToUpperCaseTransformer
+
+### Options
+
+| option | description                     |
+| ------ | ------------------------------- |
+| column | source column to be transformed |
+
+### Usage
+
+```js
+ToUpperCaseTransformer({
+  column: "column2",
+});
+```
+
+### Example
+
+examples/csv-to-upper-case.js
+
+## StringReplaceTransformer
+
+### Options
+
+| option  | description                             |
+| ------- | --------------------------------------- |
+| column  | source column to be transformed         |
+| find    | search string to be replaced            |
+| replace | value to replace the search string with |
+
+### Usage
+
+```js
+StringReplaceTransformer([
+  {
+    column: "column2",
+    find: "'",
+    replace: "''",
+  },
+]);
+```
+
+### Example
+
+examples/csv-find-replace.js

package/app/transformers/faker-transformer.js

@@ -0,0 +1,48 @@
+const { Transform } = require('node:stream')
+
+/**
+ *
+ * @param {Object} options
+ * @param {Array} options.columns {
+ *           columns: [
+ *               {
+ *                   name: "column-name",
+ *                   faker: "company.name"
+ *               }
+ *           ]
+ *       }
+ * @param {String} options.locale
+ * @returns StreamReader
+ */
+function fakerTransformer (options) {
+  const columns = options.columns
+  let faker
+  if (options.locale) {
+    faker = require(`@faker-js/faker/locale/${options.locale}`).faker
+  } else {
+    faker = require('@faker-js/faker').faker
+  }
+
+  function getFaker (fakerType) {
+    return fakerType.split('.').reduce((a, b) => {
+      return Object.keys(a).length === 0 ? faker[b] : a[b]
+    }, {})
+  }
+
+  return new Transform({
+    readableObjectMode: true,
+    writableObjectMode: true,
+    transform (chunk, _, callback) {
+      const { _columns } = chunk
+      columns.forEach(column => {
+        const colIndex = _columns.indexOf(column.name)
+        chunk[colIndex] = getFaker(column.faker)()
+      })
+      callback(null, chunk)
+    }
+  })
+}
+
+module.exports = {
+  FakerTransformer: fakerTransformer
+}

package/app/transformers/index.js

@@ -0,0 +1,9 @@
+const { ToUpperCaseTransformer } = require('./to-upper-case-transformer')
+const { FakerTransformer } = require('./faker-transformer')
+const { StringReplaceTransformer } = require('./string-replace-transformer')
+
+module.exports = {
+  StringReplaceTransformer,
+  ToUpperCaseTransformer,
+  FakerTransformer
+}

package/app/transformers/string-replace-transformer.js

@@ -0,0 +1,38 @@
+const { Transform } = require('node:stream')
+
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.column
+ * @returns Writable
+ */
+function stringReplaceTransformer (options) {
+  const replacements = options
+
+  return new Transform({
+    readableObjectMode: true,
+    writableObjectMode: true,
+    transform (chunk, _, callback) {
+      const { _columns } = chunk
+      // @ts-ignore
+      replacements.forEach(r => {
+        const colIndex = _columns.indexOf(r.column)
+        if (colIndex === -1) {
+          // column not found, skip
+          return
+        }
+        if (r.all) {
+          chunk[colIndex] = chunk[colIndex].replaceAll(r.find, r.replace)
+        } else {
+          chunk[colIndex] = chunk[colIndex].replace(r.find, r.replace)
+        }
+      })
+
+      callback(null, chunk)
+    }
+  })
+}
+
+module.exports = {
+  StringReplaceTransformer: stringReplaceTransformer
+}

package/app/transformers/to-upper-case-transformer.js

@@ -0,0 +1,29 @@
+const { Transform } = require('node:stream')
+
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.column
+ * @returns Writable
+ */
+function toUpperCaseTransformer (options) {
+  const column = options.column
+
+  return new Transform({
+    readableObjectMode: true,
+    writableObjectMode: true,
+    transform (chunk, _, callback) {
+      const { _columns } = chunk
+      const colIndex = _columns.indexOf(column)
+
+      if (colIndex !== -1 && chunk[colIndex] != null) {
+        chunk[colIndex] = chunk[colIndex].toUpperCase()
+      }
+      callback(null, chunk)
+    }
+  })
+}
+
+module.exports = {
+  ToUpperCaseTransformer: toUpperCaseTransformer
+}

package/app/validators/README.md

@@ -0,0 +1,81 @@
+# Validators
+
+Validators are expected to check specific columns within each row against specific validation rules
+
+Currently supported destinations are :
+
+- RequiredValidator
+- UniqueValidator
+- MultiToolValidator
+
+## RequiredValidator
+
+### Options
+
+| option         | description                                                   |
+| -------------- | ------------------------------------------------------------- |
+| columns        | Array of column indexes                                       |
+
+### Usage
+
+```js
+Validators.RequiredValidator({
+    columns: [0] 
+})
+```
+
+### Example
+
+examples/csv-required-validator.js
+
+## UniqueValidator
+
+### Options
+
+| option         | description                                                   |
+| -------------- | ------------------------------------------------------------- |
+| column         | Column to be tested for uniqueness                            |
+
+### Usage
+
+```js
+Validators.UniqueValidator(
+    { column: "field_one" }
+)
+```
+
+### Example
+
+examples/csv-unique-validator.js
+
+## MultiToolValidator
+
+### Options
+
+| option         | description                                                   |
+| -------------- | ------------------------------------------------------------- |
+| column         | Column to be tested for validity and validator specification  |
+
+NB Any validators specified [here](https://github.com/validatorjs/validator.js) can be used
+
+### Usage
+
+```js
+Validators.MultiToolValidator( { columns: [
+    {
+        name: "SOIL TYPE PERCENT (Percent)",
+        validator: "isAlphanumeric",
+        args: {
+                locale: "en-GB",
+                options: {
+                    ignore: "-;%/ "
+                }
+        }
+    }
+]
+})
+```
+
+### Example
+
+examples/csv-multi-tool-validator.js

package/app/validators/index.js

@@ -0,0 +1,9 @@
+const { RequiredValidator } = require('./required-validator')
+const { UniqueValidator } = require('./unique-validator')
+const { MultiToolValidator } = require('./multi-tool-validator')
+
+module.exports = {
+  RequiredValidator,
+  UniqueValidator,
+  MultiToolValidator
+}

package/app/validators/multi-tool-validator.js

@@ -0,0 +1,50 @@
+// @ts-nocheck
+const { Transform } = require('node:stream')
+const validator = require('validator')
+
+function getValidator (validatorType) {
+  return validator[validatorType]
+}
+
+/**
+ *
+ * @param {Object} options
+ * @param {Array} options.columns { columns: [
+ *   {
+ *       name: "column-name",
+ *       validator: "isAlphanumeric",
+ *       args: {
+ *               locale: "en-GB",
+ *               options: {
+ *                   ignore: "-;%/ "
+ *               }
+ *       }
+ *   }
+ *  ]
+ * }
+ * @returns Writable
+ */
+function multiToolValidator (options) {
+  return new Transform({
+    readableObjectMode: true,
+    writableObjectMode: true,
+    decodeStrings: false,
+    construct (callback) {
+      this.columns = options.columns
+      callback()
+    },
+    transform (chunk, _, callback) {
+      const { _columns } = chunk
+      this.columns.forEach(column => {
+        const colIndex = _columns.indexOf(column.name)
+        const vtr = getValidator(column.validator)
+        if (!vtr(chunk[colIndex], ...Object.values(column.args))) { chunk.errors.push(`${column.validator} returned invalid status in column ${colIndex}`) }
+      })
+      callback(null, chunk)
+    }
+  })
+}
+
+module.exports = {
+  MultiToolValidator: multiToolValidator
+}

package/app/validators/required-validator.js

@@ -0,0 +1,35 @@
+// @ts-nocheck
+const { Transform } = require('node:stream')
+
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.message
+ * @param {Array} options.columns
+ * @returns Transform
+ */
+function requiredValidator (options) {
+  return new Transform({
+    readableObjectMode: true,
+    writableObjectMode: true,
+    decodeStrings: false,
+    construct (callback) {
+      this.message = options.message ? options.message : 'Required value is null'
+      this.requiredColumns = options.columns
+      callback()
+    },
+    transform (chunk, _, callback) {
+      this.requiredColumns.forEach(column => {
+        const requiredVal = chunk[column]
+        if (requiredVal === undefined || requiredVal === '') {
+          chunk.errors.push(`${this.message} in column ${column}`)
+        }
+      })
+      callback(null, chunk)
+    }
+  })
+}
+
+module.exports = {
+  RequiredValidator: requiredValidator
+}

package/app/validators/unique-validator.js

@@ -0,0 +1,46 @@
+const { Transform } = require('node:stream')
+
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.message
+ * @param {String} options.column
+ * @returns Transform
+ */
+function uniqueValidator (options) {
+  const column = options.column
+  const message = options.message || 'Expected unique value is not unique'
+  const uniqueColumnRows = new Set()
+
+  function checkIsUnique (data) {
+    return !uniqueColumnRows.has(data)
+  }
+
+  return new Transform({
+    readableObjectMode: true,
+    writableObjectMode: true,
+    decodeStrings: false,
+    construct (callback) {
+      callback()
+    },
+    transform (chunk, _, callback) {
+      const { _columns } = chunk
+      const colIndex = _columns.indexOf(column)
+      const uniqueValue = chunk[colIndex]
+
+      if (checkIsUnique(uniqueValue)) {
+        uniqueColumnRows.add(uniqueValue)
+      } else {
+        if (!chunk.errors) {
+          chunk.errors = []
+        }
+        chunk.errors.push(`${message} ${column}`)
+      }
+      callback(null, chunk)
+    }
+  })
+}
+
+module.exports = {
+  UniqueValidator: uniqueValidator
+}

package/azure-pipelines.yml

@@ -0,0 +1,45 @@
+# Pipeline name convention
+name: '$(Build.SourceBranchName)$(System.PullRequest.SourceBranch)$(Rev:.r)'
+
+# Trigger configuration
+trigger:
+  - main
+  - feature/*
+  - release/*
+
+pr:
+  - main
+  - releases/*
+
+# Parameters
+parameters:
+  - name: prDeployment
+    type: boolean
+    default: true
+  - name: branchDeployment
+    type: boolean
+    default: true
+  - name: sndDeployment
+    type: boolean
+    default: true
+
+# Resources
+resources:
+  repositories:
+    - repository: templates
+      endpoint: DEFRA
+      type: github
+      name: defra/fcp-pipeline-common
+      # ref: pl-709   # Optional: build specific pipeline branch
+
+# Pipeline template
+extends:
+  template: fcp-common-pipeline.yaml@templates
+  parameters:
+    appFrameworkType: nodejs   # Optional: nodejs (default), dotnet, java
+    Type: npm           # Optional: kubernetes (default), npm, functions, nuget
+    npmLegacyPeerDeps: false   # Optional: default is false
+    failOnThreshold: moderate  # Optional: moderate (default), low, high, critical
+    prDeployment: ${{ parameters.prDeployment }}
+    branchDeployment: ${{ parameters.branchDeployment }}
+    sndDeployment: ${{ parameters.sndDeployment }}