ffc-pay-etl-framework 1.4.2-alpha.6 → 1.4.3-alpha.7

package/.all-contributorsrc

@@ -0,0 +1,43 @@
+{
+  "projectName": "ffc-pay-etl-framework",
+  "projectOwner": "DEFRA",
+  "repoType": "github",
+  "repoHost": "https://github.com",
+  "files": [
+    "README.md"
+  ],
+  "imageSize": 100,
+  "commit": true,
+  "commitConvention": "angular",
+  "contributors": [
+    {
+      "login": "suityou01",
+      "name": "Charlie Benger-Stevenson",
+      "avatar_url": "https://avatars.githubusercontent.com/u/7422644?v=4",
+      "profile": "https://github.com/suityou01",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "GodsonLeigh",
+      "name": "Leigh Godson",
+      "avatar_url": "https://avatars.githubusercontent.com/u/139965284?v=4",
+      "profile": "https://github.com/GodsonLeigh",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "samplackett",
+      "name": "Sam Plackett",
+      "avatar_url": "https://avatars.githubusercontent.com/u/60177449?v=4",
+      "profile": "https://github.com/samplackett",
+      "contributions": [
+        "code"
+      ]
+    }
+  ],
+  "contributorsPerLine": 7,
+  "linkToUsage": true
+}

package/.devcontainer/devcontainer.json

@@ -0,0 +1,43 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/universal
+{
+	"name": "Default Linux Universal",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	// "image": "mcr.microsoft.com/devcontainers/universal:2-linux",
+	"dockerComposeFile": "./docker-compose.yaml",
+	"service": "etl",
+	"workspaceFolder": "/workspace",
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "uname -a",
+
+	// Configure tool-specific properties.
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"ms-azuretools.vscode-docker",
+				"orta.vscode-jest",
+				"andys8.jest-snippets",
+				"chris-noring.node-snippets",
+				"cweijan.vscode-postgresql-client2",
+				"mechatroner.rainbow-csv",
+				"esbenp.prettier-vscode",
+				"sonarsource.sonarlint-vscode"
+			],
+			"settings": {
+				"jest.runMode": {
+					"type": "on-demand",
+					"coverage": false
+				}
+			}
+		}
+	},
+	// "overrideCommand": true
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

package/.devcontainer/docker-compose.yaml

@@ -0,0 +1,15 @@
+services:
+  etl:
+    image: mcr.microsoft.com/devcontainers/universal:2-linux
+    volumes:
+      - ../:/workspace:cached
+  postgres:
+    image: postgres:latest
+    ports:
+      - 5432:5432
+    volumes:
+      - ./databases/postgres:/var/lib/postgresql
+      - ./postgres_init.sql:/docker-entrypoint-initdb.d/init.sql
+    environment:
+      - POSTGRES_PASSWORD=ppp
+      - POSTGRES_DB=etl_db

package/.devcontainer/postgres_init.sql

@@ -0,0 +1,5 @@
+CREATE TABLE target (
+    target_column1 VARCHAR(100),
+    target_column2 VARCHAR(100),
+    target_column3 VARCHAR(100)
+);

package/.dockerignore

@@ -0,0 +1,10 @@
+.gitignore
+.pre-commit-config.yaml
+CODE_OF_CONDUCT.md
+CONTRIBUTING.md
+ISSUE_TEMPLATE.md
+LICENSE.md
+PULL_REQUEST_TEMPLATE.md
+README.md
+Jenkinsfile
+

package/.github/workflows/main.yaml

@@ -0,0 +1,77 @@
+name: Version Bump
+
+on:
+  pull_request:
+    types: [opened, reopened]
+
+jobs:
+  version-bump:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          fetch-depth: 0
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '20'
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Set up GPG keys for signing
+        run: |
+          echo "$GPG_PRIVATE_KEY" | gpg --batch --import
+          git config --global user.signingkey ${{ secrets.GPG_KEY_ID }}
+          git config --global commit.gpgsign true
+          git config --global user.email ${{ secrets.GPG_EMAIL }}
+          git config --global user.name ${{ secrets.GPG_NAME }}
+        env:
+          GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }}
+          GPG_KEY_ID: ${{ secrets.GPG_KEY_ID }}
+          GPG_EMAIL: ${{ secrets.GPG_EMAIL }}
+          GPG_NAME: ${{ secrets.GPG_NAME }}
+
+      - name: Commit dependency updates with GPG sign
+        run: |
+          git add package-lock.json
+          git commit -S -m "update package-lock.json" || echo "No changes to commit"
+
+      - name: Fetch main branch
+        run: git fetch origin main
+
+      - name: Merge main into feature branch
+        run: |
+          git merge origin/main --no-commit
+        continue-on-error: true
+
+      - name: Check for merge conflicts
+        run: |
+          if [[ $(git ls-files -u | wc -l) -gt 0 ]]; then
+            echo "Merge conflicts detected. Exiting without version bump."
+            exit 0
+          else
+            echo "Merge successful. Proceeding with version bump."
+          fi
+        shell: bash
+
+      - name: Commit merged changes
+        run: |
+          git commit -S -m "merge main into feature branch" || echo "No changes to commit"
+
+      - name: Bump version number
+        run: |
+          npm version patch
+
+      - name: Ensure branch is checked out
+        run: |
+          git checkout ${{ github.head_ref }}
+
+      - name: Push version bump and commit with GPG sign
+        run: |
+          git push --set-upstream origin ${{ github.head_ref }} --force
+          git push --tags

package/.snyk

@@ -0,0 +1,5 @@
+# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
+
+version: v1.25.0
+ignore: {}
+patch: {}

package/CODE_OF_CONDUCT.md

@@ -0,0 +1,51 @@
+# Contributor Code of Conduct
+
+Our [values](https://www.gov.uk/government/organisations/department-for-environment-food-rural-affairs/about#:~:text=We%20are%20responsible%20for%20improving,food%2C%20farming%20and%20fishing%20industries.) guide us in our day-to-day interactions and decision-making. Our open source projects are no exception. Trust, respect, collaboration and transparency are core values we believe should live and breathe within our projects. Our community welcomes participants from around the world with different experiences, unique perspectives, and great ideas to share.
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment include:
+
+- Using welcoming and inclusive language
+- Being respectful of differing viewpoints and experiences
+- Gracefully accepting constructive criticism
+- Attempting collaboration before conflict
+- Focusing on what is best for the community
+- Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+- Violence, threats of violence, or inciting others to commit self-harm
+- The use of sexualized language or imagery and unwelcome sexual attention or advances
+- Trolling, intentionally spreading misinformation, insulting/derogatory comments, and personal or political attacks
+- Public or private harassment
+- Publishing others' private information, such as a physical or electronic address, without explicit permission
+- Abuse of the reporting process to intentionally harass or exclude others
+- Advocating for, or encouraging, any of the above behavior
+- Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting us anonymously through [this form](https://www.gov.uk/government/organisations/department-for-environment-food-rural-affairs/about/complaints-procedure). All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
+
+If you are unsure whether an incident is a violation, or whether the space where the incident took place is covered by our Code of Conduct, **we encourage you to still report it**. We would prefer to have a few extra reports where we decide to take no action, than to leave an incident go unnoticed and unresolved that may result in an individual or group to feel like they can no longer participate in the community. Reports deemed as not a violation will also allow us to improve our Code of Conduct and processes surrounding it. If you witness a dangerous situation or someone in distress, we encourage you to report even if you are only an observer.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), [version 1.4](https://www.contributor-covenant.org/version/1/4/code-of-conduct.html).

package/CONTRIBUTING.md

@@ -0,0 +1,45 @@
+This is still a fairly new package. While it delivers some basic ETL functionality there is still a lot of work to be done.
+
+There are still a lot of things to be done, so all contributions are welcome!
+
+## Code of Conduct
+
+We take our open source community seriously and hold ourselves and other contributors to high standards of communication. By participating and contributing to this project, you agree to uphold our [Code of Conduct](https://github.com/ffc-pay-etl-framework/CODE-OF-CONDUCT.md).
+
+## Getting Started
+
+Contributions are made to this repo via Issues and Pull Requests (PRs). A few general guidelines that cover both:
+
+- Search for existing Issues and PRs before creating your own.
+- We work hard to makes sure issues are handled in a timely manner but, depending on the impact, it could take a while to investigate the root cause. A friendly ping in the comment thread to the submitter or a contributor can help draw attention if your issue is blocking.
+
+### Issues
+
+Issues should be used to report problems with the library, request a new feature, or to discuss potential changes before a PR is created. When you create a new Issue, a template will be loaded that will guide you through collecting and providing the information we need to investigate.
+
+If you find an Issue that addresses the problem you're having, please add your own reproduction information to the existing issue rather than creating a new one. Adding a [reaction](https://github.blog/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) can also help be indicating to our maintainers that a particular problem is affecting more than just the reporter.
+
+### Pull Requests
+
+PRs to our libraries are always welcome and can be a quick way to get your fix or improvement slated for the next release. In general, PRs should:
+
+- Only fix/add the functionality in question **OR** address wide-spread whitespace/style issues, not both.
+- Add unit or integration tests for fixed or changed functionality (if a test suite already exists).
+- Address a single concern in the least number of changed lines as possible.
+- Include documentation in the repo or on our [docs site](https://auth0.com/docs).
+- Be accompanied by a complete Pull Request template (loaded automatically when a PR is created).
+
+For changes that address core functionality or would require breaking changes (e.g. a major release), it's best to open an Issue to discuss your proposal first. This is not required but can save time creating and reviewing changes.
+
+In general, we follow the ["fork-and-pull" Git workflow](https://github.com/susam/gitpr)
+
+1. Fork the repository to your own Github account
+2. Clone the project to your machine
+3. Create a branch locally with a succinct but descriptive name
+4. Commit changes to the branch
+5. Following any formatting and testing guidelines specific to this repo
+6. Push changes to your fork
+7. Open a PR in our repository and follow the PR template so that we can efficiently review the changes.
+
+
+

package/ISSUE_TEMPLATE.md

@@ -0,0 +1,39 @@
+# Prerequisites
+
+Please answer the following questions for yourself before submitting an issue. 
+
+- [ ] I am running the latest version
+- [ ] I checked the documentation and found no answer
+- [ ] I checked to make sure that this issue has not already been filed
+
+# Expected Behavior
+
+Please describe the behavior you are expecting
+
+# Current Behavior
+
+What is the current behavior?
+
+# Failure Information (for bugs)
+
+Please help provide information about the failure if this is a bug. If it is not a bug, please remove the rest of this template.
+
+## Steps to Reproduce
+
+Please provide detailed steps for reproducing the issue.
+
+1. step 1
+2. step 2
+3. you get it...
+
+## Context
+
+Please provide any relevant information about your setup. This is important in case the issue is not reproducible except for under certain conditions.
+
+* Operating System:
+* NPM version:
+* Node version:
+
+## Failure Logs
+
+Please include any relevant log snippets or files here.

package/Jenkinsfile

@@ -0,0 +1,3 @@
+@Library('defra-library@v-9') _
+
+buildNpm()

package/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,32 @@
+# Description
+
+Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change.
+
+Fixes # (issue)
+
+## Type of change
+
+Please delete options that are not relevant.
+
+- [ ] Bug fix (non-breaking change which fixes an issue)
+- [ ] New feature (non-breaking change which adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ] This change requires a documentation update
+
+# How Has This Been Tested?
+
+Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration
+
+- [ ] Test A
+- [ ] Test B
+
+# Checklist:
+
+- [ ] My code follows the style guidelines of this project
+- [ ] I have performed a self-review of my own code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I have made corresponding changes to the documentation
+- [ ] My changes generate no new warnings
+- [ ] I have added tests that prove my fix is effective or that my feature works
+- [ ] New and existing unit tests pass locally with my changes
+- [ ] Any dependent changes have been merged and published in downstream modules

package/__mocks__/fs.js

@@ -0,0 +1,21 @@
+'use strict'
+const { Readable } = require('node:stream')
+
+const fs = jest.createMockFromModule('fs')
+
+const mockContent = []
+
+function __setMockFileContent (mockPath, mockFileContent) {
+  mockContent[mockPath] = mockFileContent
+}
+
+function createReadStream (path) {
+  return Readable.from(mockContent[path])
+}
+
+const writeFileSync = jest.fn()
+
+fs.createReadStream = createReadStream
+fs.writeFileSync = writeFileSync
+fs.__setMockFileContent = __setMockFileContent
+module.exports = fs

package/app/database_connections/index.js

@@ -0,0 +1,7 @@
+const { PostgresDatabaseConnection } = require('./postgresDatabaseConnection')
+const { ProvidedConnection } = require('./providedConnection')
+
+module.exports = {
+  PostgresDatabaseConnection,
+  ProvidedConnection
+}

package/app/database_connections/postgresDatabaseConnection.js

@@ -0,0 +1,47 @@
+const DEFAULT_PORT = 5432
+const { Sequelize } = require('sequelize')
+const debug = require('debug')('connection')
+
+/**
+ *
+ * @param {Object} options
+ * @param {Object} options.connectionname
+ * @param {Object} options.username
+ * @param {Object} options.password
+ * @param {Object} options.database
+ * @param {Object} options.host
+ * @param {Object} options.port
+ * @returns Connection
+ */
+async function PostgresDatabaseConnection (options) {
+  const connectionname = options.connectionname
+  const username = options.username
+  const password = options.password
+  const database = options.database
+  const host = options.host
+  const port = options.port || DEFAULT_PORT
+
+  const sequelize = new Sequelize(database, username, password, {
+    host,
+    port,
+    dialect: 'postgres',
+    logging: false
+  })
+
+  try {
+    await sequelize.authenticate()
+    debug('sequelize.authenticate succeeded')
+    return {
+      name: connectionname,
+      db: sequelize
+    }
+  } catch (e) {
+    debug('sequelize.authenticate failed')
+    debug(e)
+    throw e
+  }
+}
+
+module.exports = {
+  PostgresDatabaseConnection
+}

package/app/database_connections/providedConnection.js

@@ -0,0 +1,19 @@
+/**
+ *
+ * @param {Object} options
+ * @param {Object} options.connectionname
+ * @param {Object} options.sequelize
+ * @returns Connection
+ */
+async function ProvidedConnection (options) {
+  const connectionname = options.connectionname
+  const sequelize = options.sequelize
+  return {
+    name: connectionname,
+    db: sequelize
+  }
+}
+
+module.exports = {
+  ProvidedConnection
+}

package/app/destinations/README.md

@@ -0,0 +1,107 @@
+# Destinations
+
+Destinations are expected to write out to an output. This could be a file, a database, an endpoint, whatever.
+
+Currently supported destinations are :
+
+- CSVFileDestination
+- PostgresDestination
+- SQLFileDestination
+
+## CSVFileDestination
+
+### Options
+
+| option         | description                                                   |
+| -------------- | ------------------------------------------------------------- |
+| fileName       | absolute path to the csv file to be written                   |
+| headers        | treat the first line of the file as column header information |
+| includeErrors  | write out any rows that contain errors                        |
+| quotationMarks | wrap values in quotation marks                                |
+
+### Usage
+
+```js
+CSVFileDestination({
+  fileName: "...",
+  headers: true,
+  includeErrors: false,
+  quotationMarks: true,
+})
+```
+
+### Example
+
+examples/csv-to-csv.js
+
+## PostgresDestination
+
+### Options
+
+| option        | description                                            |
+| ------------- | ------------------------------------------------------ |
+| table         | the name of the table to be inserted into              |
+| username      | the username for the database connection               |
+| password      | the password for the database connection               |
+| database      | the name of the database for the connection            |
+| host          | the host name for the database connection              |
+| port          | the port for the database connection. Defaults to 5432 |
+| includeErrors | write out any rows that contain errors                 |
+| mapping       | field mappings from source to target                   |
+
+### Usage
+
+```js
+PostgresDestination({
+  username: "postgres",
+  password: "ppp",
+  database: "etl_db",
+  host: "postgres",
+  table: "target",
+  includeErrors: false,
+  mapping: [
+    {
+      column: "column1",
+      targetColumn: "target_column1",
+      targetType: "varchar",
+    },
+  ],
+})
+```
+
+### Example
+
+examples/csv-to-postgres.js
+
+## SQLFileDestination
+
+### options
+
+| option   | description                                 |
+| -------- | ------------------------------------------- |
+| fileName | absolute path to the sql file to be written |
+| mode     | insert or update mode                       |
+| table    | the table for the statements                |
+| mapping  | field mappings from source to target        |
+
+### Usage
+
+```js
+SQLFileDestination({
+  filename: "...",
+  mode: SQL_MODE.INSERT_MODE,
+  table: "target",
+  includeErrors: false,
+  mapping: [
+    {
+      column: "column1",
+      targetColumn: "target_column1",
+      targetType: "varchar",
+    },
+  ],
+})
+```
+
+### Example
+
+examples/csv-to-sqlfile.js

package/app/destinations/consoleDestination.js

@@ -0,0 +1,38 @@
+const { Writable } = require('node:stream')
+
+/**
+ *
+ * @param {Object} options
+ * @param {String} options.includeErrors
+ * @returns Writable
+ */
+function ConsoleDestination (options) {
+  const includeErrors = options.includeErrors
+  const writable = new Writable({
+    objectMode: true,
+    write (chunk, _, callback) {
+      if (chunk.errors.length === 0 || includeErrors) { console.log(chunk) }
+      // @ts-ignore
+      this.tasks?.forEach(task => task.write(chunk))
+      callback()
+    }
+  })
+
+  Object.assign(writable, {
+    setConnection: function (connection) {
+      this.connection = connection
+    }.bind(writable),
+    getConnectionName: function () {
+      return this.connection?.name
+    }.bind(writable),
+    setTasks: function (tasks) {
+      this.tasks = tasks
+    }.bind(writable)
+  })
+
+  return writable
+}
+
+module.exports = {
+  ConsoleDestination
+}