ffc-pay-etl-framework 1.4.5 → 2.0.0

package/app/database-connections/postgres-database-connection.js

@@ -1,7 +1,8 @@
-const DEFAULT_PORT = 5432
-const { Sequelize } = require('sequelize')
+const { Database } = require('ffc-database')
 const debug = require('debug')('connection')
 
+const DEFAULT_PORT = 5432
+
 /**
  *
  * @param {Object} options
@@ -11,32 +12,44 @@ const debug = require('debug')('connection')
  * @param {Object} options.database
  * @param {Object} options.host
  * @param {Object} options.port
+ * @param {Object} [options.schema]
+ * @param {Object} [options.ssl]
+ * @param {Object} [options.tables]
+ * @param {Object} [options.pool]
+ * @param {Boolean} [options.useAzureManagedIdentity]
+ * @param {String} [options.azureClientId]
  * @returns Connection
  */
 async function postgresDatabaseConnection (options) {
   const connectionname = options.connectionname
-  const username = options.username
-  const password = options.password
-  const database = options.database
-  const host = options.host
-  const port = options.port || DEFAULT_PORT
 
-  const sequelize = new Sequelize(database, username, password, {
-    host,
-    port,
-    dialect: 'postgres',
-    logging: false
+  const database = new Database({
+    database: options.database,
+    username: options.username,
+    password: options.password,
+    host: options.host,
+    port: options.port || DEFAULT_PORT,
+    schema: options.schema,
+    ssl: options.ssl,
+    tables: options.tables,
+    pool: options.pool,
+    useAzureManagedIdentity: options.useAzureManagedIdentity,
+    azureClientId: options.azureClientId
   })
 
   try {
-    await sequelize.authenticate()
-    debug('sequelize.authenticate succeeded')
+    const connection = database.connect()
+    await connection.client.raw('select 1')
+    debug('database connection succeeded')
     return {
       name: connectionname,
-      db: sequelize
+      db: {
+        ...connection,
+        query: (sql) => connection.client.raw(sql)
+      }
     }
   } catch (e) {
-    debug('sequelize.authenticate failed')
+    debug('database connection failed')
     debug(e)
     throw e
   }

package/app/database-connections/provided-connection.js

@@ -2,15 +2,15 @@
  *
  * @param {Object} options
  * @param {Object} options.connectionname
- * @param {Object} options.sequelize
+ * @param {Object} options.connection - the object returned by ffc-database's Database.connect()
  * @returns Connection
  */
 async function providedConnection (options) {
   const connectionname = options.connectionname
-  const sequelize = options.sequelize
+  const connection = options.connection
   return {
     name: connectionname,
-    db: sequelize
+    db: connection ? { ...connection, query: (sql) => connection.client.raw(sql) } : connection
   }
 }
 

package/app/transformers/faker-transformer.js

@@ -1,44 +1,88 @@
 const { Transform } = require('node:stream')
 
 /**
+ * Creates a Node.js Transform stream that replaces placeholder data with fake data.
  *
  * @param {Object} options
- * @param {Array} options.columns {
- *           columns: [
- *               {
- *                   name: "column-name",
- *                   faker: "company.name"
- *               }
- *           ]
- *       }
- * @param {String} options.locale
- * @returns StreamReader
+ * @param {Array} options.columns List of column names and their mapped Faker methods.
+ * @param {String} options.locale Optional locale string identifier (e.g., 'en', 'de', 'en_IN').
+ * @returns {Transform}
  */
 function fakerTransformer (options) {
   const columns = options.columns
   let faker
-  if (options.locale) {
-    faker = require(`@faker-js/faker/locale/${options.locale}`).faker
-  } else {
-    faker = require('@faker-js/faker').faker
-  }
+  let getFaker
+  let fakerLoadError
+  let fakerLoaded = false
+
+  async function loadFaker () {
+    const { Faker, allLocales, faker: defaultFaker } = await import('@faker-js/faker')
+
+    // Resolve the locale instance safely using the Faker v10 allLocales registry
+    if (options.locale) {
+      const localeKey = options.locale === 'en_IND' ? 'en_IN' : options.locale
+      const selectedLocale = allLocales[localeKey]
 
-  function getFaker (fakerType) {
-    return fakerType.split('.').reduce((a, b) => {
-      return Object.keys(a).length === 0 ? faker[b] : a[b]
-    }, {})
+      if (!selectedLocale) {
+        throw new Error(`Locale "${options.locale}" is not supported by Faker v10.`)
+      }
+
+      faker = new Faker({ locale: selectedLocale })
+    } else {
+      faker = defaultFaker
+    }
+
+    // Modern path resolver using optional chaining to dig down into Faker's Proxy structure
+    getFaker = function (fakerType) {
+      return fakerType.split('.').reduce((current, key) => current?.[key], faker)
+    }
+
+    fakerLoaded = true
   }
 
   return new Transform({
     readableObjectMode: true,
     writableObjectMode: true,
     transform (chunk, _, callback) {
-      const { _columns } = chunk
-      columns.forEach(column => {
-        const colIndex = _columns.indexOf(column.name)
-        chunk[colIndex] = getFaker(column.faker)()
-      })
-      callback(null, chunk)
+      if (fakerLoadError) {
+        callback(fakerLoadError)
+        return
+      }
+
+      const runTransform = async () => {
+        if (!fakerLoaded) {
+          await loadFaker()
+        }
+
+        const { _columns } = chunk
+
+        for (const column of columns) {
+          const colIndex = _columns.indexOf(column.name)
+
+          // Guard: Skip processing if the configured column name doesn't exist in the chunk headers
+          if (colIndex === -1) {
+            continue
+          }
+
+          const fakerMethod = getFaker(column.faker)
+
+          // Guard: Verify that the method exists and can be executed safely
+          if (typeof fakerMethod !== 'function') {
+            throw new TypeError(`Faker method "${column.faker}" is invalid or does not exist.`)
+          }
+
+          chunk[colIndex] = fakerMethod()
+        }
+
+        return chunk
+      }
+
+      runTransform()
+        .then((transformed) => callback(null, transformed))
+        .catch((err) => {
+          fakerLoadError = err
+          callback(err)
+        })
     }
   })
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ffc-pay-etl-framework",
-  "version": "1.4.5",
+  "version": "2.0.0",
   "description": "A framework for creating ETL pipelines in node",
   "main": "index.js",
   "repository": "https://github.com/DEFRA/ffc-pay-etl-framework",
@@ -12,9 +12,9 @@
   ],
   "license": "OGL-UK-3.0",
   "dependencies": {
-    "@faker-js/faker": "8.4.1",
-    "csv-parse": "6.1.0",
-    "sequelize": "6.37.8",
+    "@faker-js/faker": "10.5.0",
+    "csv-parse": "7.0.2",
+    "ffc-database": "2.0.0",
     "validator": "13.15.23"
   },
   "devDependencies": {
@@ -26,13 +26,8 @@
     "nodemon": "3.1.4",
     "prettier": "3.3.3"
   },
-  "overrides": {
-    "sequelize": {
-      "uuid": "11.1.1"
-    }
-  },
   "scripts": {
-    "test": "jest",
+    "test": "PORT=0 node --experimental-vm-modules node_modules/jest/bin/jest.js --testPathPatterns=test/unit",
     "test:unit": "PORT=0 node --experimental-vm-modules node_modules/jest/bin/jest.js --testPathPatterns=test/unit",
     "test:watch": "jest --watchAll",
     "test:lint": "eslint",
@@ -48,4 +43,4 @@
       "**/test-output/**"
     ]
   }
-}
+}