ffc-pay-etl-framework 0.1.14 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dist/cjs/database_connections/index.js +4 -0
  2. package/dist/cjs/database_connections/postgresDatabaseConnection.js +55 -0
  3. package/dist/cjs/destinations/consoleDestination.js +17 -1
  4. package/dist/cjs/destinations/csvFileDestination.js +17 -1
  5. package/dist/cjs/destinations/postgresDestination.js +21 -28
  6. package/dist/cjs/destinations/sqlFileDestination.js +22 -7
  7. package/dist/cjs/index.js +2 -0
  8. package/dist/cjs/lib/index.js +27 -1
  9. package/dist/cjs/types/database_connections/index.d.ts +3 -0
  10. package/dist/cjs/types/database_connections/index.d.ts.map +1 -0
  11. package/dist/cjs/types/database_connections/postgresDatabaseConnection.d.ts +24 -0
  12. package/dist/cjs/types/database_connections/postgresDatabaseConnection.d.ts.map +1 -0
  13. package/dist/cjs/types/destinations/consoleDestination.d.ts.map +1 -1
  14. package/dist/cjs/types/destinations/csvFileDestination.d.ts.map +1 -1
  15. package/dist/cjs/types/destinations/postgresDestination.d.ts +2 -10
  16. package/dist/cjs/types/destinations/postgresDestination.d.ts.map +1 -1
  17. package/dist/cjs/types/destinations/sqlFileDestination.d.ts.map +1 -1
  18. package/dist/cjs/types/index.d.ts +2 -1
  19. package/dist/cjs/types/lib/index.d.ts +6 -1
  20. package/dist/cjs/types/lib/index.d.ts.map +1 -1
  21. package/dist/esm/database_connections/index.js +6 -0
  22. package/dist/esm/database_connections/postgresDatabaseConnection.js +46 -0
  23. package/dist/esm/destinations/consoleDestination.js +15 -1
  24. package/dist/esm/destinations/csvFileDestination.js +15 -1
  25. package/dist/esm/destinations/postgresDestination.js +19 -28
  26. package/dist/esm/destinations/sqlFileDestination.js +21 -7
  27. package/dist/esm/index.mjs +2 -0
  28. package/dist/esm/lib/index.js +27 -1
  29. package/dist/esm/types/database_connections/index.d.ts +3 -0
  30. package/dist/esm/types/database_connections/index.d.ts.map +1 -0
  31. package/dist/esm/types/database_connections/postgresDatabaseConnection.d.ts +24 -0
  32. package/dist/esm/types/database_connections/postgresDatabaseConnection.d.ts.map +1 -0
  33. package/dist/esm/types/destinations/consoleDestination.d.ts.map +1 -1
  34. package/dist/esm/types/destinations/csvFileDestination.d.ts.map +1 -1
  35. package/dist/esm/types/destinations/postgresDestination.d.ts +2 -10
  36. package/dist/esm/types/destinations/postgresDestination.d.ts.map +1 -1
  37. package/dist/esm/types/destinations/sqlFileDestination.d.ts.map +1 -1
  38. package/dist/esm/types/index.d.ts +2 -1
  39. package/dist/esm/types/lib/index.d.ts +6 -1
  40. package/dist/esm/types/lib/index.d.ts.map +1 -1
  41. package/package.json +1 -1
package/dist/cjs/database_connections/index.js ADDED
@@ -0,0 +1,4 @@
1
+ const { PostgresDatabaseConnection } = require("./postgresDatabaseConnection");
2
+ module.exports = {
3
+ PostgresDatabaseConnection
4
+ };
package/dist/cjs/database_connections/postgresDatabaseConnection.js ADDED
@@ -0,0 +1,55 @@
1
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
2
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
3
+ return new (P || (P = Promise))(function (resolve, reject) {
4
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
5
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
6
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
7
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
8
+ });
9
+ };
10
+ const DEFAULT_PORT = 5432;
11
+ const { Sequelize } = require('sequelize');
12
+ const debug = require('debug')('connection');
13
+ /**
14
+ *
15
+ * @param {Object} options
16
+ * @param {Object} options.connectionname
17
+ * @param {Object} options.username
18
+ * @param {Object} options.password
19
+ * @param {Object} options.database
20
+ * @param {Object} options.host
21
+ * @param {Object} options.port
22
+ * @returns Connection
23
+ */
24
+ function PostgresDatabaseConnection(options) {
25
+ return __awaiter(this, void 0, void 0, function* () {
26
+ const connectionname = options.connectionname;
27
+ const username = options.username;
28
+ const password = options.password;
29
+ const database = options.database;
30
+ const host = options.host;
31
+ const port = options.port || DEFAULT_PORT;
32
+ const sequelize = new Sequelize(database, username, password, {
33
+ host: host,
34
+ port: port,
35
+ dialect: 'postgres',
36
+ logging: false
37
+ });
38
+ try {
39
+ yield sequelize.authenticate();
40
+ debug('sequelize.authenticate succeeded');
41
+ return {
42
+ name: connectionname,
43
+ db: sequelize
44
+ };
45
+ }
46
+ catch (e) {
47
+ debug('sequelize.authenticate failed');
48
+ debug(e);
49
+ throw e;
50
+ }
51
+ });
52
+ }
53
+ module.exports = {
54
+ PostgresDatabaseConnection
55
+ };
package/dist/cjs/destinations/consoleDestination.js CHANGED
@@ -7,14 +7,30 @@ const { Writable } = require("node:stream");
7
7
  */
8
8
  function ConsoleDestination(options) {
9
9
  const includeErrors = options.includeErrors;
10
- return new Writable({
10
+ const writable = new Writable({
11
11
  objectMode: true,
12
12
  write(chunk, _, callback) {
13
+ var _a;
13
14
  if (chunk.errors.length === 0 || includeErrors)
14
15
  console.log(chunk);
16
+ // @ts-ignore
17
+ (_a = this.tasks) === null || _a === void 0 ? void 0 : _a.forEach(task => task.write(chunk));
15
18
  callback();
16
19
  }
17
20
  });
21
+ Object.assign(writable, {
22
+ setConnection: function (connection) {
23
+ this.connection = connection;
24
+ }.bind(writable),
25
+ getConnectionName: function () {
26
+ var _a;
27
+ return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.name;
28
+ }.bind(writable),
29
+ setTasks: function (tasks) {
30
+ this.tasks = tasks;
31
+ }.bind(writable)
32
+ });
33
+ return writable;
18
34
  }
19
35
  module.exports = {
20
36
  ConsoleDestination
package/dist/cjs/destinations/csvFileDestination.js CHANGED
@@ -23,9 +23,10 @@ function CSVFileDestination(options) {
23
23
  fileHandle = fd;
24
24
  }));
25
25
  let headersWritten = false;
26
- return new Writable({
26
+ const writable = new Writable({
27
27
  objectMode: true,
28
28
  write(chunk, _, callback) {
29
+ var _a;
29
30
  if (!headersWritten && headers) {
30
31
  if (quotationMarks) {
31
32
  fs.writeFileSync(fileHandle, `${chunk._columns.map(c => `"${c}"`).join(",")}\n`);
@@ -43,6 +44,8 @@ function CSVFileDestination(options) {
43
44
  fs.writeFileSync(fileHandle, `${chunk.map(c => `"${c}"`).join(",")}\n`);
44
45
  }
45
46
  }
47
+ // @ts-ignore
48
+ (_a = this.tasks) === null || _a === void 0 ? void 0 : _a.forEach(task => task.write(chunk));
46
49
  lastChunk = chunk;
47
50
  callback();
48
51
  },
@@ -51,6 +54,19 @@ function CSVFileDestination(options) {
51
54
  callback();
52
55
  }
53
56
  });
57
+ Object.assign(writable, {
58
+ setConnection: function (connection) {
59
+ this.connection = connection;
60
+ }.bind(writable),
61
+ getConnectionName: function () {
62
+ var _a;
63
+ return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.name;
64
+ }.bind(writable),
65
+ setTasks: function (tasks) {
66
+ this.tasks = tasks;
67
+ }.bind(writable)
68
+ });
69
+ return writable;
54
70
  }
55
71
  util.inherits(CSVFileDestination, EventEmitter);
56
72
  module.exports = {
package/dist/cjs/destinations/postgresDestination.js CHANGED
@@ -44,11 +44,7 @@ function writeInsertStatement(columnMapping, table, chunk) {
44
44
  *
45
45
  * @param {Object} options
46
46
  * @param {Object} options.table
47
- * @param {Object} options.username
48
- * @param {Object} options.password
49
- * @param {Object} options.database
50
- * @param {Object} options.host
51
- * @param {Object} options.port
47
+ * @param {Object} options.connectionname
52
48
  * @param {Object} options.mapping
53
49
  * @param {Object} options.includeErrors
54
50
  * @returns Transform
@@ -56,35 +52,16 @@ function writeInsertStatement(columnMapping, table, chunk) {
56
52
  function PostgresDestination(options) {
57
53
  EventEmitter.call(this);
58
54
  const table = options.table;
59
- const username = options.username;
60
- const password = options.password;
61
- const database = options.database;
62
- const host = options.host;
63
- const port = options.port || DEFAULT_PORT;
55
+ const connectionname = options.connectionname;
64
56
  const mapping = options.mapping;
65
57
  const includeErrors = options.includeErrors;
66
58
  let lastChunk;
67
- const sequelize = new Sequelize(database, username, password, {
68
- host: host,
69
- port: port,
70
- dialect: 'postgres',
71
- logging: false
72
- });
73
- try {
74
- sequelize.authenticate();
75
- debug('sequelize.authenticate succeeded');
76
- }
77
- catch (e) {
78
- debug('sequelize.authenticate failed');
79
- debug(e);
80
- throw e;
81
- }
82
- return new Transform({
59
+ const transform = new Transform({
83
60
  objectMode: true,
84
61
  emitClose: true,
85
62
  construct(callback) {
86
63
  // @ts-ignore
87
- this.sequelize = sequelize;
64
+ this.connectionname = connectionname;
88
65
  callback();
89
66
  },
90
67
  write(chunk, _, callback) {
@@ -94,12 +71,15 @@ function PostgresDestination(options) {
94
71
  insertStatement = writeInsertStatement(mapping, table, chunk);
95
72
  debug('Insert statement: [%s]', insertStatement);
96
73
  // @ts-ignore
97
- this.sequelize.query(insertStatement)
74
+ this.connection.db.query(insertStatement)
98
75
  .then(result => {
76
+ var _a;
99
77
  debug('result %o', result);
100
78
  chunk._result = result;
101
79
  lastChunk = chunk;
102
80
  // @ts-ignore
81
+ (_a = this.tasks) === null || _a === void 0 ? void 0 : _a.forEach(task => task.write(chunk));
82
+ // @ts-ignore
103
83
  callback(null, chunk);
104
84
  }).catch(error => {
105
85
  debug('error %o', error);
@@ -118,6 +98,19 @@ function PostgresDestination(options) {
118
98
  callback();
119
99
  }
120
100
  });
101
+ Object.assign(Transform.prototype, {
102
+ setConnection: function (connection) {
103
+ this.connection = connection;
104
+ }.bind(transform),
105
+ getConnectionName: function () {
106
+ var _a;
107
+ return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.name;
108
+ }.bind(transform),
109
+ setTasks: function (tasks) {
110
+ this.tasks = tasks;
111
+ }.bind(transform)
112
+ });
113
+ return transform;
121
114
  }
122
115
  module.exports = {
123
116
  PostgresDestination,
package/dist/cjs/destinations/sqlFileDestination.js CHANGED
@@ -37,8 +37,9 @@ function SQLFileDestination(options) {
37
37
  });
38
38
  }
39
39
  function writeUpdateStatement(chunk) {
40
+ //TODO
40
41
  }
41
- return new Writable({
42
+ const writable = new Writable({
42
43
  objectMode: true,
43
44
  write(chunk, _, callback) {
44
45
  if (chunk.errors.length === 0)
@@ -48,15 +49,29 @@ function SQLFileDestination(options) {
48
49
  writeInsertStatement(chunk);
49
50
  }
50
51
  }
51
- else if (sqlMode === SQL_MODE.UPDATE_MODE) {
52
- // @ts-ignore
53
- if (chunk.errors.length === 0 | includeErrors) {
54
- writeUpdateStatement(chunk);
55
- }
56
- }
52
+ // TODO - Disabled because Sonar
53
+ // } else if (sqlMode === SQL_MODE.UPDATE_MODE){
54
+ // // @ts-ignore
55
+ // if(chunk.errors.length === 0 | includeErrors){
56
+ // writeUpdateStatement(chunk)
57
+ // }
58
+ // }
57
59
  callback();
58
60
  }
59
61
  });
62
+ Object.assign(writable, {
63
+ setConnection: function (connection) {
64
+ this.connection = connection;
65
+ }.bind(writable),
66
+ getConnectionName: function () {
67
+ var _a;
68
+ return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.name;
69
+ }.bind(writable),
70
+ setTasks: function (tasks) {
71
+ this.tasks = tasks;
72
+ }.bind(writable)
73
+ });
74
+ return writable;
60
75
  }
61
76
  module.exports = {
62
77
  SQLFileDestination,
package/dist/cjs/index.js CHANGED
@@ -2,6 +2,7 @@ const Validators = require("./validators");
2
2
  const Transformers = require("./transformers");
3
3
  const Loaders = require("./loaders");
4
4
  const Destinations = require("./destinations");
5
+ const Connections = require("./database_connections");
5
6
  const Etl = require("./lib");
6
7
  module.exports = {
7
8
  Etl,
@@ -9,4 +10,5 @@ module.exports = {
9
10
  Transformers,
10
11
  Loaders,
11
12
  Destinations,
13
+ Connections
12
14
  };
package/dist/cjs/lib/index.js CHANGED
@@ -7,6 +7,7 @@ const { compose } = require("node:stream");
7
7
  * @typedef {Object} Etl
8
8
  * @function loader
9
9
  * @function pump
10
+ * @function connection
10
11
  * @function validator
11
12
  * @function destination
12
13
  * @function transform
@@ -18,6 +19,8 @@ const { compose } = require("node:stream");
18
19
  function Etl() {
19
20
  EventEmitter.call(this);
20
21
  let self = this;
22
+ self.beforeETLList = [];
23
+ self.connectionList = [];
21
24
  self.validatorList = [];
22
25
  self.transformationList = [];
23
26
  self.destinationList = [];
@@ -26,6 +29,9 @@ function Etl() {
26
29
  return self;
27
30
  };
28
31
  this.pump = () => {
32
+ this.beforeETLList.forEach(task => {
33
+ task.write({});
34
+ });
29
35
  this.loader
30
36
  .pump(this.loader)
31
37
  .pipe(compose(RowMetaData(), ...self.validatorList, ...self.transformationList, ...self.destinationList.map(dl => dl.on('result', (data) => self.emit('result', data))))
@@ -33,11 +39,31 @@ function Etl() {
33
39
  ).on('finish', (data) => self.emit('finish', data));
34
40
  return self;
35
41
  };
42
+ this.beforeETL = (pipelineTask) => {
43
+ const connectionname = pipelineTask.getConnectionName();
44
+ const connection = this.connectionList.filter(c => c.name === connectionname)[0];
45
+ if (!connection) {
46
+ throw new Error(`Connection with name ${connectionname} not found`);
47
+ }
48
+ pipelineTask.setConnection(connection);
49
+ self.beforeETLList.push(pipelineTask);
50
+ return self;
51
+ };
52
+ this.connection = (connection) => {
53
+ self.connectionList.push(connection);
54
+ return self;
55
+ };
36
56
  this.validator = (validator) => {
37
57
  self.validatorList.push(validator);
38
58
  return self;
39
59
  };
40
- this.destination = (destination) => {
60
+ this.destination = (destination, ...tasks) => {
61
+ const connectionname = destination.getConnectionName();
62
+ const connection = this.connectionList.filter(c => c.name === connectionname)[0];
63
+ if (connection) {
64
+ destination.setConnection(connection);
65
+ }
66
+ tasks && destination.setTasks(tasks);
41
67
  self.destinationList.push(destination);
42
68
  return self;
43
69
  };
package/dist/cjs/types/database_connections/index.d.ts ADDED
@@ -0,0 +1,3 @@
1
+ export { PostgresDatabaseConnection };
2
+ import { PostgresDatabaseConnection } from "./postgresDatabaseConnection";
3
+ //# sourceMappingURL=index.d.ts.map
package/dist/cjs/types/database_connections/index.d.ts.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/database_connections/index.js"],"names":[],"mappings":""}
package/dist/cjs/types/database_connections/postgresDatabaseConnection.d.ts ADDED
@@ -0,0 +1,24 @@
1
+ /**
2
+ *
3
+ * @param {Object} options
4
+ * @param {Object} options.connectionname
5
+ * @param {Object} options.username
6
+ * @param {Object} options.password
7
+ * @param {Object} options.database
8
+ * @param {Object} options.host
9
+ * @param {Object} options.port
10
+ * @returns Connection
11
+ */
12
+ export function PostgresDatabaseConnection(options: {
13
+ connectionname: any;
14
+ username: any;
15
+ password: any;
16
+ database: any;
17
+ host: any;
18
+ port: any;
19
+ }): Promise<{
20
+ name: any;
21
+ db: Sequelize;
22
+ }>;
23
+ import { Sequelize } from "sequelize/types/sequelize";
24
+ //# sourceMappingURL=postgresDatabaseConnection.d.ts.map
package/dist/cjs/types/database_connections/postgresDatabaseConnection.d.ts.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"postgresDatabaseConnection.d.ts","sourceRoot":"","sources":["../../../../src/database_connections/postgresDatabaseConnection.js"],"names":[],"mappings":"AAIA;;;;;;;;;;GAUG;AACH;IAR2B,cAAc;IACd,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,IAAI;;;;GA8B9B"}
package/dist/cjs/types/destinations/consoleDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"consoleDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/consoleDestination.js"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH;IAH2B,aAAa;aAavC"}
1
+ {"version":3,"file":"consoleDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/consoleDestination.js"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH;IAH2B,aAAa;aA6BvC"}
package/dist/cjs/types/destinations/csvFileDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"csvFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/csvFileDestination.js"],"names":[],"mappings":"AAKA;;;;;;;;GAQG;AACH;IAN2B,QAAQ;IACP,OAAO;IACP,aAAa;IACb,cAAc;aAyCzC"}
1
+ {"version":3,"file":"csvFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/csvFileDestination.js"],"names":[],"mappings":"AAKA;;;;;;;;GAQG;AACH;IAN2B,QAAQ;IACP,OAAO;IACP,aAAa;IACb,cAAc;aAyDzC"}
package/dist/cjs/types/destinations/postgresDestination.d.ts CHANGED
@@ -2,22 +2,14 @@
2
2
  *
3
3
  * @param {Object} options
4
4
  * @param {Object} options.table
5
- * @param {Object} options.username
6
- * @param {Object} options.password
7
- * @param {Object} options.database
8
- * @param {Object} options.host
9
- * @param {Object} options.port
5
+ * @param {Object} options.connectionname
10
6
  * @param {Object} options.mapping
11
7
  * @param {Object} options.includeErrors
12
8
  * @returns Transform
13
9
  */
14
10
  export function PostgresDestination(options: {
15
11
  table: any;
16
- username: any;
17
- password: any;
18
- database: any;
19
- host: any;
20
- port: any;
12
+ connectionname: any;
21
13
  mapping: any;
22
14
  includeErrors: any;
23
15
  }): Transform;
package/dist/cjs/types/destinations/postgresDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AA6CA;;;;;;;;;;;;GAYG;AACH;IAV2B,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,aAAa;cAqEvC;AA7GD,yFA4BC;AArCD,gDAEC;AAED,oEAGC"}
1
+ {"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AA6CA;;;;;;;;GAQG;AACH;IAN2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;cA+DvC;AAnGD,yFA4BC;AArCD,gDAEC;AAED,oEAGC"}
package/dist/cjs/types/destinations/sqlFileDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"sqlFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/sqlFileDestination.js"],"names":[],"mappings":"AAWA;;;;;;;;;GASG;AACH;IAP2B,QAAQ;IACN,IAAI,EAAtB,QAAQ;IACQ,KAAK;IACN,OAAO;IACL,aAAa;aA6CxC;uBA3DS,MAAM"}
1
+ {"version":3,"file":"sqlFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/sqlFileDestination.js"],"names":[],"mappings":"AAWA;;;;;;;;;GASG;AACH;IAP2B,QAAQ;IACN,IAAI,EAAtB,QAAQ;IACQ,KAAK;IACN,OAAO;IACL,aAAa;aA6DxC;uBA3ES,MAAM"}
package/dist/cjs/types/index.d.ts CHANGED
@@ -3,5 +3,6 @@ import Validators = require("./validators");
3
3
  import Transformers = require("./transformers");
4
4
  import Loaders = require("./loaders");
5
5
  import Destinations = require("./destinations");
6
- export { Etl, Validators, Transformers, Loaders, Destinations };
6
+ import Connections = require("./database_connections");
7
+ export { Etl, Validators, Transformers, Loaders, Destinations, Connections };
7
8
  //# sourceMappingURL=index.d.ts.map
package/dist/cjs/types/lib/index.d.ts CHANGED
@@ -3,6 +3,7 @@ export type Etl = any;
3
3
  * @typedef {Object} Etl
4
4
  * @function loader
5
5
  * @function pump
6
+ * @function connection
6
7
  * @function validator
7
8
  * @function destination
8
9
  * @function transform
@@ -13,13 +14,17 @@ export type Etl = any;
13
14
  */
14
15
  export function Etl(): void;
15
16
  export class Etl {
17
+ beforeETLList: any[];
18
+ connectionList: any[];
16
19
  validatorList: any[];
17
20
  transformationList: any[];
18
21
  destinationList: any[];
19
22
  loader: (loader: any) => Etl;
20
23
  pump: () => Etl;
24
+ beforeETL: (pipelineTask: any) => Etl;
25
+ connection: (connection: any) => Etl;
21
26
  validator: (validator: any) => Etl;
22
- destination: (destination: any) => Etl;
27
+ destination: (destination: any, ...tasks: any[]) => Etl;
23
28
  transform: (transform: any) => Etl;
24
29
  }
25
30
  //# sourceMappingURL=index.d.ts.map
package/dist/cjs/types/lib/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAMA;;;;;;;GAOG;AAEH;;;GAGG;AACH,4BA2CC;;IAxCG,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,6BAGC;IAED,gBAaC;IAED,mCAGC;IAED,uCAGC;IAED,mCAGC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAMA;;;;;;;;GAQG;AAEH;;;GAGG;AACH,4BAwEC;;IArEG,qBAAuB;IACvB,sBAAwB;IACxB,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,6BAGC;IAED,gBAgBC;IAED,sCASC;IAED,qCAGC;IAED,mCAGC;IAED,wDAWC;IAED,mCAGC"}
package/dist/esm/database_connections/index.js ADDED
@@ -0,0 +1,6 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ const { PostgresDatabaseConnection } = require("./postgresDatabaseConnection");
4
+ module.exports = {
5
+ PostgresDatabaseConnection
6
+ };
package/dist/esm/database_connections/postgresDatabaseConnection.js ADDED
@@ -0,0 +1,46 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ const DEFAULT_PORT = 5432;
4
+ const { Sequelize } = require('sequelize');
5
+ const debug = require('debug')('connection');
6
+ /**
7
+ *
8
+ * @param {Object} options
9
+ * @param {Object} options.connectionname
10
+ * @param {Object} options.username
11
+ * @param {Object} options.password
12
+ * @param {Object} options.database
13
+ * @param {Object} options.host
14
+ * @param {Object} options.port
15
+ * @returns Connection
16
+ */
17
+ async function PostgresDatabaseConnection(options) {
18
+ const connectionname = options.connectionname;
19
+ const username = options.username;
20
+ const password = options.password;
21
+ const database = options.database;
22
+ const host = options.host;
23
+ const port = options.port || DEFAULT_PORT;
24
+ const sequelize = new Sequelize(database, username, password, {
25
+ host: host,
26
+ port: port,
27
+ dialect: 'postgres',
28
+ logging: false
29
+ });
30
+ try {
31
+ await sequelize.authenticate();
32
+ debug('sequelize.authenticate succeeded');
33
+ return {
34
+ name: connectionname,
35
+ db: sequelize
36
+ };
37
+ }
38
+ catch (e) {
39
+ debug('sequelize.authenticate failed');
40
+ debug(e);
41
+ throw e;
42
+ }
43
+ }
44
+ module.exports = {
45
+ PostgresDatabaseConnection
46
+ };
package/dist/esm/destinations/consoleDestination.js CHANGED
@@ -9,14 +9,28 @@ const { Writable } = require("node:stream");
9
9
  */
10
10
  function ConsoleDestination(options) {
11
11
  const includeErrors = options.includeErrors;
12
- return new Writable({
12
+ const writable = new Writable({
13
13
  objectMode: true,
14
14
  write(chunk, _, callback) {
15
15
  if (chunk.errors.length === 0 || includeErrors)
16
16
  console.log(chunk);
17
+ // @ts-ignore
18
+ this.tasks?.forEach(task => task.write(chunk));
17
19
  callback();
18
20
  }
19
21
  });
22
+ Object.assign(writable, {
23
+ setConnection: function (connection) {
24
+ this.connection = connection;
25
+ }.bind(writable),
26
+ getConnectionName: function () {
27
+ return this.connection?.name;
28
+ }.bind(writable),
29
+ setTasks: function (tasks) {
30
+ this.tasks = tasks;
31
+ }.bind(writable)
32
+ });
33
+ return writable;
20
34
  }
21
35
  module.exports = {
22
36
  ConsoleDestination
package/dist/esm/destinations/csvFileDestination.js CHANGED
@@ -25,7 +25,7 @@ function CSVFileDestination(options) {
25
25
  fileHandle = fd;
26
26
  }));
27
27
  let headersWritten = false;
28
- return new Writable({
28
+ const writable = new Writable({
29
29
  objectMode: true,
30
30
  write(chunk, _, callback) {
31
31
  if (!headersWritten && headers) {
@@ -45,6 +45,8 @@ function CSVFileDestination(options) {
45
45
  fs.writeFileSync(fileHandle, `${chunk.map(c => `"${c}"`).join(",")}\n`);
46
46
  }
47
47
  }
48
+ // @ts-ignore
49
+ this.tasks?.forEach(task => task.write(chunk));
48
50
  lastChunk = chunk;
49
51
  callback();
50
52
  },
@@ -53,6 +55,18 @@ function CSVFileDestination(options) {
53
55
  callback();
54
56
  }
55
57
  });
58
+ Object.assign(writable, {
59
+ setConnection: function (connection) {
60
+ this.connection = connection;
61
+ }.bind(writable),
62
+ getConnectionName: function () {
63
+ return this.connection?.name;
64
+ }.bind(writable),
65
+ setTasks: function (tasks) {
66
+ this.tasks = tasks;
67
+ }.bind(writable)
68
+ });
69
+ return writable;
56
70
  }
57
71
  util.inherits(CSVFileDestination, EventEmitter);
58
72
  module.exports = {
package/dist/esm/destinations/postgresDestination.js CHANGED
@@ -46,11 +46,7 @@ function writeInsertStatement(columnMapping, table, chunk) {
46
46
  *
47
47
  * @param {Object} options
48
48
  * @param {Object} options.table
49
- * @param {Object} options.username
50
- * @param {Object} options.password
51
- * @param {Object} options.database
52
- * @param {Object} options.host
53
- * @param {Object} options.port
49
+ * @param {Object} options.connectionname
54
50
  * @param {Object} options.mapping
55
51
  * @param {Object} options.includeErrors
56
52
  * @returns Transform
@@ -58,35 +54,16 @@ function writeInsertStatement(columnMapping, table, chunk) {
58
54
  function PostgresDestination(options) {
59
55
  EventEmitter.call(this);
60
56
  const table = options.table;
61
- const username = options.username;
62
- const password = options.password;
63
- const database = options.database;
64
- const host = options.host;
65
- const port = options.port || DEFAULT_PORT;
57
+ const connectionname = options.connectionname;
66
58
  const mapping = options.mapping;
67
59
  const includeErrors = options.includeErrors;
68
60
  let lastChunk;
69
- const sequelize = new Sequelize(database, username, password, {
70
- host: host,
71
- port: port,
72
- dialect: 'postgres',
73
- logging: false
74
- });
75
- try {
76
- sequelize.authenticate();
77
- debug('sequelize.authenticate succeeded');
78
- }
79
- catch (e) {
80
- debug('sequelize.authenticate failed');
81
- debug(e);
82
- throw e;
83
- }
84
- return new Transform({
61
+ const transform = new Transform({
85
62
  objectMode: true,
86
63
  emitClose: true,
87
64
  construct(callback) {
88
65
  // @ts-ignore
89
- this.sequelize = sequelize;
66
+ this.connectionname = connectionname;
90
67
  callback();
91
68
  },
92
69
  write(chunk, _, callback) {
@@ -96,12 +73,14 @@ function PostgresDestination(options) {
96
73
  insertStatement = writeInsertStatement(mapping, table, chunk);
97
74
  debug('Insert statement: [%s]', insertStatement);
98
75
  // @ts-ignore
99
- this.sequelize.query(insertStatement)
76
+ this.connection.db.query(insertStatement)
100
77
  .then(result => {
101
78
  debug('result %o', result);
102
79
  chunk._result = result;
103
80
  lastChunk = chunk;
104
81
  // @ts-ignore
82
+ this.tasks?.forEach(task => task.write(chunk));
83
+ // @ts-ignore
105
84
  callback(null, chunk);
106
85
  }).catch(error => {
107
86
  debug('error %o', error);
@@ -120,6 +99,18 @@ function PostgresDestination(options) {
120
99
  callback();
121
100
  }
122
101
  });
102
+ Object.assign(Transform.prototype, {
103
+ setConnection: function (connection) {
104
+ this.connection = connection;
105
+ }.bind(transform),
106
+ getConnectionName: function () {
107
+ return this.connection?.name;
108
+ }.bind(transform),
109
+ setTasks: function (tasks) {
110
+ this.tasks = tasks;
111
+ }.bind(transform)
112
+ });
113
+ return transform;
123
114
  }
124
115
  module.exports = {
125
116
  PostgresDestination,
package/dist/esm/destinations/sqlFileDestination.js CHANGED
@@ -39,8 +39,9 @@ function SQLFileDestination(options) {
39
39
  });
40
40
  }
41
41
  function writeUpdateStatement(chunk) {
42
+ //TODO
42
43
  }
43
- return new Writable({
44
+ const writable = new Writable({
44
45
  objectMode: true,
45
46
  write(chunk, _, callback) {
46
47
  if (chunk.errors.length === 0)
@@ -50,15 +51,28 @@ function SQLFileDestination(options) {
50
51
  writeInsertStatement(chunk);
51
52
  }
52
53
  }
53
- else if (sqlMode === SQL_MODE.UPDATE_MODE) {
54
- // @ts-ignore
55
- if (chunk.errors.length === 0 | includeErrors) {
56
- writeUpdateStatement(chunk);
57
- }
58
- }
54
+ // TODO - Disabled because Sonar
55
+ // } else if (sqlMode === SQL_MODE.UPDATE_MODE){
56
+ // // @ts-ignore
57
+ // if(chunk.errors.length === 0 | includeErrors){
58
+ // writeUpdateStatement(chunk)
59
+ // }
60
+ // }
59
61
  callback();
60
62
  }
61
63
  });
64
+ Object.assign(writable, {
65
+ setConnection: function (connection) {
66
+ this.connection = connection;
67
+ }.bind(writable),
68
+ getConnectionName: function () {
69
+ return this.connection?.name;
70
+ }.bind(writable),
71
+ setTasks: function (tasks) {
72
+ this.tasks = tasks;
73
+ }.bind(writable)
74
+ });
75
+ return writable;
62
76
  }
63
77
  module.exports = {
64
78
  SQLFileDestination,
package/dist/esm/index.mjs CHANGED
@@ -4,6 +4,7 @@ const Validators = require("./validators");
4
4
  const Transformers = require("./transformers");
5
5
  const Loaders = require("./loaders");
6
6
  const Destinations = require("./destinations");
7
+ const Connections = require("./database_connections");
7
8
  const Etl = require("./lib");
8
9
  module.exports = {
9
10
  Etl,
@@ -11,4 +12,5 @@ module.exports = {
11
12
  Transformers,
12
13
  Loaders,
13
14
  Destinations,
15
+ Connections
14
16
  };
package/dist/esm/lib/index.js CHANGED
@@ -9,6 +9,7 @@ const { compose } = require("node:stream");
9
9
  * @typedef {Object} Etl
10
10
  * @function loader
11
11
  * @function pump
12
+ * @function connection
12
13
  * @function validator
13
14
  * @function destination
14
15
  * @function transform
@@ -20,6 +21,8 @@ const { compose } = require("node:stream");
20
21
  function Etl() {
21
22
  EventEmitter.call(this);
22
23
  let self = this;
24
+ self.beforeETLList = [];
25
+ self.connectionList = [];
23
26
  self.validatorList = [];
24
27
  self.transformationList = [];
25
28
  self.destinationList = [];
@@ -28,6 +31,9 @@ function Etl() {
28
31
  return self;
29
32
  };
30
33
  this.pump = () => {
34
+ this.beforeETLList.forEach(task => {
35
+ task.write({});
36
+ });
31
37
  this.loader
32
38
  .pump(this.loader)
33
39
  .pipe(compose(RowMetaData(), ...self.validatorList, ...self.transformationList, ...self.destinationList.map(dl => dl.on('result', (data) => self.emit('result', data))))
@@ -35,11 +41,31 @@ function Etl() {
35
41
  ).on('finish', (data) => self.emit('finish', data));
36
42
  return self;
37
43
  };
44
+ this.beforeETL = (pipelineTask) => {
45
+ const connectionname = pipelineTask.getConnectionName();
46
+ const connection = this.connectionList.filter(c => c.name === connectionname)[0];
47
+ if (!connection) {
48
+ throw new Error(`Connection with name ${connectionname} not found`);
49
+ }
50
+ pipelineTask.setConnection(connection);
51
+ self.beforeETLList.push(pipelineTask);
52
+ return self;
53
+ };
54
+ this.connection = (connection) => {
55
+ self.connectionList.push(connection);
56
+ return self;
57
+ };
38
58
  this.validator = (validator) => {
39
59
  self.validatorList.push(validator);
40
60
  return self;
41
61
  };
42
- this.destination = (destination) => {
62
+ this.destination = (destination, ...tasks) => {
63
+ const connectionname = destination.getConnectionName();
64
+ const connection = this.connectionList.filter(c => c.name === connectionname)[0];
65
+ if (connection) {
66
+ destination.setConnection(connection);
67
+ }
68
+ tasks && destination.setTasks(tasks);
43
69
  self.destinationList.push(destination);
44
70
  return self;
45
71
  };
package/dist/esm/types/database_connections/index.d.ts ADDED
@@ -0,0 +1,3 @@
1
+ export { PostgresDatabaseConnection };
2
+ import { PostgresDatabaseConnection } from "./postgresDatabaseConnection";
3
+ //# sourceMappingURL=index.d.ts.map
package/dist/esm/types/database_connections/index.d.ts.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/database_connections/index.js"],"names":[],"mappings":""}
package/dist/esm/types/database_connections/postgresDatabaseConnection.d.ts ADDED
@@ -0,0 +1,24 @@
1
+ /**
2
+ *
3
+ * @param {Object} options
4
+ * @param {Object} options.connectionname
5
+ * @param {Object} options.username
6
+ * @param {Object} options.password
7
+ * @param {Object} options.database
8
+ * @param {Object} options.host
9
+ * @param {Object} options.port
10
+ * @returns Connection
11
+ */
12
+ export function PostgresDatabaseConnection(options: {
13
+ connectionname: any;
14
+ username: any;
15
+ password: any;
16
+ database: any;
17
+ host: any;
18
+ port: any;
19
+ }): Promise<{
20
+ name: any;
21
+ db: Sequelize;
22
+ }>;
23
+ import { Sequelize } from "../../node_modules/sequelize/types/sequelize";
24
+ //# sourceMappingURL=postgresDatabaseConnection.d.ts.map
package/dist/esm/types/database_connections/postgresDatabaseConnection.d.ts.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"postgresDatabaseConnection.d.ts","sourceRoot":"","sources":["../../../../src/database_connections/postgresDatabaseConnection.js"],"names":[],"mappings":"AAIA;;;;;;;;;;GAUG;AACH;IAR2B,cAAc;IACd,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,IAAI;;;;GA8B9B"}
package/dist/esm/types/destinations/consoleDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"consoleDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/consoleDestination.js"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH;IAH2B,aAAa;aAavC"}
1
+ {"version":3,"file":"consoleDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/consoleDestination.js"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH;IAH2B,aAAa;aA6BvC"}
package/dist/esm/types/destinations/csvFileDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"csvFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/csvFileDestination.js"],"names":[],"mappings":"AAKA;;;;;;;;GAQG;AACH;IAN2B,QAAQ;IACP,OAAO;IACP,aAAa;IACb,cAAc;aAyCzC"}
1
+ {"version":3,"file":"csvFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/csvFileDestination.js"],"names":[],"mappings":"AAKA;;;;;;;;GAQG;AACH;IAN2B,QAAQ;IACP,OAAO;IACP,aAAa;IACb,cAAc;aAyDzC"}
package/dist/esm/types/destinations/postgresDestination.d.ts CHANGED
@@ -2,22 +2,14 @@
2
2
  *
3
3
  * @param {Object} options
4
4
  * @param {Object} options.table
5
- * @param {Object} options.username
6
- * @param {Object} options.password
7
- * @param {Object} options.database
8
- * @param {Object} options.host
9
- * @param {Object} options.port
5
+ * @param {Object} options.connectionname
10
6
  * @param {Object} options.mapping
11
7
  * @param {Object} options.includeErrors
12
8
  * @returns Transform
13
9
  */
14
10
  export function PostgresDestination(options: {
15
11
  table: any;
16
- username: any;
17
- password: any;
18
- database: any;
19
- host: any;
20
- port: any;
12
+ connectionname: any;
21
13
  mapping: any;
22
14
  includeErrors: any;
23
15
  }): Transform;
package/dist/esm/types/destinations/postgresDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AA6CA;;;;;;;;;;;;GAYG;AACH;IAV2B,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,aAAa;cAqEvC;AA7GD,yFA4BC;AArCD,gDAEC;AAED,oEAGC"}
1
+ {"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AA6CA;;;;;;;;GAQG;AACH;IAN2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;cA+DvC;AAnGD,yFA4BC;AArCD,gDAEC;AAED,oEAGC"}
package/dist/esm/types/destinations/sqlFileDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"sqlFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/sqlFileDestination.js"],"names":[],"mappings":"AAWA;;;;;;;;;GASG;AACH;IAP2B,QAAQ;IACN,IAAI,EAAtB,QAAQ;IACQ,KAAK;IACN,OAAO;IACL,aAAa;aA6CxC;uBA3DS,MAAM"}
1
+ {"version":3,"file":"sqlFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/sqlFileDestination.js"],"names":[],"mappings":"AAWA;;;;;;;;;GASG;AACH;IAP2B,QAAQ;IACN,IAAI,EAAtB,QAAQ;IACQ,KAAK;IACN,OAAO;IACL,aAAa;aA6DxC;uBA3ES,MAAM"}
package/dist/esm/types/index.d.ts CHANGED
@@ -3,5 +3,6 @@ import Validators = require("./validators");
3
3
  import Transformers = require("./transformers");
4
4
  import Loaders = require("./loaders");
5
5
  import Destinations = require("./destinations");
6
- export { Etl, Validators, Transformers, Loaders, Destinations };
6
+ import Connections = require("./database_connections");
7
+ export { Etl, Validators, Transformers, Loaders, Destinations, Connections };
7
8
  //# sourceMappingURL=index.d.ts.map
package/dist/esm/types/lib/index.d.ts CHANGED
@@ -3,6 +3,7 @@ export type Etl = any;
3
3
  * @typedef {Object} Etl
4
4
  * @function loader
5
5
  * @function pump
6
+ * @function connection
6
7
  * @function validator
7
8
  * @function destination
8
9
  * @function transform
@@ -13,13 +14,17 @@ export type Etl = any;
13
14
  */
14
15
  export function Etl(): void;
15
16
  export class Etl {
17
+ beforeETLList: any[];
18
+ connectionList: any[];
16
19
  validatorList: any[];
17
20
  transformationList: any[];
18
21
  destinationList: any[];
19
22
  loader: (loader: any) => Etl;
20
23
  pump: () => Etl;
24
+ beforeETL: (pipelineTask: any) => Etl;
25
+ connection: (connection: any) => Etl;
21
26
  validator: (validator: any) => Etl;
22
- destination: (destination: any) => Etl;
27
+ destination: (destination: any, ...tasks: any[]) => Etl;
23
28
  transform: (transform: any) => Etl;
24
29
  }
25
30
  //# sourceMappingURL=index.d.ts.map
package/dist/esm/types/lib/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAMA;;;;;;;GAOG;AAEH;;;GAGG;AACH,4BA2CC;;IAxCG,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,6BAGC;IAED,gBAaC;IAED,mCAGC;IAED,uCAGC;IAED,mCAGC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAMA;;;;;;;;GAQG;AAEH;;;GAGG;AACH,4BAwEC;;IArEG,qBAAuB;IACvB,sBAAwB;IACxB,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,6BAGC;IAED,gBAgBC;IAED,sCASC;IAED,qCAGC;IAED,mCAGC;IAED,wDAWC;IAED,mCAGC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ffc-pay-etl-framework",
3
- "version": "0.1.14",
3
+ "version": "1.0.0",
4
4
  "publisher": "Defra",
5
5
  "main": "dist/cjs/index.js",
6
6
  "private": false,