ffc-pay-etl-framework 0.1.14 → 1.1.14-alpha.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. package/dist/cjs/destinations/consoleDestination.js +15 -1
  2. package/dist/cjs/destinations/csvFileDestination.js +15 -1
  3. package/dist/cjs/destinations/postgresDestination.js +19 -28
  4. package/dist/cjs/destinations/sqlFileDestination.js +13 -1
  5. package/dist/cjs/lib/index.js +34 -2
  6. package/dist/cjs/types/destinations/consoleDestination.d.ts.map +1 -1
  7. package/dist/cjs/types/destinations/csvFileDestination.d.ts.map +1 -1
  8. package/dist/cjs/types/destinations/postgresDestination.d.ts +2 -10
  9. package/dist/cjs/types/destinations/postgresDestination.d.ts.map +1 -1
  10. package/dist/cjs/types/destinations/sqlFileDestination.d.ts.map +1 -1
  11. package/dist/cjs/types/lib/index.d.ts +7 -2
  12. package/dist/cjs/types/lib/index.d.ts.map +1 -1
  13. package/dist/esm/destinations/consoleDestination.js +15 -1
  14. package/dist/esm/destinations/csvFileDestination.js +15 -1
  15. package/dist/esm/destinations/postgresDestination.js +19 -28
  16. package/dist/esm/destinations/sqlFileDestination.js +13 -1
  17. package/dist/esm/lib/index.js +34 -2
  18. package/dist/esm/types/destinations/consoleDestination.d.ts.map +1 -1
  19. package/dist/esm/types/destinations/csvFileDestination.d.ts.map +1 -1
  20. package/dist/esm/types/destinations/postgresDestination.d.ts +2 -10
  21. package/dist/esm/types/destinations/postgresDestination.d.ts.map +1 -1
  22. package/dist/esm/types/destinations/sqlFileDestination.d.ts.map +1 -1
  23. package/dist/esm/types/lib/index.d.ts +7 -2
  24. package/dist/esm/types/lib/index.d.ts.map +1 -1
  25. package/package.json +1 -1
package/dist/cjs/destinations/consoleDestination.js CHANGED
@@ -7,14 +7,28 @@ const { Writable } = require("node:stream");
7
7
  */
8
8
  function ConsoleDestination(options) {
9
9
  const includeErrors = options.includeErrors;
10
- return new Writable({
10
+ const writable = new Writable({
11
11
  objectMode: true,
12
12
  write(chunk, _, callback) {
13
13
  if (chunk.errors.length === 0 || includeErrors)
14
14
  console.log(chunk);
15
+ // @ts-ignore
16
+ this.tasks && this.tasks.forEach(task => task.write(chunk));
15
17
  callback();
16
18
  }
17
19
  });
20
+ Object.assign(writable, {
21
+ setConnection: function (connection) {
22
+ this.connection = connection;
23
+ }.bind(writable),
24
+ getConnectionName: function () {
25
+ return this.connection && this.connection.name;
26
+ }.bind(writable),
27
+ setTasks: function (tasks) {
28
+ this.tasks = tasks;
29
+ }.bind(writable)
30
+ });
31
+ return writable;
18
32
  }
19
33
  module.exports = {
20
34
  ConsoleDestination
package/dist/cjs/destinations/csvFileDestination.js CHANGED
@@ -23,7 +23,7 @@ function CSVFileDestination(options) {
23
23
  fileHandle = fd;
24
24
  }));
25
25
  let headersWritten = false;
26
- return new Writable({
26
+ const writable = new Writable({
27
27
  objectMode: true,
28
28
  write(chunk, _, callback) {
29
29
  if (!headersWritten && headers) {
@@ -43,6 +43,8 @@ function CSVFileDestination(options) {
43
43
  fs.writeFileSync(fileHandle, `${chunk.map(c => `"${c}"`).join(",")}\n`);
44
44
  }
45
45
  }
46
+ // @ts-ignore
47
+ this.tasks && this.tasks.forEach(task => task.write(chunk));
46
48
  lastChunk = chunk;
47
49
  callback();
48
50
  },
@@ -51,6 +53,18 @@ function CSVFileDestination(options) {
51
53
  callback();
52
54
  }
53
55
  });
56
+ Object.assign(writable, {
57
+ setConnection: function (connection) {
58
+ this.connection = connection;
59
+ }.bind(writable),
60
+ getConnectionName: function () {
61
+ return this.connection && this.connection.name;
62
+ }.bind(writable),
63
+ setTasks: function (tasks) {
64
+ this.tasks = tasks;
65
+ }.bind(writable)
66
+ });
67
+ return writable;
54
68
  }
55
69
  util.inherits(CSVFileDestination, EventEmitter);
56
70
  module.exports = {
package/dist/cjs/destinations/postgresDestination.js CHANGED
@@ -44,11 +44,7 @@ function writeInsertStatement(columnMapping, table, chunk) {
44
44
  *
45
45
  * @param {Object} options
46
46
  * @param {Object} options.table
47
- * @param {Object} options.username
48
- * @param {Object} options.password
49
- * @param {Object} options.database
50
- * @param {Object} options.host
51
- * @param {Object} options.port
47
+ * @param {Object} options.connectionname
52
48
  * @param {Object} options.mapping
53
49
  * @param {Object} options.includeErrors
54
50
  * @returns Transform
@@ -56,35 +52,16 @@ function writeInsertStatement(columnMapping, table, chunk) {
56
52
  function PostgresDestination(options) {
57
53
  EventEmitter.call(this);
58
54
  const table = options.table;
59
- const username = options.username;
60
- const password = options.password;
61
- const database = options.database;
62
- const host = options.host;
63
- const port = options.port || DEFAULT_PORT;
55
+ const connectionname = options.connectionname;
64
56
  const mapping = options.mapping;
65
57
  const includeErrors = options.includeErrors;
66
58
  let lastChunk;
67
- const sequelize = new Sequelize(database, username, password, {
68
- host: host,
69
- port: port,
70
- dialect: 'postgres',
71
- logging: false
72
- });
73
- try {
74
- sequelize.authenticate();
75
- debug('sequelize.authenticate succeeded');
76
- }
77
- catch (e) {
78
- debug('sequelize.authenticate failed');
79
- debug(e);
80
- throw e;
81
- }
82
- return new Transform({
59
+ const transform = new Transform({
83
60
  objectMode: true,
84
61
  emitClose: true,
85
62
  construct(callback) {
86
63
  // @ts-ignore
87
- this.sequelize = sequelize;
64
+ this.connectionname = connectionname;
88
65
  callback();
89
66
  },
90
67
  write(chunk, _, callback) {
@@ -94,12 +71,14 @@ function PostgresDestination(options) {
94
71
  insertStatement = writeInsertStatement(mapping, table, chunk);
95
72
  debug('Insert statement: [%s]', insertStatement);
96
73
  // @ts-ignore
97
- this.sequelize.query(insertStatement)
74
+ this.connection.db.query(insertStatement)
98
75
  .then(result => {
99
76
  debug('result %o', result);
100
77
  chunk._result = result;
101
78
  lastChunk = chunk;
102
79
  // @ts-ignore
80
+ this.tasks && this.tasks.forEach(task => task.write(chunk));
81
+ // @ts-ignore
103
82
  callback(null, chunk);
104
83
  }).catch(error => {
105
84
  debug('error %o', error);
@@ -118,6 +97,18 @@ function PostgresDestination(options) {
118
97
  callback();
119
98
  }
120
99
  });
100
+ Object.assign(Transform.prototype, {
101
+ setConnection: function (connection) {
102
+ this.connection = connection;
103
+ }.bind(transform),
104
+ getConnectionName: function () {
105
+ return this.connection && this.connection.name;
106
+ }.bind(transform),
107
+ setTasks: function (tasks) {
108
+ this.tasks = tasks;
109
+ }.bind(transform)
110
+ });
111
+ return transform;
121
112
  }
122
113
  module.exports = {
123
114
  PostgresDestination,
package/dist/cjs/destinations/sqlFileDestination.js CHANGED
@@ -38,7 +38,7 @@ function SQLFileDestination(options) {
38
38
  }
39
39
  function writeUpdateStatement(chunk) {
40
40
  }
41
- return new Writable({
41
+ const writable = new Writable({
42
42
  objectMode: true,
43
43
  write(chunk, _, callback) {
44
44
  if (chunk.errors.length === 0)
@@ -57,6 +57,18 @@ function SQLFileDestination(options) {
57
57
  callback();
58
58
  }
59
59
  });
60
+ Object.assign(writable, {
61
+ setConnection: function (connection) {
62
+ this.connection = connection;
63
+ }.bind(writable),
64
+ getConnectionName: function () {
65
+ return this.connection && this.connection.name;
66
+ }.bind(writable),
67
+ setTasks: function (tasks) {
68
+ this.tasks = tasks;
69
+ }.bind(writable)
70
+ });
71
+ return writable;
60
72
  }
61
73
  module.exports = {
62
74
  SQLFileDestination,
package/dist/cjs/lib/index.js CHANGED
@@ -7,6 +7,7 @@ const { compose } = require("node:stream");
7
7
  * @typedef {Object} Etl
8
8
  * @function loader
9
9
  * @function pump
10
+ * @function connection
10
11
  * @function validator
11
12
  * @function destination
12
13
  * @function transform
@@ -18,6 +19,8 @@ const { compose } = require("node:stream");
18
19
  function Etl() {
19
20
  EventEmitter.call(this);
20
21
  let self = this;
22
+ self.beforeETLList = [];
23
+ self.connectionList = [];
21
24
  self.validatorList = [];
22
25
  self.transformationList = [];
23
26
  self.destinationList = [];
@@ -26,6 +29,9 @@ function Etl() {
26
29
  return self;
27
30
  };
28
31
  this.pump = () => {
32
+ this.beforeETLList.forEach(task => {
33
+ task.write({});
34
+ });
29
35
  this.loader
30
36
  .pump(this.loader)
31
37
  .pipe(compose(RowMetaData(), ...self.validatorList, ...self.transformationList, ...self.destinationList.map(dl => dl.on('result', (data) => self.emit('result', data))))
@@ -33,15 +39,41 @@ function Etl() {
33
39
  ).on('finish', (data) => self.emit('finish', data));
34
40
  return self;
35
41
  };
42
+ this.beforeETL = (pipelineTask) => {
43
+ const connectionname = pipelineTask.getConnectionName();
44
+ const connection = this.connectionList.filter(c => c.name === connectionname);
45
+ if (!connection) {
46
+ throw new Error(`Connection with name ${connectionname} not found`);
47
+ }
48
+ pipelineTask.setConnection(connection);
49
+ self.beforeETLList.push(pipelineTask);
50
+ return self;
51
+ };
52
+ this.connection = (connection) => {
53
+ self.connectionList.push(connection);
54
+ return self;
55
+ };
36
56
  this.validator = (validator) => {
37
57
  self.validatorList.push(validator);
38
58
  return self;
39
59
  };
40
- this.destination = (destination) => {
60
+ this.destination = (destination, ...tasks) => {
61
+ const connectionname = destination.getConnectionName();
62
+ const connection = this.connectionList.filter(c => c.name === connectionname)[0];
63
+ if (connection) {
64
+ destination.setConnection(connection);
65
+ }
66
+ tasks && destination.setTasks(tasks);
41
67
  self.destinationList.push(destination);
42
68
  return self;
43
69
  };
44
- this.transform = (transform) => {
70
+ this.transform = (transform, ...tasks) => {
71
+ /* The utility of being able to perform additional tasks for each transformation remains to be seen */
72
+ tasks.forEach(task => {
73
+ const connection = this.connectionList.filter(c => c.name === connectionname)[0];
74
+ task.setConnection(connection);
75
+ });
76
+ transform.setTasks(tasks);
45
77
  self.transformationList.push(transform);
46
78
  return self;
47
79
  };
package/dist/cjs/types/destinations/consoleDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"consoleDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/consoleDestination.js"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH;IAH2B,aAAa;aAavC"}
1
+ {"version":3,"file":"consoleDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/consoleDestination.js"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH;IAH2B,aAAa;aA6BvC"}
package/dist/cjs/types/destinations/csvFileDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"csvFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/csvFileDestination.js"],"names":[],"mappings":"AAKA;;;;;;;;GAQG;AACH;IAN2B,QAAQ;IACP,OAAO;IACP,aAAa;IACb,cAAc;aAyCzC"}
1
+ {"version":3,"file":"csvFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/csvFileDestination.js"],"names":[],"mappings":"AAKA;;;;;;;;GAQG;AACH;IAN2B,QAAQ;IACP,OAAO;IACP,aAAa;IACb,cAAc;aAyDzC"}
package/dist/cjs/types/destinations/postgresDestination.d.ts CHANGED
@@ -2,22 +2,14 @@
2
2
  *
3
3
  * @param {Object} options
4
4
  * @param {Object} options.table
5
- * @param {Object} options.username
6
- * @param {Object} options.password
7
- * @param {Object} options.database
8
- * @param {Object} options.host
9
- * @param {Object} options.port
5
+ * @param {Object} options.connectionname
10
6
  * @param {Object} options.mapping
11
7
  * @param {Object} options.includeErrors
12
8
  * @returns Transform
13
9
  */
14
10
  export function PostgresDestination(options: {
15
11
  table: any;
16
- username: any;
17
- password: any;
18
- database: any;
19
- host: any;
20
- port: any;
12
+ connectionname: any;
21
13
  mapping: any;
22
14
  includeErrors: any;
23
15
  }): Transform;
package/dist/cjs/types/destinations/postgresDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AA6CA;;;;;;;;;;;;GAYG;AACH;IAV2B,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,aAAa;cAqEvC;AA7GD,yFA4BC;AArCD,gDAEC;AAED,oEAGC"}
1
+ {"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AA6CA;;;;;;;;GAQG;AACH;IAN2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;cA+DvC;AAnGD,yFA4BC;AArCD,gDAEC;AAED,oEAGC"}
package/dist/cjs/types/destinations/sqlFileDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"sqlFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/sqlFileDestination.js"],"names":[],"mappings":"AAWA;;;;;;;;;GASG;AACH;IAP2B,QAAQ;IACN,IAAI,EAAtB,QAAQ;IACQ,KAAK;IACN,OAAO;IACL,aAAa;aA6CxC;uBA3DS,MAAM"}
1
+ {"version":3,"file":"sqlFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/sqlFileDestination.js"],"names":[],"mappings":"AAWA;;;;;;;;;GASG;AACH;IAP2B,QAAQ;IACN,IAAI,EAAtB,QAAQ;IACQ,KAAK;IACN,OAAO;IACL,aAAa;aA2DxC;uBAzES,MAAM"}
package/dist/cjs/types/lib/index.d.ts CHANGED
@@ -3,6 +3,7 @@ export type Etl = any;
3
3
  * @typedef {Object} Etl
4
4
  * @function loader
5
5
  * @function pump
6
+ * @function connection
6
7
  * @function validator
7
8
  * @function destination
8
9
  * @function transform
@@ -13,13 +14,17 @@ export type Etl = any;
13
14
  */
14
15
  export function Etl(): void;
15
16
  export class Etl {
17
+ beforeETLList: any[];
18
+ connectionList: any[];
16
19
  validatorList: any[];
17
20
  transformationList: any[];
18
21
  destinationList: any[];
19
22
  loader: (loader: any) => Etl;
20
23
  pump: () => Etl;
24
+ beforeETL: (pipelineTask: any) => Etl;
25
+ connection: (connection: any) => Etl;
21
26
  validator: (validator: any) => Etl;
22
- destination: (destination: any) => Etl;
23
- transform: (transform: any) => Etl;
27
+ destination: (destination: any, ...tasks: any[]) => Etl;
28
+ transform: (transform: any, ...tasks: any[]) => Etl;
24
29
  }
25
30
  //# sourceMappingURL=index.d.ts.map
package/dist/cjs/types/lib/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAMA;;;;;;;GAOG;AAEH;;;GAGG;AACH,4BA2CC;;IAxCG,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,6BAGC;IAED,gBAaC;IAED,mCAGC;IAED,uCAGC;IAED,mCAGC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAMA;;;;;;;;GAQG;AAEH;;;GAGG;AACH,4BA8EC;;IA3EG,qBAAuB;IACvB,sBAAwB;IACxB,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,6BAGC;IAED,gBAgBC;IAED,sCASC;IAED,qCAGC;IAED,mCAGC;IAED,wDAWC;IAED,oDASC"}
package/dist/esm/destinations/consoleDestination.js CHANGED
@@ -9,14 +9,28 @@ const { Writable } = require("node:stream");
9
9
  */
10
10
  function ConsoleDestination(options) {
11
11
  const includeErrors = options.includeErrors;
12
- return new Writable({
12
+ const writable = new Writable({
13
13
  objectMode: true,
14
14
  write(chunk, _, callback) {
15
15
  if (chunk.errors.length === 0 || includeErrors)
16
16
  console.log(chunk);
17
+ // @ts-ignore
18
+ this.tasks && this.tasks.forEach(task => task.write(chunk));
17
19
  callback();
18
20
  }
19
21
  });
22
+ Object.assign(writable, {
23
+ setConnection: function (connection) {
24
+ this.connection = connection;
25
+ }.bind(writable),
26
+ getConnectionName: function () {
27
+ return this.connection && this.connection.name;
28
+ }.bind(writable),
29
+ setTasks: function (tasks) {
30
+ this.tasks = tasks;
31
+ }.bind(writable)
32
+ });
33
+ return writable;
20
34
  }
21
35
  module.exports = {
22
36
  ConsoleDestination
package/dist/esm/destinations/csvFileDestination.js CHANGED
@@ -25,7 +25,7 @@ function CSVFileDestination(options) {
25
25
  fileHandle = fd;
26
26
  }));
27
27
  let headersWritten = false;
28
- return new Writable({
28
+ const writable = new Writable({
29
29
  objectMode: true,
30
30
  write(chunk, _, callback) {
31
31
  if (!headersWritten && headers) {
@@ -45,6 +45,8 @@ function CSVFileDestination(options) {
45
45
  fs.writeFileSync(fileHandle, `${chunk.map(c => `"${c}"`).join(",")}\n`);
46
46
  }
47
47
  }
48
+ // @ts-ignore
49
+ this.tasks && this.tasks.forEach(task => task.write(chunk));
48
50
  lastChunk = chunk;
49
51
  callback();
50
52
  },
@@ -53,6 +55,18 @@ function CSVFileDestination(options) {
53
55
  callback();
54
56
  }
55
57
  });
58
+ Object.assign(writable, {
59
+ setConnection: function (connection) {
60
+ this.connection = connection;
61
+ }.bind(writable),
62
+ getConnectionName: function () {
63
+ return this.connection && this.connection.name;
64
+ }.bind(writable),
65
+ setTasks: function (tasks) {
66
+ this.tasks = tasks;
67
+ }.bind(writable)
68
+ });
69
+ return writable;
56
70
  }
57
71
  util.inherits(CSVFileDestination, EventEmitter);
58
72
  module.exports = {
package/dist/esm/destinations/postgresDestination.js CHANGED
@@ -46,11 +46,7 @@ function writeInsertStatement(columnMapping, table, chunk) {
46
46
  *
47
47
  * @param {Object} options
48
48
  * @param {Object} options.table
49
- * @param {Object} options.username
50
- * @param {Object} options.password
51
- * @param {Object} options.database
52
- * @param {Object} options.host
53
- * @param {Object} options.port
49
+ * @param {Object} options.connectionname
54
50
  * @param {Object} options.mapping
55
51
  * @param {Object} options.includeErrors
56
52
  * @returns Transform
@@ -58,35 +54,16 @@ function writeInsertStatement(columnMapping, table, chunk) {
58
54
  function PostgresDestination(options) {
59
55
  EventEmitter.call(this);
60
56
  const table = options.table;
61
- const username = options.username;
62
- const password = options.password;
63
- const database = options.database;
64
- const host = options.host;
65
- const port = options.port || DEFAULT_PORT;
57
+ const connectionname = options.connectionname;
66
58
  const mapping = options.mapping;
67
59
  const includeErrors = options.includeErrors;
68
60
  let lastChunk;
69
- const sequelize = new Sequelize(database, username, password, {
70
- host: host,
71
- port: port,
72
- dialect: 'postgres',
73
- logging: false
74
- });
75
- try {
76
- sequelize.authenticate();
77
- debug('sequelize.authenticate succeeded');
78
- }
79
- catch (e) {
80
- debug('sequelize.authenticate failed');
81
- debug(e);
82
- throw e;
83
- }
84
- return new Transform({
61
+ const transform = new Transform({
85
62
  objectMode: true,
86
63
  emitClose: true,
87
64
  construct(callback) {
88
65
  // @ts-ignore
89
- this.sequelize = sequelize;
66
+ this.connectionname = connectionname;
90
67
  callback();
91
68
  },
92
69
  write(chunk, _, callback) {
@@ -96,12 +73,14 @@ function PostgresDestination(options) {
96
73
  insertStatement = writeInsertStatement(mapping, table, chunk);
97
74
  debug('Insert statement: [%s]', insertStatement);
98
75
  // @ts-ignore
99
- this.sequelize.query(insertStatement)
76
+ this.connection.db.query(insertStatement)
100
77
  .then(result => {
101
78
  debug('result %o', result);
102
79
  chunk._result = result;
103
80
  lastChunk = chunk;
104
81
  // @ts-ignore
82
+ this.tasks && this.tasks.forEach(task => task.write(chunk));
83
+ // @ts-ignore
105
84
  callback(null, chunk);
106
85
  }).catch(error => {
107
86
  debug('error %o', error);
@@ -120,6 +99,18 @@ function PostgresDestination(options) {
120
99
  callback();
121
100
  }
122
101
  });
102
+ Object.assign(Transform.prototype, {
103
+ setConnection: function (connection) {
104
+ this.connection = connection;
105
+ }.bind(transform),
106
+ getConnectionName: function () {
107
+ return this.connection && this.connection.name;
108
+ }.bind(transform),
109
+ setTasks: function (tasks) {
110
+ this.tasks = tasks;
111
+ }.bind(transform)
112
+ });
113
+ return transform;
123
114
  }
124
115
  module.exports = {
125
116
  PostgresDestination,
package/dist/esm/destinations/sqlFileDestination.js CHANGED
@@ -40,7 +40,7 @@ function SQLFileDestination(options) {
40
40
  }
41
41
  function writeUpdateStatement(chunk) {
42
42
  }
43
- return new Writable({
43
+ const writable = new Writable({
44
44
  objectMode: true,
45
45
  write(chunk, _, callback) {
46
46
  if (chunk.errors.length === 0)
@@ -59,6 +59,18 @@ function SQLFileDestination(options) {
59
59
  callback();
60
60
  }
61
61
  });
62
+ Object.assign(writable, {
63
+ setConnection: function (connection) {
64
+ this.connection = connection;
65
+ }.bind(writable),
66
+ getConnectionName: function () {
67
+ return this.connection && this.connection.name;
68
+ }.bind(writable),
69
+ setTasks: function (tasks) {
70
+ this.tasks = tasks;
71
+ }.bind(writable)
72
+ });
73
+ return writable;
62
74
  }
63
75
  module.exports = {
64
76
  SQLFileDestination,
package/dist/esm/lib/index.js CHANGED
@@ -9,6 +9,7 @@ const { compose } = require("node:stream");
9
9
  * @typedef {Object} Etl
10
10
  * @function loader
11
11
  * @function pump
12
+ * @function connection
12
13
  * @function validator
13
14
  * @function destination
14
15
  * @function transform
@@ -20,6 +21,8 @@ const { compose } = require("node:stream");
20
21
  function Etl() {
21
22
  EventEmitter.call(this);
22
23
  let self = this;
24
+ self.beforeETLList = [];
25
+ self.connectionList = [];
23
26
  self.validatorList = [];
24
27
  self.transformationList = [];
25
28
  self.destinationList = [];
@@ -28,6 +31,9 @@ function Etl() {
28
31
  return self;
29
32
  };
30
33
  this.pump = () => {
34
+ this.beforeETLList.forEach(task => {
35
+ task.write({});
36
+ });
31
37
  this.loader
32
38
  .pump(this.loader)
33
39
  .pipe(compose(RowMetaData(), ...self.validatorList, ...self.transformationList, ...self.destinationList.map(dl => dl.on('result', (data) => self.emit('result', data))))
@@ -35,15 +41,41 @@ function Etl() {
35
41
  ).on('finish', (data) => self.emit('finish', data));
36
42
  return self;
37
43
  };
44
+ this.beforeETL = (pipelineTask) => {
45
+ const connectionname = pipelineTask.getConnectionName();
46
+ const connection = this.connectionList.filter(c => c.name === connectionname);
47
+ if (!connection) {
48
+ throw new Error(`Connection with name ${connectionname} not found`);
49
+ }
50
+ pipelineTask.setConnection(connection);
51
+ self.beforeETLList.push(pipelineTask);
52
+ return self;
53
+ };
54
+ this.connection = (connection) => {
55
+ self.connectionList.push(connection);
56
+ return self;
57
+ };
38
58
  this.validator = (validator) => {
39
59
  self.validatorList.push(validator);
40
60
  return self;
41
61
  };
42
- this.destination = (destination) => {
62
+ this.destination = (destination, ...tasks) => {
63
+ const connectionname = destination.getConnectionName();
64
+ const connection = this.connectionList.filter(c => c.name === connectionname)[0];
65
+ if (connection) {
66
+ destination.setConnection(connection);
67
+ }
68
+ tasks && destination.setTasks(tasks);
43
69
  self.destinationList.push(destination);
44
70
  return self;
45
71
  };
46
- this.transform = (transform) => {
72
+ this.transform = (transform, ...tasks) => {
73
+ /* The utility of being able to perform additional tasks for each transformation remains to be seen */
74
+ tasks.forEach(task => {
75
+ const connection = this.connectionList.filter(c => c.name === connectionname)[0];
76
+ task.setConnection(connection);
77
+ });
78
+ transform.setTasks(tasks);
47
79
  self.transformationList.push(transform);
48
80
  return self;
49
81
  };
package/dist/esm/types/destinations/consoleDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"consoleDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/consoleDestination.js"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH;IAH2B,aAAa;aAavC"}
1
+ {"version":3,"file":"consoleDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/consoleDestination.js"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH;IAH2B,aAAa;aA6BvC"}
package/dist/esm/types/destinations/csvFileDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"csvFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/csvFileDestination.js"],"names":[],"mappings":"AAKA;;;;;;;;GAQG;AACH;IAN2B,QAAQ;IACP,OAAO;IACP,aAAa;IACb,cAAc;aAyCzC"}
1
+ {"version":3,"file":"csvFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/csvFileDestination.js"],"names":[],"mappings":"AAKA;;;;;;;;GAQG;AACH;IAN2B,QAAQ;IACP,OAAO;IACP,aAAa;IACb,cAAc;aAyDzC"}
package/dist/esm/types/destinations/postgresDestination.d.ts CHANGED
@@ -2,22 +2,14 @@
2
2
  *
3
3
  * @param {Object} options
4
4
  * @param {Object} options.table
5
- * @param {Object} options.username
6
- * @param {Object} options.password
7
- * @param {Object} options.database
8
- * @param {Object} options.host
9
- * @param {Object} options.port
5
+ * @param {Object} options.connectionname
10
6
  * @param {Object} options.mapping
11
7
  * @param {Object} options.includeErrors
12
8
  * @returns Transform
13
9
  */
14
10
  export function PostgresDestination(options: {
15
11
  table: any;
16
- username: any;
17
- password: any;
18
- database: any;
19
- host: any;
20
- port: any;
12
+ connectionname: any;
21
13
  mapping: any;
22
14
  includeErrors: any;
23
15
  }): Transform;
package/dist/esm/types/destinations/postgresDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AA6CA;;;;;;;;;;;;GAYG;AACH;IAV2B,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,aAAa;cAqEvC;AA7GD,yFA4BC;AArCD,gDAEC;AAED,oEAGC"}
1
+ {"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AA6CA;;;;;;;;GAQG;AACH;IAN2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;cA+DvC;AAnGD,yFA4BC;AArCD,gDAEC;AAED,oEAGC"}
package/dist/esm/types/destinations/sqlFileDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"sqlFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/sqlFileDestination.js"],"names":[],"mappings":"AAWA;;;;;;;;;GASG;AACH;IAP2B,QAAQ;IACN,IAAI,EAAtB,QAAQ;IACQ,KAAK;IACN,OAAO;IACL,aAAa;aA6CxC;uBA3DS,MAAM"}
1
+ {"version":3,"file":"sqlFileDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/sqlFileDestination.js"],"names":[],"mappings":"AAWA;;;;;;;;;GASG;AACH;IAP2B,QAAQ;IACN,IAAI,EAAtB,QAAQ;IACQ,KAAK;IACN,OAAO;IACL,aAAa;aA2DxC;uBAzES,MAAM"}
package/dist/esm/types/lib/index.d.ts CHANGED
@@ -3,6 +3,7 @@ export type Etl = any;
3
3
  * @typedef {Object} Etl
4
4
  * @function loader
5
5
  * @function pump
6
+ * @function connection
6
7
  * @function validator
7
8
  * @function destination
8
9
  * @function transform
@@ -13,13 +14,17 @@ export type Etl = any;
13
14
  */
14
15
  export function Etl(): void;
15
16
  export class Etl {
17
+ beforeETLList: any[];
18
+ connectionList: any[];
16
19
  validatorList: any[];
17
20
  transformationList: any[];
18
21
  destinationList: any[];
19
22
  loader: (loader: any) => Etl;
20
23
  pump: () => Etl;
24
+ beforeETL: (pipelineTask: any) => Etl;
25
+ connection: (connection: any) => Etl;
21
26
  validator: (validator: any) => Etl;
22
- destination: (destination: any) => Etl;
23
- transform: (transform: any) => Etl;
27
+ destination: (destination: any, ...tasks: any[]) => Etl;
28
+ transform: (transform: any, ...tasks: any[]) => Etl;
24
29
  }
25
30
  //# sourceMappingURL=index.d.ts.map
package/dist/esm/types/lib/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAMA;;;;;;;GAOG;AAEH;;;GAGG;AACH,4BA2CC;;IAxCG,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,6BAGC;IAED,gBAaC;IAED,mCAGC;IAED,uCAGC;IAED,mCAGC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAMA;;;;;;;;GAQG;AAEH;;;GAGG;AACH,4BA8EC;;IA3EG,qBAAuB;IACvB,sBAAwB;IACxB,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,6BAGC;IAED,gBAgBC;IAED,sCASC;IAED,qCAGC;IAED,mCAGC;IAED,wDAWC;IAED,oDASC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ffc-pay-etl-framework",
3
- "version": "0.1.14",
3
+ "version": "1.1.14-alpha.4",
4
4
  "publisher": "Defra",
5
5
  "main": "dist/cjs/index.js",
6
6
  "private": false,