ffc-pay-etl-framework 1.0.0 → 1.1.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (13) hide show
  1. package/dist/cjs/destinations/postgresDestination.js +15 -3
  2. package/dist/cjs/lib/index.js +9 -2
  3. package/dist/cjs/types/destinations/postgresDestination.d.ts +2 -0
  4. package/dist/cjs/types/destinations/postgresDestination.d.ts.map +1 -1
  5. package/dist/cjs/types/lib/index.d.ts +1 -0
  6. package/dist/cjs/types/lib/index.d.ts.map +1 -1
  7. package/dist/esm/destinations/postgresDestination.js +15 -3
  8. package/dist/esm/lib/index.js +9 -2
  9. package/dist/esm/types/destinations/postgresDestination.d.ts +2 -0
  10. package/dist/esm/types/destinations/postgresDestination.d.ts.map +1 -1
  11. package/dist/esm/types/lib/index.d.ts +1 -0
  12. package/dist/esm/types/lib/index.d.ts.map +1 -1
  13. package/package.json +2 -2
package/dist/cjs/destinations/postgresDestination.js CHANGED
@@ -10,6 +10,12 @@ function getMappingForColumn(mapping, column) {
10
10
  const [map] = mapping.filter(m => m.column === column);
11
11
  return map;
12
12
  }
13
+ function hasReturningColumns(mapping) {
14
+ return mapping.filter(m => m.returning === true).length > 0;
15
+ }
16
+ function getReturningColumns(mapping) {
17
+ return mapping.filter(m => m.returning === true).map(m => m.targetColumn);
18
+ }
13
19
  function writeInsertStatement(columnMapping, table, chunk) {
14
20
  let statement = `INSERT INTO ${table} (${chunk._columns.map(column => {
15
21
  const mapping = getMappingForColumn(columnMapping, column);
@@ -24,8 +30,9 @@ function writeInsertStatement(columnMapping, table, chunk) {
24
30
  })
25
31
  .join(",")}) VALUES (${chunk._columns.map((column, index) => {
26
32
  const mapping = getMappingForColumn(columnMapping, column);
27
- if (!mapping)
33
+ if (!mapping) {
28
34
  debug('Mapping not found for column %s', column);
35
+ }
29
36
  if (mapping.targetType === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
30
37
  debug('Source data is not a number');
31
38
  return 0;
@@ -38,6 +45,9 @@ function writeInsertStatement(columnMapping, table, chunk) {
38
45
  }
39
46
  return chunk[index] ? chunk[index] : 'null';
40
47
  })})`;
48
+ if (hasReturningColumns(columnMapping)) {
49
+ statement = statement + ` RETURNING ${getReturningColumns(columnMapping).join(',')}`;
50
+ }
41
51
  return statement;
42
52
  }
43
53
  /**
@@ -67,7 +77,7 @@ function PostgresDestination(options) {
67
77
  write(chunk, _, callback) {
68
78
  let insertStatement;
69
79
  // @ts-ignore
70
- if (chunk.errors.length === 0 | options.includeErrors) {
80
+ if (chunk.errors.length === 0 || options.includeErrors) {
71
81
  insertStatement = writeInsertStatement(mapping, table, chunk);
72
82
  debug('Insert statement: [%s]', insertStatement);
73
83
  // @ts-ignore
@@ -116,5 +126,7 @@ module.exports = {
116
126
  PostgresDestination,
117
127
  writeInsertStatement,
118
128
  isKeyWord,
119
- getMappingForColumn
129
+ getMappingForColumn,
130
+ hasReturningColumns,
131
+ getReturningColumns
120
132
  };
package/dist/cjs/lib/index.js CHANGED
@@ -18,7 +18,8 @@ const { compose } = require("node:stream");
18
18
  */
19
19
  function Etl() {
20
20
  EventEmitter.call(this);
21
- let self = this;
21
+ const self = this;
22
+ self.store = [];
22
23
  self.beforeETLList = [];
23
24
  self.connectionList = [];
24
25
  self.validatorList = [];
@@ -46,6 +47,7 @@ function Etl() {
46
47
  throw new Error(`Connection with name ${connectionname} not found`);
47
48
  }
48
49
  pipelineTask.setConnection(connection);
50
+ pipelineTask.setETL(self);
49
51
  self.beforeETLList.push(pipelineTask);
50
52
  return self;
51
53
  };
@@ -63,7 +65,12 @@ function Etl() {
63
65
  if (connection) {
64
66
  destination.setConnection(connection);
65
67
  }
66
- tasks && destination.setTasks(tasks);
68
+ if (tasks) {
69
+ for (const task of tasks) {
70
+ task.setETL(self);
71
+ }
72
+ destination.setTasks(tasks);
73
+ }
67
74
  self.destinationList.push(destination);
68
75
  return self;
69
76
  };
package/dist/cjs/types/destinations/postgresDestination.d.ts CHANGED
@@ -16,5 +16,7 @@ export function PostgresDestination(options: {
16
16
  export function writeInsertStatement(columnMapping: any, table: any, chunk: any): string;
17
17
  export function isKeyWord(column: any): boolean;
18
18
  export function getMappingForColumn(mapping: any, column: any): any;
19
+ export function hasReturningColumns(mapping: any): boolean;
20
+ export function getReturningColumns(mapping: any): any;
19
21
  import { Transform } from "stream";
20
22
  //# sourceMappingURL=postgresDestination.d.ts.map
package/dist/cjs/types/destinations/postgresDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AA6CA;;;;;;;;GAQG;AACH;IAN2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;cA+DvC;AAnGD,yFA4BC;AArCD,gDAEC;AAED,oEAGC"}
1
+ {"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AA0DA;;;;;;;;GAQG;AACH;IAN2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;cA+DvC;AAxGD,yFAiCC;AAlDD,gDAEC;AAED,oEAGC;AAED,2DAEC;AAED,uDAEC"}
package/dist/cjs/types/lib/index.d.ts CHANGED
@@ -14,6 +14,7 @@ export type Etl = any;
14
14
  */
15
15
  export function Etl(): void;
16
16
  export class Etl {
17
+ store: any[];
17
18
  beforeETLList: any[];
18
19
  connectionList: any[];
19
20
  validatorList: any[];
package/dist/cjs/types/lib/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAMA;;;;;;;;GAQG;AAEH;;;GAGG;AACH,4BAwEC;;IArEG,qBAAuB;IACvB,sBAAwB;IACxB,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,6BAGC;IAED,gBAgBC;IAED,sCASC;IAED,qCAGC;IAED,mCAGC;IAED,wDAWC;IAED,mCAGC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAMA;;;;;;;;GAQG;AAEH;;;GAGG;AACH,4BA+EC;;IA5EG,aAAe;IACf,qBAAuB;IACvB,sBAAwB;IACxB,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,6BAGC;IAED,gBAgBC;IAED,sCAUC;IAED,qCAGC;IAED,mCAGC;IAED,wDAgBC;IAED,mCAGC"}
package/dist/esm/destinations/postgresDestination.js CHANGED
@@ -12,6 +12,12 @@ function getMappingForColumn(mapping, column) {
12
12
  const [map] = mapping.filter(m => m.column === column);
13
13
  return map;
14
14
  }
15
+ function hasReturningColumns(mapping) {
16
+ return mapping.filter(m => m.returning === true).length > 0;
17
+ }
18
+ function getReturningColumns(mapping) {
19
+ return mapping.filter(m => m.returning === true).map(m => m.targetColumn);
20
+ }
15
21
  function writeInsertStatement(columnMapping, table, chunk) {
16
22
  let statement = `INSERT INTO ${table} (${chunk._columns.map(column => {
17
23
  const mapping = getMappingForColumn(columnMapping, column);
@@ -26,8 +32,9 @@ function writeInsertStatement(columnMapping, table, chunk) {
26
32
  })
27
33
  .join(",")}) VALUES (${chunk._columns.map((column, index) => {
28
34
  const mapping = getMappingForColumn(columnMapping, column);
29
- if (!mapping)
35
+ if (!mapping) {
30
36
  debug('Mapping not found for column %s', column);
37
+ }
31
38
  if (mapping.targetType === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
32
39
  debug('Source data is not a number');
33
40
  return 0;
@@ -40,6 +47,9 @@ function writeInsertStatement(columnMapping, table, chunk) {
40
47
  }
41
48
  return chunk[index] ? chunk[index] : 'null';
42
49
  })})`;
50
+ if (hasReturningColumns(columnMapping)) {
51
+ statement = statement + ` RETURNING ${getReturningColumns(columnMapping).join(',')}`;
52
+ }
43
53
  return statement;
44
54
  }
45
55
  /**
@@ -69,7 +79,7 @@ function PostgresDestination(options) {
69
79
  write(chunk, _, callback) {
70
80
  let insertStatement;
71
81
  // @ts-ignore
72
- if (chunk.errors.length === 0 | options.includeErrors) {
82
+ if (chunk.errors.length === 0 || options.includeErrors) {
73
83
  insertStatement = writeInsertStatement(mapping, table, chunk);
74
84
  debug('Insert statement: [%s]', insertStatement);
75
85
  // @ts-ignore
@@ -116,5 +126,7 @@ module.exports = {
116
126
  PostgresDestination,
117
127
  writeInsertStatement,
118
128
  isKeyWord,
119
- getMappingForColumn
129
+ getMappingForColumn,
130
+ hasReturningColumns,
131
+ getReturningColumns
120
132
  };
package/dist/esm/lib/index.js CHANGED
@@ -20,7 +20,8 @@ const { compose } = require("node:stream");
20
20
  */
21
21
  function Etl() {
22
22
  EventEmitter.call(this);
23
- let self = this;
23
+ const self = this;
24
+ self.store = [];
24
25
  self.beforeETLList = [];
25
26
  self.connectionList = [];
26
27
  self.validatorList = [];
@@ -48,6 +49,7 @@ function Etl() {
48
49
  throw new Error(`Connection with name ${connectionname} not found`);
49
50
  }
50
51
  pipelineTask.setConnection(connection);
52
+ pipelineTask.setETL(self);
51
53
  self.beforeETLList.push(pipelineTask);
52
54
  return self;
53
55
  };
@@ -65,7 +67,12 @@ function Etl() {
65
67
  if (connection) {
66
68
  destination.setConnection(connection);
67
69
  }
68
- tasks && destination.setTasks(tasks);
70
+ if (tasks) {
71
+ for (const task of tasks) {
72
+ task.setETL(self);
73
+ }
74
+ destination.setTasks(tasks);
75
+ }
69
76
  self.destinationList.push(destination);
70
77
  return self;
71
78
  };
package/dist/esm/types/destinations/postgresDestination.d.ts CHANGED
@@ -16,5 +16,7 @@ export function PostgresDestination(options: {
16
16
  export function writeInsertStatement(columnMapping: any, table: any, chunk: any): string;
17
17
  export function isKeyWord(column: any): boolean;
18
18
  export function getMappingForColumn(mapping: any, column: any): any;
19
+ export function hasReturningColumns(mapping: any): boolean;
20
+ export function getReturningColumns(mapping: any): any;
19
21
  import { Transform } from "stream";
20
22
  //# sourceMappingURL=postgresDestination.d.ts.map
package/dist/esm/types/destinations/postgresDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AA6CA;;;;;;;;GAQG;AACH;IAN2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;cA+DvC;AAnGD,yFA4BC;AArCD,gDAEC;AAED,oEAGC"}
1
+ {"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AA0DA;;;;;;;;GAQG;AACH;IAN2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;cA+DvC;AAxGD,yFAiCC;AAlDD,gDAEC;AAED,oEAGC;AAED,2DAEC;AAED,uDAEC"}
package/dist/esm/types/lib/index.d.ts CHANGED
@@ -14,6 +14,7 @@ export type Etl = any;
14
14
  */
15
15
  export function Etl(): void;
16
16
  export class Etl {
17
+ store: any[];
17
18
  beforeETLList: any[];
18
19
  connectionList: any[];
19
20
  validatorList: any[];
package/dist/esm/types/lib/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAMA;;;;;;;;GAQG;AAEH;;;GAGG;AACH,4BAwEC;;IArEG,qBAAuB;IACvB,sBAAwB;IACxB,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,6BAGC;IAED,gBAgBC;IAED,sCASC;IAED,qCAGC;IAED,mCAGC;IAED,wDAWC;IAED,mCAGC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/index.js"],"names":[],"mappings":";AAMA;;;;;;;;GAQG;AAEH;;;GAGG;AACH,4BA+EC;;IA5EG,aAAe;IACf,qBAAuB;IACvB,sBAAwB;IACxB,qBAAuB;IACvB,0BAA4B;IAC5B,uBAAyB;IAEzB,6BAGC;IAED,gBAgBC;IAED,sCAUC;IAED,qCAGC;IAED,mCAGC;IAED,wDAgBC;IAED,mCAGC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ffc-pay-etl-framework",
3
- "version": "1.0.0",
3
+ "version": "1.1.0-alpha.1",
4
4
  "publisher": "Defra",
5
5
  "main": "dist/cjs/index.js",
6
6
  "private": false,
@@ -13,7 +13,7 @@
13
13
  "build:esm": "tsc -p ./tsconfig.esm.json && mv dist/esm/index.js dist/esm/index.mjs",
14
14
  "build:cjs": "tsc -p ./tsconfig.cjs.json",
15
15
  "prepack": "npm run build",
16
- "example": "node ./src/examples/csv-multi-tool-validator.js"
16
+ "example": "node ./src/examples/sql-return-values.js"
17
17
  },
18
18
  "keywords": [
19
19
  "ETL"