ffc-pay-etl-framework 1.1.12-alpha.2 → 1.1.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/dist/cjs/destinations/postgresDestination.js +9 -4
  2. package/dist/cjs/types/destinations/postgresDestination.d.ts +3 -1
  3. package/dist/cjs/types/destinations/postgresDestination.d.ts.map +1 -1
  4. package/dist/esm/destinations/postgresDestination.js +8 -4
  5. package/dist/esm/types/destinations/postgresDestination.d.ts +3 -1
  6. package/dist/esm/types/destinations/postgresDestination.d.ts.map +1 -1
  7. package/package.json +2 -2
package/dist/cjs/destinations/postgresDestination.js CHANGED
@@ -18,12 +18,14 @@ function hasReturningColumns(mapping) {
18
18
  function getReturningColumns(mapping) {
19
19
  return mapping.filter(m => m.returning === true).map(m => m === null || m === void 0 ? void 0 : m.targetColumn);
20
20
  }
21
- function writeInsertStatement(columnMapping, table, chunk, schema) {
22
- let statement = `INSERT INTO ${schema !== null && schema !== void 0 ? schema : 'public'}."${table}" (${chunk._columns.map(column => {
21
+ function writeInsertStatement(columnMapping, table, chunk, schema, ignoredColumns = []) {
22
+ const filteredColumns = chunk._columns.filter(column => !ignoredColumns.includes(column));
23
+ let statement = `INSERT INTO ${schema !== null && schema !== void 0 ? schema : 'public'}."${table}" (${filteredColumns.map(column => {
23
24
  const mapping = getMappingForColumn(columnMapping, column);
24
25
  return (mapping === null || mapping === void 0 ? void 0 : mapping.targetColumn) ? `"${mapping.targetColumn}"` : `"${mapping === null || mapping === void 0 ? void 0 : mapping.column}"`;
25
26
  })
26
- .join(",")}) VALUES (${chunk._columns.map((column, index) => {
27
+ .join(",")}) VALUES (${filteredColumns.map(column => {
28
+ const index = chunk._columns.indexOf(column);
27
29
  const mapping = getMappingForColumn(columnMapping, column);
28
30
  if ((mapping === null || mapping === void 0 ? void 0 : mapping.targetType) === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
29
31
  debug('Source data is not a number');
@@ -53,15 +55,18 @@ function writeInsertStatement(columnMapping, table, chunk, schema) {
53
55
  * @param {Object} options.mapping
54
56
  * @param {Object} options.includeErrors
55
57
  * @param {String} [options.schema]
58
+ * @param {Array<String>} [options.ignoredColumns]
56
59
  * @returns Transform
57
60
  */
58
61
  function PostgresDestination(options) {
62
+ var _a;
59
63
  EventEmitter.call(this);
60
64
  const table = options.table;
61
65
  const connectionname = options.connectionname;
62
66
  const mapping = options.mapping;
63
67
  const includeErrors = options.includeErrors;
64
68
  const schema = options.schema;
69
+ const ignoredColumns = (_a = options.ignoredColumns) !== null && _a !== void 0 ? _a : [];
65
70
  let lastChunk;
66
71
  const transform = new Transform({
67
72
  objectMode: true,
@@ -75,7 +80,7 @@ function PostgresDestination(options) {
75
80
  let insertStatement;
76
81
  // @ts-ignore
77
82
  if (chunk.errors.length === 0 || options.includeErrors) {
78
- insertStatement = writeInsertStatement(mapping, table, chunk, schema);
83
+ insertStatement = writeInsertStatement(mapping, table, chunk, schema, ignoredColumns);
79
84
  debug('Insert statement: [%s]', insertStatement);
80
85
  // @ts-ignore
81
86
  this.connection.db.query(insertStatement)
package/dist/cjs/types/destinations/postgresDestination.d.ts CHANGED
@@ -6,6 +6,7 @@
6
6
  * @param {Object} options.mapping
7
7
  * @param {Object} options.includeErrors
8
8
  * @param {String} [options.schema]
9
+ * @param {Array<String>} [options.ignoredColumns]
9
10
  * @returns Transform
10
11
  */
11
12
  export function PostgresDestination(options: {
@@ -14,8 +15,9 @@ export function PostgresDestination(options: {
14
15
  mapping: any;
15
16
  includeErrors: any;
16
17
  schema?: string;
18
+ ignoredColumns?: Array<string>;
17
19
  }): Transform;
18
- export function writeInsertStatement(columnMapping: any, table: any, chunk: any, schema: any): string;
20
+ export function writeInsertStatement(columnMapping: any, table: any, chunk: any, schema: any, ignoredColumns?: any[]): string;
19
21
  export function isKeyWord(column: any): boolean;
20
22
  export function getMappingForColumn(mapping: any, column: any): any;
21
23
  export function hasReturningColumns(mapping: any): boolean;
package/dist/cjs/types/destinations/postgresDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAmDA;;;;;;;;;GASG;AACH;IAP2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;IACZ,MAAM;cAiEjC;AApGD,sGA0BC;AA5CD,gDAEC;AAED,oEAIC;AAED,2DAEC;AAED,uDAEC"}
1
+ {"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAqDA;;;;;;;;;;GAUG;AACH;IAR2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;IACZ,MAAM;IACC,cAAc,GAAtC,aAAa;cAkEvB;AAxGD,8HA4BC;AA9CD,gDAEC;AAED,oEAIC;AAED,2DAEC;AAED,uDAEC"}
package/dist/esm/destinations/postgresDestination.js CHANGED
@@ -20,12 +20,14 @@ function hasReturningColumns(mapping) {
20
20
  function getReturningColumns(mapping) {
21
21
  return mapping.filter(m => m.returning === true).map(m => m?.targetColumn);
22
22
  }
23
- function writeInsertStatement(columnMapping, table, chunk, schema) {
24
- let statement = `INSERT INTO ${schema ?? 'public'}."${table}" (${chunk._columns.map(column => {
23
+ function writeInsertStatement(columnMapping, table, chunk, schema, ignoredColumns = []) {
24
+ const filteredColumns = chunk._columns.filter(column => !ignoredColumns.includes(column));
25
+ let statement = `INSERT INTO ${schema ?? 'public'}."${table}" (${filteredColumns.map(column => {
25
26
  const mapping = getMappingForColumn(columnMapping, column);
26
27
  return mapping?.targetColumn ? `"${mapping.targetColumn}"` : `"${mapping?.column}"`;
27
28
  })
28
- .join(",")}) VALUES (${chunk._columns.map((column, index) => {
29
+ .join(",")}) VALUES (${filteredColumns.map(column => {
30
+ const index = chunk._columns.indexOf(column);
29
31
  const mapping = getMappingForColumn(columnMapping, column);
30
32
  if (mapping?.targetType === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
31
33
  debug('Source data is not a number');
@@ -55,6 +57,7 @@ function writeInsertStatement(columnMapping, table, chunk, schema) {
55
57
  * @param {Object} options.mapping
56
58
  * @param {Object} options.includeErrors
57
59
  * @param {String} [options.schema]
60
+ * @param {Array<String>} [options.ignoredColumns]
58
61
  * @returns Transform
59
62
  */
60
63
  function PostgresDestination(options) {
@@ -64,6 +67,7 @@ function PostgresDestination(options) {
64
67
  const mapping = options.mapping;
65
68
  const includeErrors = options.includeErrors;
66
69
  const schema = options.schema;
70
+ const ignoredColumns = options.ignoredColumns ?? [];
67
71
  let lastChunk;
68
72
  const transform = new Transform({
69
73
  objectMode: true,
@@ -77,7 +81,7 @@ function PostgresDestination(options) {
77
81
  let insertStatement;
78
82
  // @ts-ignore
79
83
  if (chunk.errors.length === 0 || options.includeErrors) {
80
- insertStatement = writeInsertStatement(mapping, table, chunk, schema);
84
+ insertStatement = writeInsertStatement(mapping, table, chunk, schema, ignoredColumns);
81
85
  debug('Insert statement: [%s]', insertStatement);
82
86
  // @ts-ignore
83
87
  this.connection.db.query(insertStatement)
package/dist/esm/types/destinations/postgresDestination.d.ts CHANGED
@@ -6,6 +6,7 @@
6
6
  * @param {Object} options.mapping
7
7
  * @param {Object} options.includeErrors
8
8
  * @param {String} [options.schema]
9
+ * @param {Array<String>} [options.ignoredColumns]
9
10
  * @returns Transform
10
11
  */
11
12
  export function PostgresDestination(options: {
@@ -14,8 +15,9 @@ export function PostgresDestination(options: {
14
15
  mapping: any;
15
16
  includeErrors: any;
16
17
  schema?: string;
18
+ ignoredColumns?: Array<string>;
17
19
  }): Transform;
18
- export function writeInsertStatement(columnMapping: any, table: any, chunk: any, schema: any): string;
20
+ export function writeInsertStatement(columnMapping: any, table: any, chunk: any, schema: any, ignoredColumns?: any[]): string;
19
21
  export function isKeyWord(column: any): boolean;
20
22
  export function getMappingForColumn(mapping: any, column: any): any;
21
23
  export function hasReturningColumns(mapping: any): boolean;
package/dist/esm/types/destinations/postgresDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAmDA;;;;;;;;;GASG;AACH;IAP2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;IACZ,MAAM;cAiEjC;AApGD,sGA0BC;AA5CD,gDAEC;AAED,oEAIC;AAED,2DAEC;AAED,uDAEC"}
1
+ {"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAqDA;;;;;;;;;;GAUG;AACH;IAR2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;IACZ,MAAM;IACC,cAAc,GAAtC,aAAa;cAkEvB;AAxGD,8HA4BC;AA9CD,gDAEC;AAED,oEAIC;AAED,2DAEC;AAED,uDAEC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ffc-pay-etl-framework",
3
- "version": "1.1.12-alpha.2",
3
+ "version": "1.1.12",
4
4
  "publisher": "Defra",
5
5
  "main": "dist/cjs/index.js",
6
6
  "private": false,
@@ -75,7 +75,7 @@
75
75
  "@faker-js/faker": "^8.4.1",
76
76
  "csv": "^6.3.9",
77
77
  "mariadb": "^3.3.1",
78
- "mysql2": "^3.13.0",
78
+ "mysql2": "^3.10.3",
79
79
  "oracledb": "^6.5.1",
80
80
  "pg": "^8.12.0",
81
81
  "pg-hstore": "^2.3.4",