ffc-pay-etl-framework 0.1.12 → 0.1.13-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/destinations/postgresDestination.js +42 -36
- package/dist/cjs/types/destinations/postgresDestination.d.ts +3 -0
- package/dist/cjs/types/destinations/postgresDestination.d.ts.map +1 -1
- package/dist/esm/destinations/postgresDestination.js +42 -36
- package/dist/esm/types/destinations/postgresDestination.d.ts +3 -0
- package/dist/esm/types/destinations/postgresDestination.d.ts.map +1 -1
- package/package.json +2 -1
|
@@ -1,9 +1,45 @@
|
|
|
1
1
|
const EventEmitter = require('node:events');
|
|
2
|
-
const util = require('node:util');
|
|
3
2
|
const { Transform } = require('node:stream');
|
|
4
3
|
const { Sequelize } = require('sequelize');
|
|
5
4
|
const debug = require('debug')('destination');
|
|
6
5
|
const DEFAULT_PORT = 5432;
|
|
6
|
+
function isKeyWord(column) {
|
|
7
|
+
return ['USER'].includes(column.toUpperCase());
|
|
8
|
+
}
|
|
9
|
+
function getMappingForColumn(mapping, column) {
|
|
10
|
+
const [map] = mapping.filter(m => m.column === column);
|
|
11
|
+
return map;
|
|
12
|
+
}
|
|
13
|
+
function writeInsertStatement(columnMapping, table, chunk) {
|
|
14
|
+
let statement = `INSERT INTO ${table} (${chunk._columns.map(column => {
|
|
15
|
+
const mapping = getMappingForColumn(columnMapping, column);
|
|
16
|
+
return mapping
|
|
17
|
+
?
|
|
18
|
+
isKeyWord(mapping.targetColumn)
|
|
19
|
+
? `"${mapping.targetColumn}"`
|
|
20
|
+
: mapping.targetColumn
|
|
21
|
+
: isKeyWord(column)
|
|
22
|
+
? `"${mapping.column}"`
|
|
23
|
+
: mapping.column;
|
|
24
|
+
})
|
|
25
|
+
.join(",")}) VALUES (${chunk._columns.map((column, index) => {
|
|
26
|
+
const mapping = getMappingForColumn(columnMapping, column);
|
|
27
|
+
if (!mapping)
|
|
28
|
+
debug('Mapping not found for column %s', column);
|
|
29
|
+
if (mapping.targetType === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
|
|
30
|
+
debug('Source data is not a number');
|
|
31
|
+
return 0;
|
|
32
|
+
}
|
|
33
|
+
if (mapping.targetType === "varchar" || mapping.targetType === "char") {
|
|
34
|
+
return `'${chunk[index]}'`;
|
|
35
|
+
}
|
|
36
|
+
if (mapping.targetType === "date") {
|
|
37
|
+
return `to_date('${chunk[index]}','${mapping.format}')`;
|
|
38
|
+
}
|
|
39
|
+
return chunk[index] ? chunk[index] : 'null';
|
|
40
|
+
})})`;
|
|
41
|
+
return statement;
|
|
42
|
+
}
|
|
7
43
|
/**
|
|
8
44
|
*
|
|
9
45
|
* @param {Object} options
|
|
@@ -43,39 +79,6 @@ function PostgresDestination(options) {
|
|
|
43
79
|
debug(e);
|
|
44
80
|
throw e;
|
|
45
81
|
}
|
|
46
|
-
function getMappingForColumn(column) {
|
|
47
|
-
const [map] = mapping.filter(m => m.column === column);
|
|
48
|
-
return map;
|
|
49
|
-
}
|
|
50
|
-
function isKeyWord(column) {
|
|
51
|
-
return ['USER'].includes(column.toUpperCase());
|
|
52
|
-
}
|
|
53
|
-
function writeInsertStatement(chunk) {
|
|
54
|
-
let statement = `INSERT INTO ${table} (${chunk._columns.map(column => {
|
|
55
|
-
let mapping = getMappingForColumn(column);
|
|
56
|
-
return mapping
|
|
57
|
-
?
|
|
58
|
-
isKeyWord(mapping.targetColumn)
|
|
59
|
-
? `"${mapping.targetColumn}"`
|
|
60
|
-
: mapping.targetColumn
|
|
61
|
-
: isKeyWord(column)
|
|
62
|
-
? `"${mapping.column}"`
|
|
63
|
-
: mapping.column;
|
|
64
|
-
})
|
|
65
|
-
.join(",")}) VALUES (${chunk._columns.map((column, index) => {
|
|
66
|
-
let mapping = getMappingForColumn(column);
|
|
67
|
-
if (!mapping)
|
|
68
|
-
debug('Mapping not found for column %s', column);
|
|
69
|
-
if (mapping.targetType === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
|
|
70
|
-
debug('Source data is not a number');
|
|
71
|
-
return 0;
|
|
72
|
-
}
|
|
73
|
-
if (mapping.targetType === "varchar" || mapping.targetType === "char")
|
|
74
|
-
return `'${chunk[index]}'`;
|
|
75
|
-
return chunk[index] ? chunk[index] : 'null';
|
|
76
|
-
})})`;
|
|
77
|
-
return statement;
|
|
78
|
-
}
|
|
79
82
|
return new Transform({
|
|
80
83
|
objectMode: true,
|
|
81
84
|
emitClose: true,
|
|
@@ -88,7 +91,7 @@ function PostgresDestination(options) {
|
|
|
88
91
|
let insertStatement;
|
|
89
92
|
// @ts-ignore
|
|
90
93
|
if (chunk.errors.length === 0 | options.includeErrors) {
|
|
91
|
-
insertStatement = writeInsertStatement(chunk);
|
|
94
|
+
insertStatement = writeInsertStatement(mapping, table, chunk);
|
|
92
95
|
debug('Insert statement: [%s]', insertStatement);
|
|
93
96
|
// @ts-ignore
|
|
94
97
|
this.sequelize.query(insertStatement)
|
|
@@ -117,5 +120,8 @@ function PostgresDestination(options) {
|
|
|
117
120
|
});
|
|
118
121
|
}
|
|
119
122
|
module.exports = {
|
|
120
|
-
PostgresDestination
|
|
123
|
+
PostgresDestination,
|
|
124
|
+
writeInsertStatement,
|
|
125
|
+
isKeyWord,
|
|
126
|
+
getMappingForColumn
|
|
121
127
|
};
|
|
@@ -21,5 +21,8 @@ export function PostgresDestination(options: {
|
|
|
21
21
|
mapping: any;
|
|
22
22
|
includeErrors: any;
|
|
23
23
|
}): Transform;
|
|
24
|
+
export function writeInsertStatement(columnMapping: any, table: any, chunk: any): string;
|
|
25
|
+
export function isKeyWord(column: any): boolean;
|
|
26
|
+
export function getMappingForColumn(mapping: any, column: any): any;
|
|
24
27
|
import { Transform } from "stream";
|
|
25
28
|
//# sourceMappingURL=postgresDestination.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AA6CA;;;;;;;;;;;;GAYG;AACH;IAV2B,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,aAAa;cAqEvC;AA7GD,yFA4BC;AArCD,gDAEC;AAED,oEAGC"}
|
|
@@ -1,11 +1,47 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
const EventEmitter = require('node:events');
|
|
4
|
-
const util = require('node:util');
|
|
5
4
|
const { Transform } = require('node:stream');
|
|
6
5
|
const { Sequelize } = require('sequelize');
|
|
7
6
|
const debug = require('debug')('destination');
|
|
8
7
|
const DEFAULT_PORT = 5432;
|
|
8
|
+
function isKeyWord(column) {
|
|
9
|
+
return ['USER'].includes(column.toUpperCase());
|
|
10
|
+
}
|
|
11
|
+
function getMappingForColumn(mapping, column) {
|
|
12
|
+
const [map] = mapping.filter(m => m.column === column);
|
|
13
|
+
return map;
|
|
14
|
+
}
|
|
15
|
+
function writeInsertStatement(columnMapping, table, chunk) {
|
|
16
|
+
let statement = `INSERT INTO ${table} (${chunk._columns.map(column => {
|
|
17
|
+
const mapping = getMappingForColumn(columnMapping, column);
|
|
18
|
+
return mapping
|
|
19
|
+
?
|
|
20
|
+
isKeyWord(mapping.targetColumn)
|
|
21
|
+
? `"${mapping.targetColumn}"`
|
|
22
|
+
: mapping.targetColumn
|
|
23
|
+
: isKeyWord(column)
|
|
24
|
+
? `"${mapping.column}"`
|
|
25
|
+
: mapping.column;
|
|
26
|
+
})
|
|
27
|
+
.join(",")}) VALUES (${chunk._columns.map((column, index) => {
|
|
28
|
+
const mapping = getMappingForColumn(columnMapping, column);
|
|
29
|
+
if (!mapping)
|
|
30
|
+
debug('Mapping not found for column %s', column);
|
|
31
|
+
if (mapping.targetType === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
|
|
32
|
+
debug('Source data is not a number');
|
|
33
|
+
return 0;
|
|
34
|
+
}
|
|
35
|
+
if (mapping.targetType === "varchar" || mapping.targetType === "char") {
|
|
36
|
+
return `'${chunk[index]}'`;
|
|
37
|
+
}
|
|
38
|
+
if (mapping.targetType === "date") {
|
|
39
|
+
return `to_date('${chunk[index]}','${mapping.format}')`;
|
|
40
|
+
}
|
|
41
|
+
return chunk[index] ? chunk[index] : 'null';
|
|
42
|
+
})})`;
|
|
43
|
+
return statement;
|
|
44
|
+
}
|
|
9
45
|
/**
|
|
10
46
|
*
|
|
11
47
|
* @param {Object} options
|
|
@@ -45,39 +81,6 @@ function PostgresDestination(options) {
|
|
|
45
81
|
debug(e);
|
|
46
82
|
throw e;
|
|
47
83
|
}
|
|
48
|
-
function getMappingForColumn(column) {
|
|
49
|
-
const [map] = mapping.filter(m => m.column === column);
|
|
50
|
-
return map;
|
|
51
|
-
}
|
|
52
|
-
function isKeyWord(column) {
|
|
53
|
-
return ['USER'].includes(column.toUpperCase());
|
|
54
|
-
}
|
|
55
|
-
function writeInsertStatement(chunk) {
|
|
56
|
-
let statement = `INSERT INTO ${table} (${chunk._columns.map(column => {
|
|
57
|
-
let mapping = getMappingForColumn(column);
|
|
58
|
-
return mapping
|
|
59
|
-
?
|
|
60
|
-
isKeyWord(mapping.targetColumn)
|
|
61
|
-
? `"${mapping.targetColumn}"`
|
|
62
|
-
: mapping.targetColumn
|
|
63
|
-
: isKeyWord(column)
|
|
64
|
-
? `"${mapping.column}"`
|
|
65
|
-
: mapping.column;
|
|
66
|
-
})
|
|
67
|
-
.join(",")}) VALUES (${chunk._columns.map((column, index) => {
|
|
68
|
-
let mapping = getMappingForColumn(column);
|
|
69
|
-
if (!mapping)
|
|
70
|
-
debug('Mapping not found for column %s', column);
|
|
71
|
-
if (mapping.targetType === "number" && (isNaN(chunk[index]) || chunk[index] === '')) {
|
|
72
|
-
debug('Source data is not a number');
|
|
73
|
-
return 0;
|
|
74
|
-
}
|
|
75
|
-
if (mapping.targetType === "varchar" || mapping.targetType === "char")
|
|
76
|
-
return `'${chunk[index]}'`;
|
|
77
|
-
return chunk[index] ? chunk[index] : 'null';
|
|
78
|
-
})})`;
|
|
79
|
-
return statement;
|
|
80
|
-
}
|
|
81
84
|
return new Transform({
|
|
82
85
|
objectMode: true,
|
|
83
86
|
emitClose: true,
|
|
@@ -90,7 +93,7 @@ function PostgresDestination(options) {
|
|
|
90
93
|
let insertStatement;
|
|
91
94
|
// @ts-ignore
|
|
92
95
|
if (chunk.errors.length === 0 | options.includeErrors) {
|
|
93
|
-
insertStatement = writeInsertStatement(chunk);
|
|
96
|
+
insertStatement = writeInsertStatement(mapping, table, chunk);
|
|
94
97
|
debug('Insert statement: [%s]', insertStatement);
|
|
95
98
|
// @ts-ignore
|
|
96
99
|
this.sequelize.query(insertStatement)
|
|
@@ -119,5 +122,8 @@ function PostgresDestination(options) {
|
|
|
119
122
|
});
|
|
120
123
|
}
|
|
121
124
|
module.exports = {
|
|
122
|
-
PostgresDestination
|
|
125
|
+
PostgresDestination,
|
|
126
|
+
writeInsertStatement,
|
|
127
|
+
isKeyWord,
|
|
128
|
+
getMappingForColumn
|
|
123
129
|
};
|
|
@@ -21,5 +21,8 @@ export function PostgresDestination(options: {
|
|
|
21
21
|
mapping: any;
|
|
22
22
|
includeErrors: any;
|
|
23
23
|
}): Transform;
|
|
24
|
+
export function writeInsertStatement(columnMapping: any, table: any, chunk: any): string;
|
|
25
|
+
export function isKeyWord(column: any): boolean;
|
|
26
|
+
export function getMappingForColumn(mapping: any, column: any): any;
|
|
24
27
|
import { Transform } from "stream";
|
|
25
28
|
//# sourceMappingURL=postgresDestination.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AA6CA;;;;;;;;;;;;GAYG;AACH;IAV2B,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,aAAa;cAqEvC;AA7GD,yFA4BC;AArCD,gDAEC;AAED,oEAGC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ffc-pay-etl-framework",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.13-alpha.10",
|
|
4
4
|
"publisher": "Defra",
|
|
5
5
|
"main": "dist/cjs/index.js",
|
|
6
6
|
"private": false,
|
|
@@ -63,6 +63,7 @@
|
|
|
63
63
|
"jest-junit": "^16.0.0",
|
|
64
64
|
"nodemon": "^3.1.4",
|
|
65
65
|
"prettier": "3.3.3",
|
|
66
|
+
"sequelize-mock": "^0.10.2",
|
|
66
67
|
"supports-color": "^9.4.0"
|
|
67
68
|
},
|
|
68
69
|
"dependencies": {
|