ffc-pay-etl-framework 0.0.6 → 0.0.7-alpha.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. package/dist/cjs/destinations/postgresDestination.js +16 -3
  2. package/dist/cjs/lib/index.js +1 -1
  3. package/dist/cjs/loaders/csvloader.js +1 -0
  4. package/dist/cjs/types/destinations/postgresDestination.d.ts.map +1 -1
  5. package/dist/cjs/types/loaders/csvloader.d.ts.map +1 -1
  6. package/dist/esm/destinations/postgresDestination.js +16 -3
  7. package/dist/esm/lib/index.js +1 -1
  8. package/dist/esm/loaders/csvloader.js +1 -0
  9. package/dist/esm/types/destinations/postgresDestination.d.ts.map +1 -1
  10. package/dist/esm/types/loaders/csvloader.d.ts.map +1 -1
  11. package/package.json +1 -1
package/dist/cjs/destinations/postgresDestination.js CHANGED
@@ -37,6 +37,7 @@ function PostgresDestination(options) {
37
37
  catch (e) {
38
38
  debug('sequelize.authenticate failed');
39
39
  debug(e);
40
+ throw e;
40
41
  }
41
42
  function getMappingForColumn(column) {
42
43
  const [map] = mapping.filter(m => m.column === column);
@@ -53,29 +54,41 @@ function PostgresDestination(options) {
53
54
  debug('Mapping not found for column %s', column);
54
55
  if (mapping.targetType === "varchar" || mapping.targetType === "char")
55
56
  return `'${chunk[index]}'`;
56
- return chunk[index];
57
+ return chunk[index] ? chunk[index] : 'null';
57
58
  })})`;
58
59
  return statement;
59
60
  }
60
61
  return new Transform({
61
62
  objectMode: true,
62
- transform(chunk, _, callback) {
63
+ emitClose: true,
64
+ construct(callback) {
65
+ // @ts-ignore
66
+ this.sequelize = sequelize;
67
+ callback();
68
+ },
69
+ write(chunk, _, callback) {
63
70
  let insertStatement;
64
71
  // @ts-ignore
65
72
  if (chunk.errors.length === 0 | options.includeErrors) {
66
73
  insertStatement = writeInsertStatement(chunk);
67
74
  debug('Insert statement: [%s]', insertStatement);
68
- sequelize.query(insertStatement)
75
+ // @ts-ignore
76
+ this.sequelize.query(insertStatement)
69
77
  .then(result => {
70
78
  debug('result %o', result);
71
79
  chunk._result = result;
80
+ // @ts-ignore
72
81
  callback(null, chunk);
73
82
  }).catch(error => {
74
83
  debug('error %o', error);
75
84
  chunk.errors.push(error);
85
+ // @ts-ignore
76
86
  callback(null, chunk);
77
87
  });
78
88
  }
89
+ else {
90
+ debug('Chunk has errors %o', chunk);
91
+ }
79
92
  }
80
93
  });
81
94
  }
package/dist/cjs/lib/index.js CHANGED
@@ -25,7 +25,7 @@ function Etl() {
25
25
  this.pump = () => {
26
26
  this.loader
27
27
  .pump(this.loader)
28
- .pipe(compose(RowMetaData, ...self.validatorList, ...self.transformationList, ...self.destinationList));
28
+ .pipe(compose(RowMetaData(), ...self.validatorList, ...self.transformationList, ...self.destinationList));
29
29
  };
30
30
  this.validator = (validator) => {
31
31
  self.validatorList.push(validator);
package/dist/cjs/loaders/csvloader.js CHANGED
@@ -18,6 +18,7 @@ function CSVLoader(options) {
18
18
  .pipe(new Transform({
19
19
  readableObjectMode: true,
20
20
  writableObjectMode: true,
21
+ emitClose: true,
21
22
  transform(chunk, _, callback) {
22
23
  chunk["_columns"] = options.columns;
23
24
  callback(null, chunk);
package/dist/cjs/types/destinations/postgresDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAIA;;;;;;;;;;;;GAYG;AACH,6CAVG;IAAwB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,aAAa;CACrC,aAkEF"}
1
+ {"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAIA;;;;;;;;;;;;GAYG;AACH,6CAVG;IAAwB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,aAAa;CACrC,aA+EF"}
package/dist/cjs/types/loaders/csvloader.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"csvloader.d.ts","sourceRoot":"","sources":["../../../../src/loaders/csvloader.js"],"names":[],"mappings":"AAMA;;;;;GAKG;AACH,mCAHG;IAAwB,IAAI;CAC5B,iBAmBF"}
1
+ {"version":3,"file":"csvloader.d.ts","sourceRoot":"","sources":["../../../../src/loaders/csvloader.js"],"names":[],"mappings":"AAMA;;;;;GAKG;AACH,mCAHG;IAAwB,IAAI;CAC5B,iBAoBF"}
package/dist/esm/destinations/postgresDestination.js CHANGED
@@ -38,6 +38,7 @@ function PostgresDestination(options) {
38
38
  catch (e) {
39
39
  debug('sequelize.authenticate failed');
40
40
  debug(e);
41
+ throw e;
41
42
  }
42
43
  function getMappingForColumn(column) {
43
44
  const [map] = mapping.filter(m => m.column === column);
@@ -54,29 +55,41 @@ function PostgresDestination(options) {
54
55
  debug('Mapping not found for column %s', column);
55
56
  if (mapping.targetType === "varchar" || mapping.targetType === "char")
56
57
  return `'${chunk[index]}'`;
57
- return chunk[index];
58
+ return chunk[index] ? chunk[index] : 'null';
58
59
  })})`;
59
60
  return statement;
60
61
  }
61
62
  return new Transform({
62
63
  objectMode: true,
63
- transform(chunk, _, callback) {
64
+ emitClose: true,
65
+ construct(callback) {
66
+ // @ts-ignore
67
+ this.sequelize = sequelize;
68
+ callback();
69
+ },
70
+ write(chunk, _, callback) {
64
71
  let insertStatement;
65
72
  // @ts-ignore
66
73
  if (chunk.errors.length === 0 | options.includeErrors) {
67
74
  insertStatement = writeInsertStatement(chunk);
68
75
  debug('Insert statement: [%s]', insertStatement);
69
- sequelize.query(insertStatement)
76
+ // @ts-ignore
77
+ this.sequelize.query(insertStatement)
70
78
  .then(result => {
71
79
  debug('result %o', result);
72
80
  chunk._result = result;
81
+ // @ts-ignore
73
82
  callback(null, chunk);
74
83
  }).catch(error => {
75
84
  debug('error %o', error);
76
85
  chunk.errors.push(error);
86
+ // @ts-ignore
77
87
  callback(null, chunk);
78
88
  });
79
89
  }
90
+ else {
91
+ debug('Chunk has errors %o', chunk);
92
+ }
80
93
  }
81
94
  });
82
95
  }
package/dist/esm/lib/index.js CHANGED
@@ -26,7 +26,7 @@ function Etl() {
26
26
  this.pump = () => {
27
27
  this.loader
28
28
  .pump(this.loader)
29
- .pipe(compose(RowMetaData, ...self.validatorList, ...self.transformationList, ...self.destinationList));
29
+ .pipe(compose(RowMetaData(), ...self.validatorList, ...self.transformationList, ...self.destinationList));
30
30
  };
31
31
  this.validator = (validator) => {
32
32
  self.validatorList.push(validator);
package/dist/esm/loaders/csvloader.js CHANGED
@@ -19,6 +19,7 @@ function CSVLoader(options) {
19
19
  .pipe(new Transform({
20
20
  readableObjectMode: true,
21
21
  writableObjectMode: true,
22
+ emitClose: true,
22
23
  transform(chunk, _, callback) {
23
24
  chunk["_columns"] = options.columns;
24
25
  callback(null, chunk);
package/dist/esm/types/destinations/postgresDestination.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAIA;;;;;;;;;;;;GAYG;AACH,6CAVG;IAAwB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,aAAa;CACrC,aAkEF"}
1
+ {"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAIA;;;;;;;;;;;;GAYG;AACH,6CAVG;IAAwB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,IAAI;IACJ,IAAI;IACJ,OAAO;IACP,aAAa;CACrC,aA+EF"}
package/dist/esm/types/loaders/csvloader.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"csvloader.d.ts","sourceRoot":"","sources":["../../../../src/loaders/csvloader.js"],"names":[],"mappings":"AAMA;;;;;GAKG;AACH,mCAHG;IAAwB,IAAI;CAC5B,iBAmBF"}
1
+ {"version":3,"file":"csvloader.d.ts","sourceRoot":"","sources":["../../../../src/loaders/csvloader.js"],"names":[],"mappings":"AAMA;;;;;GAKG;AACH,mCAHG;IAAwB,IAAI;CAC5B,iBAoBF"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ffc-pay-etl-framework",
3
- "version": "0.0.6",
3
+ "version": "0.0.7-alpha.4",
4
4
  "publisher": "Defra",
5
5
  "main": "dist/cjs/index.js",
6
6
  "private": false,