ffc-pay-etl-framework 1.1.10 → 1.1.11-alpha.2

package/dist/cjs/destinations/postgresDestination.js

@@ -18,8 +18,8 @@ function hasReturningColumns(mapping) {
 function getReturningColumns(mapping) {
     return mapping.filter(m => m.returning === true).map(m => m === null || m === void 0 ? void 0 : m.targetColumn);
 }
-function writeInsertStatement(columnMapping, table, chunk) {
-    let statement = `INSERT INTO "${table}" (${chunk._columns.map(column => {
+function writeInsertStatement(columnMapping, table, chunk, schema) {
+    let statement = `INSERT INTO ${schema !== null && schema !== void 0 ? schema : 'public'}."${table}" (${chunk._columns.map(column => {
         const mapping = getMappingForColumn(columnMapping, column);
         return (mapping === null || mapping === void 0 ? void 0 : mapping.targetColumn) ? `"${mapping.targetColumn}"` : `"${mapping === null || mapping === void 0 ? void 0 : mapping.column}"`;
     })
@@ -52,6 +52,7 @@ function writeInsertStatement(columnMapping, table, chunk) {
  * @param {Object} options.connectionname
  * @param {Object} options.mapping
  * @param {Object} options.includeErrors
+ * @param {String} [options.schema]
  * @returns Transform
  */
 function PostgresDestination(options) {
@@ -60,6 +61,7 @@ function PostgresDestination(options) {
     const connectionname = options.connectionname;
     const mapping = options.mapping;
     const includeErrors = options.includeErrors;
+    const schema = options.schema;
     let lastChunk;
     const transform = new Transform({
         objectMode: true,
@@ -73,7 +75,7 @@ function PostgresDestination(options) {
             let insertStatement;
             // @ts-ignore
             if (chunk.errors.length === 0 || options.includeErrors) {
-                insertStatement = writeInsertStatement(mapping, table, chunk);
+                insertStatement = writeInsertStatement(mapping, table, chunk, schema);
                 debug('Insert statement: [%s]', insertStatement);
                 // @ts-ignore
                 this.connection.db.query(insertStatement)

package/dist/cjs/loaders/csvloader.js

@@ -8,10 +8,11 @@ const { parse } = require("csv-parse");
  * @param {String} [options.path]
  * @param {ReadableStream} [options.stream]
  * @param {Number} [options.startingLine]
+ * @param {Boolean} [options.relax]
  * @returns StreamReader
  */
 function CSVLoader(options) {
-    var _a;
+    var _a, _b;
     let csvLoader;
     if (options.path) {
         csvLoader = fs.createReadStream(options.path);
@@ -21,11 +22,11 @@ function CSVLoader(options) {
     }
     let lineCount = 1;
     const fromLine = (_a = options.startingLine) !== null && _a !== void 0 ? _a : 2;
+    const relaxQuotes = (_b = options.relax) !== null && _b !== void 0 ? _b : false;
     csvLoader._columns = options.columns;
     csvLoader.pump = (csvLoader) => {
-        return csvLoader
-            .pipe(parse({ delimiter: ",", from_line: fromLine }))
-            .pipe(new Transform({
+        const parser = parse({ delimiter: ",", from_line: fromLine, relax_quotes: relaxQuotes });
+        const transformer = new Transform({
             readableObjectMode: true,
             writableObjectMode: true,
             emitClose: true,
@@ -41,7 +42,10 @@ function CSVLoader(options) {
                 });
                 callback(null, chunk);
             }
-        }));
+        });
+        return csvLoader
+            .pipe(parser)
+            .pipe(transformer);
     };
     return csvLoader;
 }

package/dist/cjs/types/destinations/postgresDestination.d.ts

@@ -5,6 +5,7 @@
  * @param {Object} options.connectionname
  * @param {Object} options.mapping
  * @param {Object} options.includeErrors
+ * @param {String} [options.schema]
  * @returns Transform
  */
 export function PostgresDestination(options: {
@@ -12,8 +13,9 @@ export function PostgresDestination(options: {
     connectionname: any;
     mapping: any;
     includeErrors: any;
+    schema?: string;
 }): Transform;
-export function writeInsertStatement(columnMapping: any, table: any, chunk: any): string;
+export function writeInsertStatement(columnMapping: any, table: any, chunk: any, schema: any): string;
 export function isKeyWord(column: any): boolean;
 export function getMappingForColumn(mapping: any, column: any): any;
 export function hasReturningColumns(mapping: any): boolean;

package/dist/cjs/types/destinations/postgresDestination.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAmDA;;;;;;;;GAQG;AACH;IAN2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;cAgEvC;AAlGD,yFA0BC;AA5CD,gDAEC;AAED,oEAIC;AAED,2DAEC;AAED,uDAEC"}
+{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAmDA;;;;;;;;;GASG;AACH;IAP2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;IACZ,MAAM;cAiEjC;AApGD,sGA0BC;AA5CD,gDAEC;AAED,oEAIC;AAED,2DAEC;AAED,uDAEC"}

package/dist/cjs/types/loaders/csvloader.d.ts

@@ -5,12 +5,14 @@
  * @param {String} [options.path]
  * @param {ReadableStream} [options.stream]
  * @param {Number} [options.startingLine]
+ * @param {Boolean} [options.relax]
  * @returns StreamReader
  */
 export function CSVLoader(options: {
     path?: string;
     stream?: ReadableStream;
     startingLine?: number;
+    relax?: boolean;
 }): ReadableStream<any> | fs.ReadStream;
 import fs = require("fs");
 //# sourceMappingURL=csvloader.d.ts.map

package/dist/cjs/types/loaders/csvloader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csvloader.d.ts","sourceRoot":"","sources":["../../../../src/loaders/csvloader.js"],"names":[],"mappings":";AAKA;;;;;;;GAOG;AACH;IAL4B,IAAI;IACI,MAAM,GAA/B,cAAc;IACG,YAAY;wCAsCvC"}
+{"version":3,"file":"csvloader.d.ts","sourceRoot":"","sources":["../../../../src/loaders/csvloader.js"],"names":[],"mappings":";AAKA;;;;;;;;GAQG;AACH;IAN4B,IAAI;IACI,MAAM,GAA/B,cAAc;IACG,YAAY;IACX,KAAK;wCAyCjC"}

package/dist/esm/destinations/postgresDestination.js

@@ -20,8 +20,8 @@ function hasReturningColumns(mapping) {
 function getReturningColumns(mapping) {
     return mapping.filter(m => m.returning === true).map(m => m?.targetColumn);
 }
-function writeInsertStatement(columnMapping, table, chunk) {
-    let statement = `INSERT INTO "${table}" (${chunk._columns.map(column => {
+function writeInsertStatement(columnMapping, table, chunk, schema) {
+    let statement = `INSERT INTO ${schema ?? 'public'}."${table}" (${chunk._columns.map(column => {
         const mapping = getMappingForColumn(columnMapping, column);
         return mapping?.targetColumn ? `"${mapping.targetColumn}"` : `"${mapping?.column}"`;
     })
@@ -54,6 +54,7 @@ function writeInsertStatement(columnMapping, table, chunk) {
  * @param {Object} options.connectionname
  * @param {Object} options.mapping
  * @param {Object} options.includeErrors
+ * @param {String} [options.schema]
  * @returns Transform
  */
 function PostgresDestination(options) {
@@ -62,6 +63,7 @@ function PostgresDestination(options) {
     const connectionname = options.connectionname;
     const mapping = options.mapping;
     const includeErrors = options.includeErrors;
+    const schema = options.schema;
     let lastChunk;
     const transform = new Transform({
         objectMode: true,
@@ -75,7 +77,7 @@ function PostgresDestination(options) {
             let insertStatement;
             // @ts-ignore
             if (chunk.errors.length === 0 || options.includeErrors) {
-                insertStatement = writeInsertStatement(mapping, table, chunk);
+                insertStatement = writeInsertStatement(mapping, table, chunk, schema);
                 debug('Insert statement: [%s]', insertStatement);
                 // @ts-ignore
                 this.connection.db.query(insertStatement)

package/dist/esm/loaders/csvloader.js

@@ -10,6 +10,7 @@ const { parse } = require("csv-parse");
  * @param {String} [options.path]
  * @param {ReadableStream} [options.stream]
  * @param {Number} [options.startingLine]
+ * @param {Boolean} [options.relax]
  * @returns StreamReader
  */
 function CSVLoader(options) {
@@ -22,11 +23,11 @@ function CSVLoader(options) {
     }
     let lineCount = 1;
     const fromLine = options.startingLine ?? 2;
+    const relaxQuotes = options.relax ?? false;
     csvLoader._columns = options.columns;
     csvLoader.pump = (csvLoader) => {
-        return csvLoader
-            .pipe(parse({ delimiter: ",", from_line: fromLine }))
-            .pipe(new Transform({
+        const parser = parse({ delimiter: ",", from_line: fromLine, relax_quotes: relaxQuotes });
+        const transformer = new Transform({
             readableObjectMode: true,
             writableObjectMode: true,
             emitClose: true,
@@ -42,7 +43,10 @@ function CSVLoader(options) {
                 });
                 callback(null, chunk);
             }
-        }));
+        });
+        return csvLoader
+            .pipe(parser)
+            .pipe(transformer);
     };
     return csvLoader;
 }

package/dist/esm/types/destinations/postgresDestination.d.ts

@@ -5,6 +5,7 @@
  * @param {Object} options.connectionname
  * @param {Object} options.mapping
  * @param {Object} options.includeErrors
+ * @param {String} [options.schema]
  * @returns Transform
  */
 export function PostgresDestination(options: {
@@ -12,8 +13,9 @@ export function PostgresDestination(options: {
     connectionname: any;
     mapping: any;
     includeErrors: any;
+    schema?: string;
 }): Transform;
-export function writeInsertStatement(columnMapping: any, table: any, chunk: any): string;
+export function writeInsertStatement(columnMapping: any, table: any, chunk: any, schema: any): string;
 export function isKeyWord(column: any): boolean;
 export function getMappingForColumn(mapping: any, column: any): any;
 export function hasReturningColumns(mapping: any): boolean;

package/dist/esm/types/destinations/postgresDestination.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAmDA;;;;;;;;GAQG;AACH;IAN2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;cAgEvC;AAlGD,yFA0BC;AA5CD,gDAEC;AAED,oEAIC;AAED,2DAEC;AAED,uDAEC"}
+{"version":3,"file":"postgresDestination.d.ts","sourceRoot":"","sources":["../../../../src/destinations/postgresDestination.js"],"names":[],"mappings":"AAmDA;;;;;;;;;GASG;AACH;IAP2B,KAAK;IACL,cAAc;IACd,OAAO;IACP,aAAa;IACZ,MAAM;cAiEjC;AApGD,sGA0BC;AA5CD,gDAEC;AAED,oEAIC;AAED,2DAEC;AAED,uDAEC"}

package/dist/esm/types/loaders/csvloader.d.ts

@@ -5,12 +5,14 @@
  * @param {String} [options.path]
  * @param {ReadableStream} [options.stream]
  * @param {Number} [options.startingLine]
+ * @param {Boolean} [options.relax]
  * @returns StreamReader
  */
 export function CSVLoader(options: {
     path?: string;
     stream?: ReadableStream;
     startingLine?: number;
+    relax?: boolean;
 }): ReadableStream<any> | fs.ReadStream;
 import fs = require("fs");
 //# sourceMappingURL=csvloader.d.ts.map

package/dist/esm/types/loaders/csvloader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csvloader.d.ts","sourceRoot":"","sources":["../../../../src/loaders/csvloader.js"],"names":[],"mappings":";AAKA;;;;;;;GAOG;AACH;IAL4B,IAAI;IACI,MAAM,GAA/B,cAAc;IACG,YAAY;wCAsCvC"}
+{"version":3,"file":"csvloader.d.ts","sourceRoot":"","sources":["../../../../src/loaders/csvloader.js"],"names":[],"mappings":";AAKA;;;;;;;;GAQG;AACH;IAN4B,IAAI;IACI,MAAM,GAA/B,cAAc;IACG,YAAY;IACX,KAAK;wCAyCjC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ffc-pay-etl-framework",
-  "version": "1.1.10",
+  "version": "1.1.11-alpha.2",
   "publisher": "Defra",
   "main": "dist/cjs/index.js",
   "private": false,