bellboy 8.1.2 → 8.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -121,6 +121,7 @@ const job = new bellboy.Job(
121
121
 
122
122
  Event listeners, which can be registered with `job.on` or `job.onAny` methods, allow you to listen to specific events in the job lifecycle and to interact with them.
123
123
 
124
+ - When multiple listeners are registered for the same event, those added using `.on` will always be executed first, regardless of the order in which they were added compared to `.onAny`. This ensures that specific event listeners have priority over generic ones.
124
125
  - When multiple listeners are registered for a single event, those added by [reporters](#reporters) will be executed first, followed by the order of registration for the remaining listeners.
125
126
  - Job always waits for the code inside a listener to complete.
126
127
  - Any error thrown inside a listener will be ignored and warning message will be printed out.
@@ -350,6 +351,7 @@ For delimited data produces rows described [here](#delimited-produced-row).
350
351
  If `true`, first row will be processed as a header.
351
352
  - **qualifier** `string` `only for delimited`\
352
353
  Symbol placed around a field to signify that it is the same field.
354
+ - **encoding** `string` `only for delimited`
353
355
  - **jsonPath** `RegExp | string`\
354
356
  Path to the array to be streamed. This option is described in detail inside [JsonProcessor](#json-processor) section.
355
357
  - **saxOptions** `object` `only for xml`\
@@ -480,6 +482,7 @@ Processes files with delimited data in the directory.
480
482
  If `true`, first row will be processed as a header.
481
483
  - **qualifier** `string` \
482
484
  Symbol placed around a field to signify that it is the same field.
485
+ - **encoding** `string` `only for delimited`
483
486
 
484
487
  #### Produced row <div id='delimited-produced-row'/>
485
488
 
@@ -63,10 +63,10 @@ var PostgresDestination = /** @class */ (function (_super) {
63
63
  }
64
64
  PostgresDestination.prototype.loadBatch = function (data) {
65
65
  return __awaiter(this, void 0, void 0, function () {
66
- var dataToUpload, columnDict, upsertConstraints, columnIndex, i, transformed, _i, _a, key, columns, cs, db;
66
+ var dataToUpload, columnDict, upsertConstraints, columnIndex, i, transformed, _i, _a, key, columnKeys, _b, dataToUpload_1, item, itemDataKeys, _c, columnKeys_1, key, columns, cs, db;
67
67
  var _this = this;
68
- return __generator(this, function (_b) {
69
- switch (_b.label) {
68
+ return __generator(this, function (_d) {
69
+ switch (_d.label) {
70
70
  case 0:
71
71
  dataToUpload = [];
72
72
  columnDict = {};
@@ -84,11 +84,22 @@ var PostgresDestination = /** @class */ (function (_super) {
84
84
  }
85
85
  dataToUpload.push(transformed);
86
86
  }
87
+ columnKeys = Object.values(columnDict);
88
+ for (_b = 0, dataToUpload_1 = dataToUpload; _b < dataToUpload_1.length; _b++) {
89
+ item = dataToUpload_1[_b];
90
+ itemDataKeys = Object.keys(item);
91
+ for (_c = 0, columnKeys_1 = columnKeys; _c < columnKeys_1.length; _c++) {
92
+ key = columnKeys_1[_c];
93
+ if (!itemDataKeys.includes(key)) {
94
+ item[key] = undefined;
95
+ }
96
+ }
97
+ }
87
98
  columns = Object.keys(columnDict).map(function (x) { return ({ name: x, prop: columnDict[x] }); });
88
99
  cs = new pgp.helpers.ColumnSet(columns, { table: this.table });
89
100
  return [4 /*yield*/, utils_1.getDb(this.connection, 'postgres')];
90
101
  case 1:
91
- db = _b.sent();
102
+ db = _d.sent();
92
103
  return [4 /*yield*/, db.tx(function (t) { return __awaiter(_this, void 0, void 0, function () {
93
104
  var query, columnsString;
94
105
  return __generator(this, function (_a) {
@@ -104,7 +115,7 @@ var PostgresDestination = /** @class */ (function (_super) {
104
115
  });
105
116
  }); })];
106
117
  case 2:
107
- _b.sent();
118
+ _d.sent();
108
119
  return [2 /*return*/];
109
120
  }
110
121
  });
@@ -5,6 +5,7 @@ export declare class DelimitedProcessor extends DirectoryProcessor {
5
5
  protected hasHeader: boolean;
6
6
  protected delimiter?: string;
7
7
  protected qualifier?: string;
8
+ protected encoding?: string;
8
9
  constructor(config: IDelimitedProcessorConfig);
9
10
  process(processStream: processStream): Promise<void>;
10
11
  }
@@ -56,6 +56,7 @@ exports.DelimitedProcessor = void 0;
56
56
  var csv_parse_1 = require("csv-parse");
57
57
  var fs_1 = __importDefault(require("fs"));
58
58
  var path_1 = __importDefault(require("path"));
59
+ var iconv_lite_1 = __importDefault(require("iconv-lite"));
59
60
  var utils_1 = require("../utils");
60
61
  var directory_processor_1 = require("./base/directory-processor");
61
62
  var DelimitedProcessor = /** @class */ (function (_super) {
@@ -69,11 +70,12 @@ var DelimitedProcessor = /** @class */ (function (_super) {
69
70
  _this.hasHeader = !!config.hasHeader;
70
71
  _this.delimiter = config.delimiter;
71
72
  _this.qualifier = config.qualifier;
73
+ _this.encoding = config.encoding;
72
74
  return _this;
73
75
  }
74
76
  DelimitedProcessor.prototype.process = function (processStream) {
75
77
  return __awaiter(this, void 0, void 0, function () {
76
- var _i, _a, file, filePath, fileReadStream, parser, readStream, generator;
78
+ var _i, _a, file, filePath, fileReadStream, streamToParse, parser, readStream, generator;
77
79
  return __generator(this, function (_b) {
78
80
  switch (_b.label) {
79
81
  case 0:
@@ -84,6 +86,7 @@ var DelimitedProcessor = /** @class */ (function (_super) {
84
86
  file = _a[_i];
85
87
  filePath = path_1.default.join(this.path, file);
86
88
  fileReadStream = fs_1.default.createReadStream(filePath);
89
+ streamToParse = this.encoding ? fileReadStream.pipe(iconv_lite_1.default.decodeStream(this.encoding)) : fileReadStream;
87
90
  parser = csv_parse_1.parse({
88
91
  quote: this.qualifier,
89
92
  delimiter: this.delimiter,
@@ -92,7 +95,7 @@ var DelimitedProcessor = /** @class */ (function (_super) {
92
95
  relax_quotes: true,
93
96
  relax_column_count: true,
94
97
  });
95
- readStream = fileReadStream.pipe(parser);
98
+ readStream = streamToParse.pipe(parser);
96
99
  generator = utils_1.getDelimitedGenerator({
97
100
  hasHeader: this.hasHeader,
98
101
  readStream: readStream,
@@ -12,6 +12,7 @@ export declare class HttpProcessor extends Processor {
12
12
  protected qualifier?: string;
13
13
  protected authorizationRequest?: AuthorizationRequest;
14
14
  protected saxOptions?: any;
15
+ protected encoding?: string;
15
16
  constructor(config: IJsonHttpProcessorConfig | IDelimitedHttpProcessorConfig | IXmlHttpProcessorConfig);
16
17
  protected processHttpData(processStream: processStream, options: AxiosRequestConfig): Promise<void>;
17
18
  process(processStream: processStream): Promise<void>;
@@ -69,6 +69,7 @@ var csv_parse_1 = require("csv-parse");
69
69
  var Pick_1 = require("stream-json/filters/Pick");
70
70
  var Parser_1 = require("stream-json/Parser");
71
71
  var StreamValues_1 = require("stream-json/streamers/StreamValues");
72
+ var iconv_lite_1 = __importDefault(require("iconv-lite"));
72
73
  var utils_1 = require("../utils");
73
74
  var processor_1 = require("./base/processor");
74
75
  var saxStream = require('sax-stream');
@@ -90,6 +91,7 @@ var HttpProcessor = /** @class */ (function (_super) {
90
91
  _this.hasHeader = !!config.hasHeader;
91
92
  _this.delimiter = config.delimiter;
92
93
  _this.qualifier = config.qualifier;
94
+ _this.encoding = config.encoding;
93
95
  }
94
96
  else if (config.dataFormat === 'json' && config.jsonPath) {
95
97
  if (config.jsonPath instanceof RegExp) {
@@ -108,7 +110,7 @@ var HttpProcessor = /** @class */ (function (_super) {
108
110
  }
109
111
  HttpProcessor.prototype.processHttpData = function (processStream, options) {
110
112
  return __awaiter(this, void 0, void 0, function () {
111
- var requestStream, parser_1, delimitedStream, generator, requestStream, jsonStream, requestStream, xmlStream, nextOptions;
113
+ var requestStream, decodedStream, parser_1, delimitedStream, generator, requestStream, jsonStream, requestStream, xmlStream, nextOptions;
112
114
  return __generator(this, function (_a) {
113
115
  switch (_a.label) {
114
116
  case 0:
@@ -116,14 +118,16 @@ var HttpProcessor = /** @class */ (function (_super) {
116
118
  return [4 /*yield*/, axios_1.default(__assign(__assign({}, options), { responseType: 'stream' }))];
117
119
  case 1:
118
120
  requestStream = _a.sent();
121
+ decodedStream = this.encoding ? requestStream.data.pipe(iconv_lite_1.default.decodeStream(this.encoding)) : requestStream.data;
119
122
  parser_1 = csv_parse_1.parse({
120
123
  quote: this.qualifier,
121
124
  delimiter: this.delimiter,
122
125
  record_delimiter: this.rowSeparator,
123
126
  raw: true,
124
127
  relax_quotes: true,
128
+ relax_column_count: true,
125
129
  });
126
- delimitedStream = requestStream.data.pipe(parser_1);
130
+ delimitedStream = decodedStream.pipe(parser_1);
127
131
  generator = utils_1.getDelimitedGenerator({
128
132
  readStream: delimitedStream,
129
133
  hasHeader: this.hasHeader,
package/lib/types.d.ts CHANGED
@@ -122,6 +122,7 @@ export interface IDelimitedHttpProcessorConfig extends IHttpProcessorConfig {
122
122
  delimiter?: string;
123
123
  hasHeader?: boolean;
124
124
  qualifier?: string;
125
+ encoding?: string;
125
126
  }
126
127
  export interface IDatabaseProcessorConfig extends IProcessorConfig {
127
128
  query: string;
@@ -134,6 +135,7 @@ export interface IDelimitedProcessorConfig extends IDirectoryProcessorConfig {
134
135
  delimiter?: string;
135
136
  hasHeader?: boolean;
136
137
  qualifier?: string;
138
+ encoding?: string;
137
139
  }
138
140
  export interface IMssqlProcessorConfig extends IDatabaseProcessorConfig {
139
141
  connection: IMssqlDbConnection;
package/package.json CHANGED
@@ -1,12 +1,12 @@
1
1
  {
2
2
  "name": "bellboy",
3
- "version": "8.1.2",
3
+ "version": "8.2.1",
4
4
  "description": "Highly performant JavaScript data stream ETL engine.",
5
5
  "main": "lib/index",
6
6
  "types": "lib/index",
7
7
  "scripts": {
8
8
  "build": "tsc -p .",
9
- "test": "jest --coverage",
9
+ "test": "jest --coverage ./tests/postgres-destination.spec.ts",
10
10
  "lint": "tslint -c tslint.json -p tsconfig.json",
11
11
  "codecov": "codecov"
12
12
  },
@@ -54,6 +54,7 @@
54
54
  "async-mqtt": "^2.2.1",
55
55
  "axios": "^0.21.1",
56
56
  "csv": "^6.0.5",
57
+ "iconv-lite": "^0.6.3",
57
58
  "mssql": "8.1.0",
58
59
  "mysql2-stream-fix": "^3.9.1",
59
60
  "pg": "8.0.3",
@@ -66,4 +67,4 @@
66
67
  "tail": "^2.0.2",
67
68
  "xlstream": "^2.5.3"
68
69
  }
69
- }
70
+ }