bellboy 8.1.2 → 8.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/lib/destinations/postgres-destination.js +16 -5
- package/lib/processors/delimited-processor.d.ts +1 -0
- package/lib/processors/delimited-processor.js +5 -2
- package/lib/processors/http-processor.d.ts +1 -0
- package/lib/processors/http-processor.js +6 -2
- package/lib/types.d.ts +2 -0
- package/package.json +4 -3
package/README.md
CHANGED
|
@@ -121,6 +121,7 @@ const job = new bellboy.Job(
|
|
|
121
121
|
|
|
122
122
|
Event listeners, which can be registered with `job.on` or `job.onAny` methods, allow you to listen to specific events in the job lifecycle and to interact with them.
|
|
123
123
|
|
|
124
|
+
- When multiple listeners are registered for the same event, those added using `.on` will always be executed first, regardless of the order in which they were added compared to `.onAny`. This ensures that specific event listeners have priority over generic ones.
|
|
124
125
|
- When multiple listeners are registered for a single event, those added by [reporters](#reporters) will be executed first, followed by the order of registration for the remaining listeners.
|
|
125
126
|
- Job always waits for the code inside a listener to complete.
|
|
126
127
|
- Any error thrown inside a listener will be ignored and warning message will be printed out.
|
|
@@ -350,6 +351,7 @@ For delimited data produces rows described [here](#delimited-produced-row).
|
|
|
350
351
|
If `true`, first row will be processed as a header.
|
|
351
352
|
- **qualifier** `string` `only for delimited`\
|
|
352
353
|
Symbol placed around a field to signify that it is the same field.
|
|
354
|
+
- **encoding** `string` `only for delimited`
|
|
353
355
|
- **jsonPath** `RegExp | string`\
|
|
354
356
|
Path to the array to be streamed. This option is described in detail inside [JsonProcessor](#json-processor) section.
|
|
355
357
|
- **saxOptions** `object` `only for xml`\
|
|
@@ -480,6 +482,7 @@ Processes files with delimited data in the directory.
|
|
|
480
482
|
If `true`, first row will be processed as a header.
|
|
481
483
|
- **qualifier** `string` \
|
|
482
484
|
Symbol placed around a field to signify that it is the same field.
|
|
485
|
+
- **encoding** `string` `only for delimited`
|
|
483
486
|
|
|
484
487
|
#### Produced row <div id='delimited-produced-row'/>
|
|
485
488
|
|
|
@@ -63,10 +63,10 @@ var PostgresDestination = /** @class */ (function (_super) {
|
|
|
63
63
|
}
|
|
64
64
|
PostgresDestination.prototype.loadBatch = function (data) {
|
|
65
65
|
return __awaiter(this, void 0, void 0, function () {
|
|
66
|
-
var dataToUpload, columnDict, upsertConstraints, columnIndex, i, transformed, _i, _a, key, columns, cs, db;
|
|
66
|
+
var dataToUpload, columnDict, upsertConstraints, columnIndex, i, transformed, _i, _a, key, columnKeys, _b, dataToUpload_1, item, itemDataKeys, _c, columnKeys_1, key, columns, cs, db;
|
|
67
67
|
var _this = this;
|
|
68
|
-
return __generator(this, function (
|
|
69
|
-
switch (
|
|
68
|
+
return __generator(this, function (_d) {
|
|
69
|
+
switch (_d.label) {
|
|
70
70
|
case 0:
|
|
71
71
|
dataToUpload = [];
|
|
72
72
|
columnDict = {};
|
|
@@ -84,11 +84,22 @@ var PostgresDestination = /** @class */ (function (_super) {
|
|
|
84
84
|
}
|
|
85
85
|
dataToUpload.push(transformed);
|
|
86
86
|
}
|
|
87
|
+
columnKeys = Object.values(columnDict);
|
|
88
|
+
for (_b = 0, dataToUpload_1 = dataToUpload; _b < dataToUpload_1.length; _b++) {
|
|
89
|
+
item = dataToUpload_1[_b];
|
|
90
|
+
itemDataKeys = Object.keys(item);
|
|
91
|
+
for (_c = 0, columnKeys_1 = columnKeys; _c < columnKeys_1.length; _c++) {
|
|
92
|
+
key = columnKeys_1[_c];
|
|
93
|
+
if (!itemDataKeys.includes(key)) {
|
|
94
|
+
item[key] = undefined;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
87
98
|
columns = Object.keys(columnDict).map(function (x) { return ({ name: x, prop: columnDict[x] }); });
|
|
88
99
|
cs = new pgp.helpers.ColumnSet(columns, { table: this.table });
|
|
89
100
|
return [4 /*yield*/, utils_1.getDb(this.connection, 'postgres')];
|
|
90
101
|
case 1:
|
|
91
|
-
db =
|
|
102
|
+
db = _d.sent();
|
|
92
103
|
return [4 /*yield*/, db.tx(function (t) { return __awaiter(_this, void 0, void 0, function () {
|
|
93
104
|
var query, columnsString;
|
|
94
105
|
return __generator(this, function (_a) {
|
|
@@ -104,7 +115,7 @@ var PostgresDestination = /** @class */ (function (_super) {
|
|
|
104
115
|
});
|
|
105
116
|
}); })];
|
|
106
117
|
case 2:
|
|
107
|
-
|
|
118
|
+
_d.sent();
|
|
108
119
|
return [2 /*return*/];
|
|
109
120
|
}
|
|
110
121
|
});
|
|
@@ -5,6 +5,7 @@ export declare class DelimitedProcessor extends DirectoryProcessor {
|
|
|
5
5
|
protected hasHeader: boolean;
|
|
6
6
|
protected delimiter?: string;
|
|
7
7
|
protected qualifier?: string;
|
|
8
|
+
protected encoding?: string;
|
|
8
9
|
constructor(config: IDelimitedProcessorConfig);
|
|
9
10
|
process(processStream: processStream): Promise<void>;
|
|
10
11
|
}
|
|
@@ -56,6 +56,7 @@ exports.DelimitedProcessor = void 0;
|
|
|
56
56
|
var csv_parse_1 = require("csv-parse");
|
|
57
57
|
var fs_1 = __importDefault(require("fs"));
|
|
58
58
|
var path_1 = __importDefault(require("path"));
|
|
59
|
+
var iconv_lite_1 = __importDefault(require("iconv-lite"));
|
|
59
60
|
var utils_1 = require("../utils");
|
|
60
61
|
var directory_processor_1 = require("./base/directory-processor");
|
|
61
62
|
var DelimitedProcessor = /** @class */ (function (_super) {
|
|
@@ -69,11 +70,12 @@ var DelimitedProcessor = /** @class */ (function (_super) {
|
|
|
69
70
|
_this.hasHeader = !!config.hasHeader;
|
|
70
71
|
_this.delimiter = config.delimiter;
|
|
71
72
|
_this.qualifier = config.qualifier;
|
|
73
|
+
_this.encoding = config.encoding;
|
|
72
74
|
return _this;
|
|
73
75
|
}
|
|
74
76
|
DelimitedProcessor.prototype.process = function (processStream) {
|
|
75
77
|
return __awaiter(this, void 0, void 0, function () {
|
|
76
|
-
var _i, _a, file, filePath, fileReadStream, parser, readStream, generator;
|
|
78
|
+
var _i, _a, file, filePath, fileReadStream, streamToParse, parser, readStream, generator;
|
|
77
79
|
return __generator(this, function (_b) {
|
|
78
80
|
switch (_b.label) {
|
|
79
81
|
case 0:
|
|
@@ -84,6 +86,7 @@ var DelimitedProcessor = /** @class */ (function (_super) {
|
|
|
84
86
|
file = _a[_i];
|
|
85
87
|
filePath = path_1.default.join(this.path, file);
|
|
86
88
|
fileReadStream = fs_1.default.createReadStream(filePath);
|
|
89
|
+
streamToParse = this.encoding ? fileReadStream.pipe(iconv_lite_1.default.decodeStream(this.encoding)) : fileReadStream;
|
|
87
90
|
parser = csv_parse_1.parse({
|
|
88
91
|
quote: this.qualifier,
|
|
89
92
|
delimiter: this.delimiter,
|
|
@@ -92,7 +95,7 @@ var DelimitedProcessor = /** @class */ (function (_super) {
|
|
|
92
95
|
relax_quotes: true,
|
|
93
96
|
relax_column_count: true,
|
|
94
97
|
});
|
|
95
|
-
readStream =
|
|
98
|
+
readStream = streamToParse.pipe(parser);
|
|
96
99
|
generator = utils_1.getDelimitedGenerator({
|
|
97
100
|
hasHeader: this.hasHeader,
|
|
98
101
|
readStream: readStream,
|
|
@@ -12,6 +12,7 @@ export declare class HttpProcessor extends Processor {
|
|
|
12
12
|
protected qualifier?: string;
|
|
13
13
|
protected authorizationRequest?: AuthorizationRequest;
|
|
14
14
|
protected saxOptions?: any;
|
|
15
|
+
protected encoding?: string;
|
|
15
16
|
constructor(config: IJsonHttpProcessorConfig | IDelimitedHttpProcessorConfig | IXmlHttpProcessorConfig);
|
|
16
17
|
protected processHttpData(processStream: processStream, options: AxiosRequestConfig): Promise<void>;
|
|
17
18
|
process(processStream: processStream): Promise<void>;
|
|
@@ -69,6 +69,7 @@ var csv_parse_1 = require("csv-parse");
|
|
|
69
69
|
var Pick_1 = require("stream-json/filters/Pick");
|
|
70
70
|
var Parser_1 = require("stream-json/Parser");
|
|
71
71
|
var StreamValues_1 = require("stream-json/streamers/StreamValues");
|
|
72
|
+
var iconv_lite_1 = __importDefault(require("iconv-lite"));
|
|
72
73
|
var utils_1 = require("../utils");
|
|
73
74
|
var processor_1 = require("./base/processor");
|
|
74
75
|
var saxStream = require('sax-stream');
|
|
@@ -90,6 +91,7 @@ var HttpProcessor = /** @class */ (function (_super) {
|
|
|
90
91
|
_this.hasHeader = !!config.hasHeader;
|
|
91
92
|
_this.delimiter = config.delimiter;
|
|
92
93
|
_this.qualifier = config.qualifier;
|
|
94
|
+
_this.encoding = config.encoding;
|
|
93
95
|
}
|
|
94
96
|
else if (config.dataFormat === 'json' && config.jsonPath) {
|
|
95
97
|
if (config.jsonPath instanceof RegExp) {
|
|
@@ -108,7 +110,7 @@ var HttpProcessor = /** @class */ (function (_super) {
|
|
|
108
110
|
}
|
|
109
111
|
HttpProcessor.prototype.processHttpData = function (processStream, options) {
|
|
110
112
|
return __awaiter(this, void 0, void 0, function () {
|
|
111
|
-
var requestStream, parser_1, delimitedStream, generator, requestStream, jsonStream, requestStream, xmlStream, nextOptions;
|
|
113
|
+
var requestStream, decodedStream, parser_1, delimitedStream, generator, requestStream, jsonStream, requestStream, xmlStream, nextOptions;
|
|
112
114
|
return __generator(this, function (_a) {
|
|
113
115
|
switch (_a.label) {
|
|
114
116
|
case 0:
|
|
@@ -116,14 +118,16 @@ var HttpProcessor = /** @class */ (function (_super) {
|
|
|
116
118
|
return [4 /*yield*/, axios_1.default(__assign(__assign({}, options), { responseType: 'stream' }))];
|
|
117
119
|
case 1:
|
|
118
120
|
requestStream = _a.sent();
|
|
121
|
+
decodedStream = this.encoding ? requestStream.data.pipe(iconv_lite_1.default.decodeStream(this.encoding)) : requestStream.data;
|
|
119
122
|
parser_1 = csv_parse_1.parse({
|
|
120
123
|
quote: this.qualifier,
|
|
121
124
|
delimiter: this.delimiter,
|
|
122
125
|
record_delimiter: this.rowSeparator,
|
|
123
126
|
raw: true,
|
|
124
127
|
relax_quotes: true,
|
|
128
|
+
relax_column_count: true,
|
|
125
129
|
});
|
|
126
|
-
delimitedStream =
|
|
130
|
+
delimitedStream = decodedStream.pipe(parser_1);
|
|
127
131
|
generator = utils_1.getDelimitedGenerator({
|
|
128
132
|
readStream: delimitedStream,
|
|
129
133
|
hasHeader: this.hasHeader,
|
package/lib/types.d.ts
CHANGED
|
@@ -122,6 +122,7 @@ export interface IDelimitedHttpProcessorConfig extends IHttpProcessorConfig {
|
|
|
122
122
|
delimiter?: string;
|
|
123
123
|
hasHeader?: boolean;
|
|
124
124
|
qualifier?: string;
|
|
125
|
+
encoding?: string;
|
|
125
126
|
}
|
|
126
127
|
export interface IDatabaseProcessorConfig extends IProcessorConfig {
|
|
127
128
|
query: string;
|
|
@@ -134,6 +135,7 @@ export interface IDelimitedProcessorConfig extends IDirectoryProcessorConfig {
|
|
|
134
135
|
delimiter?: string;
|
|
135
136
|
hasHeader?: boolean;
|
|
136
137
|
qualifier?: string;
|
|
138
|
+
encoding?: string;
|
|
137
139
|
}
|
|
138
140
|
export interface IMssqlProcessorConfig extends IDatabaseProcessorConfig {
|
|
139
141
|
connection: IMssqlDbConnection;
|
package/package.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bellboy",
|
|
3
|
-
"version": "8.1
|
|
3
|
+
"version": "8.2.1",
|
|
4
4
|
"description": "Highly performant JavaScript data stream ETL engine.",
|
|
5
5
|
"main": "lib/index",
|
|
6
6
|
"types": "lib/index",
|
|
7
7
|
"scripts": {
|
|
8
8
|
"build": "tsc -p .",
|
|
9
|
-
"test": "jest --coverage",
|
|
9
|
+
"test": "jest --coverage ./tests/postgres-destination.spec.ts",
|
|
10
10
|
"lint": "tslint -c tslint.json -p tsconfig.json",
|
|
11
11
|
"codecov": "codecov"
|
|
12
12
|
},
|
|
@@ -54,6 +54,7 @@
|
|
|
54
54
|
"async-mqtt": "^2.2.1",
|
|
55
55
|
"axios": "^0.21.1",
|
|
56
56
|
"csv": "^6.0.5",
|
|
57
|
+
"iconv-lite": "^0.6.3",
|
|
57
58
|
"mssql": "8.1.0",
|
|
58
59
|
"mysql2-stream-fix": "^3.9.1",
|
|
59
60
|
"pg": "8.0.3",
|
|
@@ -66,4 +67,4 @@
|
|
|
66
67
|
"tail": "^2.0.2",
|
|
67
68
|
"xlstream": "^2.5.3"
|
|
68
69
|
}
|
|
69
|
-
}
|
|
70
|
+
}
|