bellboy 8.1.1 → 8.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/lib/destinations/mysql-destination.js +1 -1
- package/lib/processors/delimited-processor.d.ts +1 -0
- package/lib/processors/delimited-processor.js +5 -2
- package/lib/processors/http-processor.d.ts +1 -0
- package/lib/processors/http-processor.js +6 -2
- package/lib/processors/mysql-processor.js +4 -27
- package/lib/types.d.ts +2 -0
- package/lib/utils.js +1 -1
- package/package.json +4 -3
package/README.md
CHANGED
|
@@ -350,6 +350,7 @@ For delimited data produces rows described [here](#delimited-produced-row).
|
|
|
350
350
|
If `true`, first row will be processed as a header.
|
|
351
351
|
- **qualifier** `string` `only for delimited`\
|
|
352
352
|
Symbol placed around a field to signify that it is the same field.
|
|
353
|
+
- **encoding** `string` `only for delimited`
|
|
353
354
|
- **jsonPath** `RegExp | string`\
|
|
354
355
|
Path to the array to be streamed. This option is described in detail inside [JsonProcessor](#json-processor) section.
|
|
355
356
|
- **saxOptions** `object` `only for xml`\
|
|
@@ -480,6 +481,7 @@ Processes files with delimited data in the directory.
|
|
|
480
481
|
If `true`, first row will be processed as a header.
|
|
481
482
|
- **qualifier** `string` \
|
|
482
483
|
Symbol placed around a field to signify that it is the same field.
|
|
484
|
+
- **encoding** `string` `only for delimited`
|
|
483
485
|
|
|
484
486
|
#### Produced row <div id='delimited-produced-row'/>
|
|
485
487
|
|
|
@@ -54,7 +54,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
54
54
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
55
55
|
exports.MySqlDestination = void 0;
|
|
56
56
|
var database_destination_1 = require("./base/database-destination");
|
|
57
|
-
var promise_1 = __importDefault(require("mysql2/promise"));
|
|
57
|
+
var promise_1 = __importDefault(require("mysql2-stream-fix/promise"));
|
|
58
58
|
var MySqlDestination = /** @class */ (function (_super) {
|
|
59
59
|
__extends(MySqlDestination, _super);
|
|
60
60
|
function MySqlDestination(config) {
|
|
@@ -5,6 +5,7 @@ export declare class DelimitedProcessor extends DirectoryProcessor {
|
|
|
5
5
|
protected hasHeader: boolean;
|
|
6
6
|
protected delimiter?: string;
|
|
7
7
|
protected qualifier?: string;
|
|
8
|
+
protected encoding?: string;
|
|
8
9
|
constructor(config: IDelimitedProcessorConfig);
|
|
9
10
|
process(processStream: processStream): Promise<void>;
|
|
10
11
|
}
|
|
@@ -56,6 +56,7 @@ exports.DelimitedProcessor = void 0;
|
|
|
56
56
|
var csv_parse_1 = require("csv-parse");
|
|
57
57
|
var fs_1 = __importDefault(require("fs"));
|
|
58
58
|
var path_1 = __importDefault(require("path"));
|
|
59
|
+
var iconv_lite_1 = __importDefault(require("iconv-lite"));
|
|
59
60
|
var utils_1 = require("../utils");
|
|
60
61
|
var directory_processor_1 = require("./base/directory-processor");
|
|
61
62
|
var DelimitedProcessor = /** @class */ (function (_super) {
|
|
@@ -69,11 +70,12 @@ var DelimitedProcessor = /** @class */ (function (_super) {
|
|
|
69
70
|
_this.hasHeader = !!config.hasHeader;
|
|
70
71
|
_this.delimiter = config.delimiter;
|
|
71
72
|
_this.qualifier = config.qualifier;
|
|
73
|
+
_this.encoding = config.encoding;
|
|
72
74
|
return _this;
|
|
73
75
|
}
|
|
74
76
|
DelimitedProcessor.prototype.process = function (processStream) {
|
|
75
77
|
return __awaiter(this, void 0, void 0, function () {
|
|
76
|
-
var _i, _a, file, filePath, fileReadStream, parser, readStream, generator;
|
|
78
|
+
var _i, _a, file, filePath, fileReadStream, streamToParse, parser, readStream, generator;
|
|
77
79
|
return __generator(this, function (_b) {
|
|
78
80
|
switch (_b.label) {
|
|
79
81
|
case 0:
|
|
@@ -84,6 +86,7 @@ var DelimitedProcessor = /** @class */ (function (_super) {
|
|
|
84
86
|
file = _a[_i];
|
|
85
87
|
filePath = path_1.default.join(this.path, file);
|
|
86
88
|
fileReadStream = fs_1.default.createReadStream(filePath);
|
|
89
|
+
streamToParse = this.encoding ? fileReadStream.pipe(iconv_lite_1.default.decodeStream(this.encoding)) : fileReadStream;
|
|
87
90
|
parser = csv_parse_1.parse({
|
|
88
91
|
quote: this.qualifier,
|
|
89
92
|
delimiter: this.delimiter,
|
|
@@ -92,7 +95,7 @@ var DelimitedProcessor = /** @class */ (function (_super) {
|
|
|
92
95
|
relax_quotes: true,
|
|
93
96
|
relax_column_count: true,
|
|
94
97
|
});
|
|
95
|
-
readStream =
|
|
98
|
+
readStream = streamToParse.pipe(parser);
|
|
96
99
|
generator = utils_1.getDelimitedGenerator({
|
|
97
100
|
hasHeader: this.hasHeader,
|
|
98
101
|
readStream: readStream,
|
|
@@ -12,6 +12,7 @@ export declare class HttpProcessor extends Processor {
|
|
|
12
12
|
protected qualifier?: string;
|
|
13
13
|
protected authorizationRequest?: AuthorizationRequest;
|
|
14
14
|
protected saxOptions?: any;
|
|
15
|
+
protected encoding?: string;
|
|
15
16
|
constructor(config: IJsonHttpProcessorConfig | IDelimitedHttpProcessorConfig | IXmlHttpProcessorConfig);
|
|
16
17
|
protected processHttpData(processStream: processStream, options: AxiosRequestConfig): Promise<void>;
|
|
17
18
|
process(processStream: processStream): Promise<void>;
|
|
@@ -69,6 +69,7 @@ var csv_parse_1 = require("csv-parse");
|
|
|
69
69
|
var Pick_1 = require("stream-json/filters/Pick");
|
|
70
70
|
var Parser_1 = require("stream-json/Parser");
|
|
71
71
|
var StreamValues_1 = require("stream-json/streamers/StreamValues");
|
|
72
|
+
var iconv_lite_1 = __importDefault(require("iconv-lite"));
|
|
72
73
|
var utils_1 = require("../utils");
|
|
73
74
|
var processor_1 = require("./base/processor");
|
|
74
75
|
var saxStream = require('sax-stream');
|
|
@@ -90,6 +91,7 @@ var HttpProcessor = /** @class */ (function (_super) {
|
|
|
90
91
|
_this.hasHeader = !!config.hasHeader;
|
|
91
92
|
_this.delimiter = config.delimiter;
|
|
92
93
|
_this.qualifier = config.qualifier;
|
|
94
|
+
_this.encoding = config.encoding;
|
|
93
95
|
}
|
|
94
96
|
else if (config.dataFormat === 'json' && config.jsonPath) {
|
|
95
97
|
if (config.jsonPath instanceof RegExp) {
|
|
@@ -108,7 +110,7 @@ var HttpProcessor = /** @class */ (function (_super) {
|
|
|
108
110
|
}
|
|
109
111
|
HttpProcessor.prototype.processHttpData = function (processStream, options) {
|
|
110
112
|
return __awaiter(this, void 0, void 0, function () {
|
|
111
|
-
var requestStream, parser_1, delimitedStream, generator, requestStream, jsonStream, requestStream, xmlStream, nextOptions;
|
|
113
|
+
var requestStream, decodedStream, parser_1, delimitedStream, generator, requestStream, jsonStream, requestStream, xmlStream, nextOptions;
|
|
112
114
|
return __generator(this, function (_a) {
|
|
113
115
|
switch (_a.label) {
|
|
114
116
|
case 0:
|
|
@@ -116,14 +118,16 @@ var HttpProcessor = /** @class */ (function (_super) {
|
|
|
116
118
|
return [4 /*yield*/, axios_1.default(__assign(__assign({}, options), { responseType: 'stream' }))];
|
|
117
119
|
case 1:
|
|
118
120
|
requestStream = _a.sent();
|
|
121
|
+
decodedStream = this.encoding ? requestStream.data.pipe(iconv_lite_1.default.decodeStream(this.encoding)) : requestStream.data;
|
|
119
122
|
parser_1 = csv_parse_1.parse({
|
|
120
123
|
quote: this.qualifier,
|
|
121
124
|
delimiter: this.delimiter,
|
|
122
125
|
record_delimiter: this.rowSeparator,
|
|
123
126
|
raw: true,
|
|
124
127
|
relax_quotes: true,
|
|
128
|
+
relax_column_count: true,
|
|
125
129
|
});
|
|
126
|
-
delimitedStream =
|
|
130
|
+
delimitedStream = decodedStream.pipe(parser_1);
|
|
127
131
|
generator = utils_1.getDelimitedGenerator({
|
|
128
132
|
readStream: delimitedStream,
|
|
129
133
|
hasHeader: this.hasHeader,
|
|
@@ -53,9 +53,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
53
53
|
};
|
|
54
54
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
55
55
|
exports.MySqlProcessor = void 0;
|
|
56
|
-
var stream_1 = require("stream");
|
|
57
56
|
var database_processor_1 = require("./base/database-processor");
|
|
58
|
-
var promise_1 = __importDefault(require("mysql2/promise"));
|
|
57
|
+
var promise_1 = __importDefault(require("mysql2-stream-fix/promise"));
|
|
59
58
|
var MySqlProcessor = /** @class */ (function (_super) {
|
|
60
59
|
__extends(MySqlProcessor, _super);
|
|
61
60
|
function MySqlProcessor(config) {
|
|
@@ -65,35 +64,13 @@ var MySqlProcessor = /** @class */ (function (_super) {
|
|
|
65
64
|
}
|
|
66
65
|
MySqlProcessor.prototype.process = function (processStream) {
|
|
67
66
|
return __awaiter(this, void 0, void 0, function () {
|
|
68
|
-
var dbConnection, query
|
|
67
|
+
var dbConnection, query;
|
|
69
68
|
return __generator(this, function (_a) {
|
|
70
69
|
switch (_a.label) {
|
|
71
70
|
case 0:
|
|
72
71
|
dbConnection = promise_1.default.createPool(this.connection);
|
|
73
|
-
query = dbConnection.pool.query(this.query);
|
|
74
|
-
|
|
75
|
-
stream._read = function () {
|
|
76
|
-
//dbConnection && dbConnection.resume();
|
|
77
|
-
};
|
|
78
|
-
query.on('result', function (row) {
|
|
79
|
-
if (!stream.push(row)) {
|
|
80
|
-
dbConnection.pause();
|
|
81
|
-
}
|
|
82
|
-
stream.emit('result', row);
|
|
83
|
-
});
|
|
84
|
-
query.on('error', function (err) {
|
|
85
|
-
stream.emit('error', err);
|
|
86
|
-
});
|
|
87
|
-
query.on('end', function () {
|
|
88
|
-
stream.push(null);
|
|
89
|
-
setImmediate(function () {
|
|
90
|
-
stream.emit('close');
|
|
91
|
-
});
|
|
92
|
-
});
|
|
93
|
-
query.on('fields', function (fields) {
|
|
94
|
-
stream.emit('fields', fields);
|
|
95
|
-
});
|
|
96
|
-
return [4 /*yield*/, processStream(stream)];
|
|
72
|
+
query = dbConnection.pool.query(this.query).stream();
|
|
73
|
+
return [4 /*yield*/, processStream(query)];
|
|
97
74
|
case 1:
|
|
98
75
|
_a.sent();
|
|
99
76
|
return [4 /*yield*/, dbConnection.end()];
|
package/lib/types.d.ts
CHANGED
|
@@ -122,6 +122,7 @@ export interface IDelimitedHttpProcessorConfig extends IHttpProcessorConfig {
|
|
|
122
122
|
delimiter?: string;
|
|
123
123
|
hasHeader?: boolean;
|
|
124
124
|
qualifier?: string;
|
|
125
|
+
encoding?: string;
|
|
125
126
|
}
|
|
126
127
|
export interface IDatabaseProcessorConfig extends IProcessorConfig {
|
|
127
128
|
query: string;
|
|
@@ -134,6 +135,7 @@ export interface IDelimitedProcessorConfig extends IDirectoryProcessorConfig {
|
|
|
134
135
|
delimiter?: string;
|
|
135
136
|
hasHeader?: boolean;
|
|
136
137
|
qualifier?: string;
|
|
138
|
+
encoding?: string;
|
|
137
139
|
}
|
|
138
140
|
export interface IMssqlProcessorConfig extends IDatabaseProcessorConfig {
|
|
139
141
|
connection: IMssqlDbConnection;
|
package/lib/utils.js
CHANGED
|
@@ -90,7 +90,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
90
90
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
91
91
|
exports.applyHttpAuthorization = exports.getDelimitedGenerator = exports.removeCircularReferencesFromChunk = exports.getValueFromJSONChunk = exports.closeDbConnection = exports.getDb = void 0;
|
|
92
92
|
var axios_1 = __importDefault(require("axios"));
|
|
93
|
-
var promise_1 = __importDefault(require("mysql2/promise"));
|
|
93
|
+
var promise_1 = __importDefault(require("mysql2-stream-fix/promise"));
|
|
94
94
|
var stream_1 = require("stream");
|
|
95
95
|
var cachedDbConnections = new Map();
|
|
96
96
|
function getCachedDbConnection(databaseConfig) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bellboy",
|
|
3
|
-
"version": "8.
|
|
3
|
+
"version": "8.2.0",
|
|
4
4
|
"description": "Highly performant JavaScript data stream ETL engine.",
|
|
5
5
|
"main": "lib/index",
|
|
6
6
|
"types": "lib/index",
|
|
@@ -54,8 +54,9 @@
|
|
|
54
54
|
"async-mqtt": "^2.2.1",
|
|
55
55
|
"axios": "^0.21.1",
|
|
56
56
|
"csv": "^6.0.5",
|
|
57
|
+
"iconv-lite": "^0.6.3",
|
|
57
58
|
"mssql": "8.1.0",
|
|
58
|
-
"mysql2": "^3.
|
|
59
|
+
"mysql2-stream-fix": "^3.9.1",
|
|
59
60
|
"pg": "8.0.3",
|
|
60
61
|
"pg-promise": "10.5.3",
|
|
61
62
|
"pg-query-stream": "3.0.7",
|
|
@@ -66,4 +67,4 @@
|
|
|
66
67
|
"tail": "^2.0.2",
|
|
67
68
|
"xlstream": "^2.5.3"
|
|
68
69
|
}
|
|
69
|
-
}
|
|
70
|
+
}
|