bellboy 6.2.2 → 6.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -8
- package/lib/destinations/mssql-destination.js +38 -9
- package/lib/processors/delimited-processor.d.ts +0 -1
- package/lib/processors/delimited-processor.js +10 -8
- package/lib/processors/http-processor.d.ts +0 -1
- package/lib/processors/http-processor.js +10 -7
- package/lib/types.d.ts +0 -2
- package/lib/utils.d.ts +2 -5
- package/lib/utils.js +6 -16
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -113,12 +113,12 @@ const job = new bellboy.Job(
|
|
|
113
113
|
|
|
114
114
|
#### Events and event listeners <div id='events'/>
|
|
115
115
|
|
|
116
|
-
Event listeners, which can be registered with `job.on` or `job.onAny`
|
|
116
|
+
Event listeners, which can be registered with `job.on` or `job.onAny` methods, allow you to listen to specific events in the job lifecycle and to interact with them.
|
|
117
117
|
|
|
118
118
|
- Multiple listeners for one event will be executed in the order they were registered.
|
|
119
|
-
- Job always waits code inside listener to complete.
|
|
120
|
-
- Any error thrown inside listener will be ignored and warning message will be printed out.
|
|
121
|
-
- `job.stop()` method can be used inside listener to stop job execution and throw an error if needed.
|
|
119
|
+
- Job always waits for the code inside a listener to complete.
|
|
120
|
+
- Any error thrown inside a listener will be ignored and warning message will be printed out.
|
|
121
|
+
- `job.stop()` method can be used inside a listener to stop job execution and throw an error if needed.
|
|
122
122
|
|
|
123
123
|
```ts
|
|
124
124
|
job.on(
|
|
@@ -340,8 +340,6 @@ For delimited data produces rows described [here](#delimited-produced-row).
|
|
|
340
340
|
If `true`, first row will be processed as a header.
|
|
341
341
|
- **qualifier** `string` `only for delimited`\
|
|
342
342
|
Symbol placed around a field to signify that it is the same field.
|
|
343
|
-
- **trimQualifier** `boolean` `only for delimited`\
|
|
344
|
-
Trim qualifier from produced row item.
|
|
345
343
|
- **jsonPath** `RegExp`\
|
|
346
344
|
Path to the array to be streamed. This option is described in detail inside [JsonProcessor](#json-processor) section.
|
|
347
345
|
- **nextRequest** `async function(header)`\
|
|
@@ -459,8 +457,6 @@ Processes files with delimited data in the directory.
|
|
|
459
457
|
If `true`, first row will be processed as a header.
|
|
460
458
|
- **qualifier** `string` \
|
|
461
459
|
Symbol placed around a field to signify that it is the same field.
|
|
462
|
-
- **trimQualifier** `boolean` \
|
|
463
|
-
Trim qualifier from produced row item.
|
|
464
460
|
|
|
465
461
|
#### Produced row <div id='delimited-produced-row'/>
|
|
466
462
|
|
|
@@ -12,6 +12,17 @@ var __extends = (this && this.__extends) || (function () {
|
|
|
12
12
|
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
|
|
13
13
|
};
|
|
14
14
|
})();
|
|
15
|
+
var __assign = (this && this.__assign) || function () {
|
|
16
|
+
__assign = Object.assign || function(t) {
|
|
17
|
+
for (var s, i = 1, n = arguments.length; i < n; i++) {
|
|
18
|
+
s = arguments[i];
|
|
19
|
+
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
|
|
20
|
+
t[p] = s[p];
|
|
21
|
+
}
|
|
22
|
+
return t;
|
|
23
|
+
};
|
|
24
|
+
return __assign.apply(this, arguments);
|
|
25
|
+
};
|
|
15
26
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
16
27
|
if (k2 === undefined) k2 = k;
|
|
17
28
|
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
|
|
@@ -69,7 +80,6 @@ var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
|
69
80
|
};
|
|
70
81
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
71
82
|
exports.MssqlDestination = void 0;
|
|
72
|
-
var utils_1 = require("../utils");
|
|
73
83
|
var database_destination_1 = require("./base/database-destination");
|
|
74
84
|
var MssqlDestination = /** @class */ (function (_super) {
|
|
75
85
|
__extends(MssqlDestination, _super);
|
|
@@ -80,7 +90,7 @@ var MssqlDestination = /** @class */ (function (_super) {
|
|
|
80
90
|
}
|
|
81
91
|
MssqlDestination.prototype.loadBatch = function (data) {
|
|
82
92
|
return __awaiter(this, void 0, void 0, function () {
|
|
83
|
-
var sql, _a, db, query, table, columns,
|
|
93
|
+
var sql, _a, pool, db, query, table, columns, _i, data_1, x, args, ii, transaction, err_1;
|
|
84
94
|
var _b;
|
|
85
95
|
return __generator(this, function (_c) {
|
|
86
96
|
switch (_c.label) {
|
|
@@ -96,7 +106,8 @@ var MssqlDestination = /** @class */ (function (_super) {
|
|
|
96
106
|
_c.label = 4;
|
|
97
107
|
case 4:
|
|
98
108
|
sql = _a;
|
|
99
|
-
|
|
109
|
+
pool = new sql.ConnectionPool(__assign({}, this.connection));
|
|
110
|
+
return [4 /*yield*/, pool.connect()];
|
|
100
111
|
case 5:
|
|
101
112
|
db = _c.sent();
|
|
102
113
|
return [4 /*yield*/, db.request().query("SELECT TOP(0) * FROM " + this.table)];
|
|
@@ -104,10 +115,6 @@ var MssqlDestination = /** @class */ (function (_super) {
|
|
|
104
115
|
query = _c.sent();
|
|
105
116
|
table = query.recordset.toTable(this.table);
|
|
106
117
|
columns = table.columns.map(function (x) { return x.name; });
|
|
107
|
-
transaction = new sql.Transaction(db);
|
|
108
|
-
return [4 /*yield*/, transaction.begin()];
|
|
109
|
-
case 7:
|
|
110
|
-
_c.sent();
|
|
111
118
|
for (_i = 0, data_1 = data; _i < data_1.length; _i++) {
|
|
112
119
|
x = data_1[_i];
|
|
113
120
|
args = [];
|
|
@@ -116,11 +123,33 @@ var MssqlDestination = /** @class */ (function (_super) {
|
|
|
116
123
|
}
|
|
117
124
|
(_b = table.rows).add.apply(_b, args);
|
|
118
125
|
}
|
|
119
|
-
|
|
126
|
+
transaction = new sql.Transaction(db);
|
|
127
|
+
_c.label = 7;
|
|
128
|
+
case 7:
|
|
129
|
+
_c.trys.push([7, 11, , 15]);
|
|
130
|
+
return [4 /*yield*/, transaction.begin()];
|
|
120
131
|
case 8:
|
|
121
132
|
_c.sent();
|
|
122
|
-
return [4 /*yield*/, transaction.
|
|
133
|
+
return [4 /*yield*/, transaction.request().bulk(table)];
|
|
123
134
|
case 9:
|
|
135
|
+
_c.sent();
|
|
136
|
+
return [4 /*yield*/, transaction.commit()];
|
|
137
|
+
case 10:
|
|
138
|
+
_c.sent();
|
|
139
|
+
return [3 /*break*/, 15];
|
|
140
|
+
case 11:
|
|
141
|
+
err_1 = _c.sent();
|
|
142
|
+
if (!!transaction._aborted) return [3 /*break*/, 13];
|
|
143
|
+
return [4 /*yield*/, transaction.rollback()];
|
|
144
|
+
case 12:
|
|
145
|
+
_c.sent();
|
|
146
|
+
_c.label = 13;
|
|
147
|
+
case 13: return [4 /*yield*/, pool.close()];
|
|
148
|
+
case 14:
|
|
149
|
+
_c.sent();
|
|
150
|
+
throw err_1;
|
|
151
|
+
case 15: return [4 /*yield*/, pool.close()];
|
|
152
|
+
case 16:
|
|
124
153
|
_c.sent();
|
|
125
154
|
return [2 /*return*/];
|
|
126
155
|
}
|
|
@@ -5,7 +5,6 @@ export declare class DelimitedProcessor extends DirectoryProcessor {
|
|
|
5
5
|
protected hasHeader: boolean;
|
|
6
6
|
protected delimiter?: string;
|
|
7
7
|
protected qualifier?: string;
|
|
8
|
-
protected trimQualifier: boolean;
|
|
9
8
|
constructor(config: IDelimitedProcessorConfig);
|
|
10
9
|
process(processStream: processStream): Promise<void>;
|
|
11
10
|
}
|
|
@@ -53,29 +53,27 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
53
53
|
};
|
|
54
54
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
55
55
|
exports.DelimitedProcessor = void 0;
|
|
56
|
+
var csv_parse_1 = require("csv-parse");
|
|
56
57
|
var fs_1 = __importDefault(require("fs"));
|
|
57
58
|
var path_1 = __importDefault(require("path"));
|
|
58
59
|
var utils_1 = require("../utils");
|
|
59
60
|
var directory_processor_1 = require("./base/directory-processor");
|
|
60
|
-
var split2 = require('split2');
|
|
61
61
|
var DelimitedProcessor = /** @class */ (function (_super) {
|
|
62
62
|
__extends(DelimitedProcessor, _super);
|
|
63
63
|
function DelimitedProcessor(config) {
|
|
64
64
|
var _this = _super.call(this, config) || this;
|
|
65
|
-
_this.trimQualifier = false;
|
|
66
65
|
if (!config.rowSeparator) {
|
|
67
66
|
throw new Error('No rowSeparator specified.');
|
|
68
67
|
}
|
|
69
68
|
_this.rowSeparator = config.rowSeparator;
|
|
70
69
|
_this.hasHeader = !!config.hasHeader;
|
|
71
|
-
_this.trimQualifier = !!config.trimQualifier;
|
|
72
70
|
_this.delimiter = config.delimiter;
|
|
73
71
|
_this.qualifier = config.qualifier;
|
|
74
72
|
return _this;
|
|
75
73
|
}
|
|
76
74
|
DelimitedProcessor.prototype.process = function (processStream) {
|
|
77
75
|
return __awaiter(this, void 0, void 0, function () {
|
|
78
|
-
var _i, _a, file, filePath, fileReadStream, readStream, generator;
|
|
76
|
+
var _i, _a, file, filePath, fileReadStream, parser, readStream, generator;
|
|
79
77
|
return __generator(this, function (_b) {
|
|
80
78
|
switch (_b.label) {
|
|
81
79
|
case 0:
|
|
@@ -86,13 +84,17 @@ var DelimitedProcessor = /** @class */ (function (_super) {
|
|
|
86
84
|
file = _a[_i];
|
|
87
85
|
filePath = path_1.default.join(this.path, file);
|
|
88
86
|
fileReadStream = fs_1.default.createReadStream(filePath);
|
|
89
|
-
|
|
90
|
-
|
|
87
|
+
parser = csv_parse_1.parse({
|
|
88
|
+
quote: this.qualifier,
|
|
91
89
|
delimiter: this.delimiter,
|
|
90
|
+
record_delimiter: this.rowSeparator,
|
|
91
|
+
raw: true,
|
|
92
|
+
relax_quotes: true,
|
|
93
|
+
});
|
|
94
|
+
readStream = fileReadStream.pipe(parser);
|
|
95
|
+
generator = utils_1.getDelimitedGenerator({
|
|
92
96
|
hasHeader: this.hasHeader,
|
|
93
|
-
qualifier: this.qualifier,
|
|
94
97
|
readStream: readStream,
|
|
95
|
-
trimQualifier: this.trimQualifier,
|
|
96
98
|
});
|
|
97
99
|
return [4 /*yield*/, processStream(generator(), file, filePath)];
|
|
98
100
|
case 2:
|
|
@@ -10,7 +10,6 @@ export declare class HttpProcessor extends Processor {
|
|
|
10
10
|
protected hasHeader: boolean;
|
|
11
11
|
protected delimiter?: string;
|
|
12
12
|
protected qualifier?: string;
|
|
13
|
-
protected trimQualifier: boolean;
|
|
14
13
|
constructor(config: IJsonHttpProcessorConfig | IDelimitedHttpProcessorConfig);
|
|
15
14
|
protected processHttpData(processStream: processStream, options: AxiosRequestConfig): Promise<void>;
|
|
16
15
|
process(processStream: processStream): Promise<void>;
|
|
@@ -65,6 +65,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
65
65
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
66
66
|
exports.HttpProcessor = void 0;
|
|
67
67
|
var axios_1 = __importDefault(require("axios"));
|
|
68
|
+
var csv_parse_1 = require("csv-parse");
|
|
68
69
|
var Pick_1 = require("stream-json/filters/Pick");
|
|
69
70
|
var Parser_1 = require("stream-json/Parser");
|
|
70
71
|
var StreamValues_1 = require("stream-json/streamers/StreamValues");
|
|
@@ -76,7 +77,6 @@ var HttpProcessor = /** @class */ (function (_super) {
|
|
|
76
77
|
function HttpProcessor(config) {
|
|
77
78
|
var _this = _super.call(this, config) || this;
|
|
78
79
|
_this.hasHeader = false;
|
|
79
|
-
_this.trimQualifier = false;
|
|
80
80
|
if (!config.connection) {
|
|
81
81
|
throw new Error("No connection specified.");
|
|
82
82
|
}
|
|
@@ -88,7 +88,6 @@ var HttpProcessor = /** @class */ (function (_super) {
|
|
|
88
88
|
}
|
|
89
89
|
_this.rowSeparator = config.rowSeparator;
|
|
90
90
|
_this.hasHeader = !!config.hasHeader;
|
|
91
|
-
_this.trimQualifier = !!config.trimQualifier;
|
|
92
91
|
_this.delimiter = config.delimiter;
|
|
93
92
|
_this.qualifier = config.qualifier;
|
|
94
93
|
}
|
|
@@ -100,7 +99,7 @@ var HttpProcessor = /** @class */ (function (_super) {
|
|
|
100
99
|
}
|
|
101
100
|
HttpProcessor.prototype.processHttpData = function (processStream, options) {
|
|
102
101
|
return __awaiter(this, void 0, void 0, function () {
|
|
103
|
-
var requestStream, delimitedStream, generator, requestStream, jsonStream, nextOptions;
|
|
102
|
+
var requestStream, parser_1, delimitedStream, generator, requestStream, jsonStream, nextOptions;
|
|
104
103
|
return __generator(this, function (_a) {
|
|
105
104
|
switch (_a.label) {
|
|
106
105
|
case 0:
|
|
@@ -108,13 +107,17 @@ var HttpProcessor = /** @class */ (function (_super) {
|
|
|
108
107
|
return [4 /*yield*/, axios_1.default(__assign(__assign({}, options), { responseType: 'stream' }))];
|
|
109
108
|
case 1:
|
|
110
109
|
requestStream = _a.sent();
|
|
111
|
-
|
|
110
|
+
parser_1 = csv_parse_1.parse({
|
|
111
|
+
quote: this.qualifier,
|
|
112
|
+
delimiter: this.delimiter,
|
|
113
|
+
record_delimiter: this.rowSeparator,
|
|
114
|
+
raw: true,
|
|
115
|
+
relax_quotes: true,
|
|
116
|
+
});
|
|
117
|
+
delimitedStream = requestStream.data.pipe(parser_1);
|
|
112
118
|
generator = utils_1.getDelimitedGenerator({
|
|
113
119
|
readStream: delimitedStream,
|
|
114
|
-
delimiter: this.delimiter,
|
|
115
120
|
hasHeader: this.hasHeader,
|
|
116
|
-
qualifier: this.qualifier,
|
|
117
|
-
trimQualifier: this.trimQualifier,
|
|
118
121
|
});
|
|
119
122
|
return [4 /*yield*/, processStream(generator())];
|
|
120
123
|
case 2:
|
package/lib/types.d.ts
CHANGED
|
@@ -87,7 +87,6 @@ export interface IDelimitedHttpProcessorConfig extends IHttpProcessorConfig {
|
|
|
87
87
|
rowSeparator: string;
|
|
88
88
|
delimiter?: string;
|
|
89
89
|
hasHeader?: boolean;
|
|
90
|
-
trimQualifier?: boolean;
|
|
91
90
|
qualifier?: string;
|
|
92
91
|
}
|
|
93
92
|
export interface IDatabaseProcessorConfig extends IProcessorConfig {
|
|
@@ -101,7 +100,6 @@ export interface IDelimitedProcessorConfig extends IDirectoryProcessorConfig {
|
|
|
101
100
|
rowSeparator: string;
|
|
102
101
|
delimiter?: string;
|
|
103
102
|
hasHeader?: boolean;
|
|
104
|
-
trimQualifier?: boolean;
|
|
105
103
|
qualifier?: string;
|
|
106
104
|
}
|
|
107
105
|
export interface IMssqlProcessorConfig extends IDatabaseProcessorConfig {
|
package/lib/utils.d.ts
CHANGED
|
@@ -4,15 +4,12 @@ import { DbTypes, IDbConnection } from './types';
|
|
|
4
4
|
export declare function getDb(databaseConfig: IDbConnection, dbType: DbTypes): Promise<any>;
|
|
5
5
|
export declare function closeDbConnection(databaseConfig: IDbConnection): Promise<void>;
|
|
6
6
|
export declare function getValueFromJSONChunk(): Transform;
|
|
7
|
-
export declare function getDelimitedGenerator({ readStream,
|
|
7
|
+
export declare function getDelimitedGenerator({ readStream, hasHeader, }: {
|
|
8
8
|
readStream: any;
|
|
9
|
-
delimiter?: string;
|
|
10
|
-
qualifier?: string;
|
|
11
9
|
hasHeader: boolean;
|
|
12
|
-
trimQualifier: boolean;
|
|
13
10
|
}): () => AsyncGenerator<{
|
|
14
11
|
header: string[];
|
|
15
12
|
arr: string[];
|
|
16
13
|
obj: any;
|
|
17
|
-
row:
|
|
14
|
+
row: string;
|
|
18
15
|
}, void, unknown>;
|
package/lib/utils.js
CHANGED
|
@@ -169,29 +169,19 @@ function getValueFromJSONChunk() {
|
|
|
169
169
|
exports.getValueFromJSONChunk = getValueFromJSONChunk;
|
|
170
170
|
;
|
|
171
171
|
function getDelimitedGenerator(_a) {
|
|
172
|
-
var readStream = _a.readStream,
|
|
172
|
+
var readStream = _a.readStream, hasHeader = _a.hasHeader;
|
|
173
173
|
var header = [];
|
|
174
|
-
var
|
|
175
|
-
|
|
176
|
-
var arr = [];
|
|
177
|
-
if (qualifier) {
|
|
178
|
-
arr = row.split(splitRegExp);
|
|
179
|
-
if (trimQualifier) {
|
|
180
|
-
arr = arr.map(function (x) { return x[0] === qualifier && x[x.length - 1] === qualifier ? x.slice(1, -1) : x; });
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
else {
|
|
184
|
-
arr = row.split(delimiter);
|
|
185
|
-
}
|
|
174
|
+
var processRow = function (_a) {
|
|
175
|
+
var raw = _a.raw, record = _a.record;
|
|
186
176
|
if (hasHeader && !header.length) {
|
|
187
|
-
header =
|
|
177
|
+
header = record.map(function (x) { return x.trim(); });
|
|
188
178
|
}
|
|
189
179
|
else {
|
|
190
180
|
var obj = void 0;
|
|
191
181
|
if (header.length) {
|
|
192
|
-
obj = Object.fromEntries(header.map(function (x, i) { return [x,
|
|
182
|
+
obj = Object.fromEntries(header.map(function (x, i) { return [x, record[i]]; }));
|
|
193
183
|
}
|
|
194
|
-
return { header: header, arr:
|
|
184
|
+
return { header: header, arr: record, obj: obj, row: raw };
|
|
195
185
|
}
|
|
196
186
|
};
|
|
197
187
|
var generator = function () {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bellboy",
|
|
3
|
-
"version": "6.
|
|
3
|
+
"version": "6.3.2",
|
|
4
4
|
"description": "Highly performant JavaScript data stream ETL engine.",
|
|
5
5
|
"main": "lib/index",
|
|
6
6
|
"types": "lib/index",
|
|
@@ -47,6 +47,7 @@
|
|
|
47
47
|
"dependencies": {
|
|
48
48
|
"async-mqtt": "^2.2.1",
|
|
49
49
|
"axios": "^0.21.1",
|
|
50
|
+
"csv": "^6.0.5",
|
|
50
51
|
"mssql": "^6.3.1",
|
|
51
52
|
"pg": "8.0.3",
|
|
52
53
|
"pg-promise": "10.5.3",
|