bellboy 4.2.0 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +344 -270
- package/lib/destinations/http-destination.d.ts +2 -3
- package/lib/destinations/http-destination.js +2 -2
- package/lib/processors/delimited-processor.d.ts +1 -0
- package/lib/processors/delimited-processor.js +16 -104
- package/lib/processors/excel-processor.d.ts +1 -0
- package/lib/processors/excel-processor.js +2 -0
- package/lib/processors/http-processor.d.ts +7 -3
- package/lib/processors/http-processor.js +51 -22
- package/lib/types.d.ts +10 -5
- package/lib/utils.d.ts +12 -0
- package/lib/utils.js +93 -1
- package/package.json +4 -4
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { AxiosRequestConfig } from 'axios';
|
|
2
2
|
import { IHttpDestinationConfig } from '../types';
|
|
3
3
|
import { Destination } from './base/destination';
|
|
4
|
-
import { UriOptions, UrlOptions } from 'request';
|
|
5
4
|
export declare class HttpDestination extends Destination {
|
|
6
|
-
protected request:
|
|
5
|
+
protected request: AxiosRequestConfig;
|
|
7
6
|
constructor(config: IHttpDestinationConfig);
|
|
8
7
|
loadBatch(data: any[]): Promise<void>;
|
|
9
8
|
}
|
|
@@ -64,7 +64,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
64
64
|
};
|
|
65
65
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
66
66
|
exports.HttpDestination = void 0;
|
|
67
|
-
var
|
|
67
|
+
var axios_1 = __importDefault(require("axios"));
|
|
68
68
|
var destination_1 = require("./base/destination");
|
|
69
69
|
var HttpDestination = /** @class */ (function (_super) {
|
|
70
70
|
__extends(HttpDestination, _super);
|
|
@@ -83,7 +83,7 @@ var HttpDestination = /** @class */ (function (_super) {
|
|
|
83
83
|
_a.label = 1;
|
|
84
84
|
case 1:
|
|
85
85
|
if (!(i < data.length)) return [3 /*break*/, 4];
|
|
86
|
-
return [4 /*yield*/,
|
|
86
|
+
return [4 /*yield*/, axios_1.default(__assign(__assign({}, this.request), { data: data[i] }))];
|
|
87
87
|
case 2:
|
|
88
88
|
_a.sent();
|
|
89
89
|
_a.label = 3;
|
|
@@ -5,6 +5,7 @@ export declare class DelimitedProcessor extends DirectoryProcessor {
|
|
|
5
5
|
protected hasHeader: boolean;
|
|
6
6
|
protected delimiter?: string;
|
|
7
7
|
protected qualifier?: string;
|
|
8
|
+
protected trimQualifier: boolean;
|
|
8
9
|
constructor(config: IDelimitedProcessorConfig);
|
|
9
10
|
process(processStream: processStream): Promise<void>;
|
|
10
11
|
}
|
|
@@ -48,25 +48,6 @@ var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
|
48
48
|
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
49
49
|
}
|
|
50
50
|
};
|
|
51
|
-
var __asyncValues = (this && this.__asyncValues) || function (o) {
|
|
52
|
-
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
|
|
53
|
-
var m = o[Symbol.asyncIterator], i;
|
|
54
|
-
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
|
|
55
|
-
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
|
|
56
|
-
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
|
|
57
|
-
};
|
|
58
|
-
var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); }
|
|
59
|
-
var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
|
|
60
|
-
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
|
|
61
|
-
var g = generator.apply(thisArg, _arguments || []), i, q = [];
|
|
62
|
-
return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i;
|
|
63
|
-
function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; }
|
|
64
|
-
function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
|
|
65
|
-
function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
|
|
66
|
-
function fulfill(value) { resume("next", value); }
|
|
67
|
-
function reject(value) { resume("throw", value); }
|
|
68
|
-
function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
|
|
69
|
-
};
|
|
70
51
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
71
52
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
72
53
|
};
|
|
@@ -74,118 +55,49 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
74
55
|
exports.DelimitedProcessor = void 0;
|
|
75
56
|
var fs_1 = __importDefault(require("fs"));
|
|
76
57
|
var path_1 = __importDefault(require("path"));
|
|
58
|
+
var utils_1 = require("../utils");
|
|
77
59
|
var directory_processor_1 = require("./base/directory-processor");
|
|
78
60
|
var split2 = require('split2');
|
|
79
61
|
var DelimitedProcessor = /** @class */ (function (_super) {
|
|
80
62
|
__extends(DelimitedProcessor, _super);
|
|
81
63
|
function DelimitedProcessor(config) {
|
|
82
64
|
var _this = _super.call(this, config) || this;
|
|
65
|
+
_this.trimQualifier = false;
|
|
83
66
|
if (!config.rowSeparator) {
|
|
84
67
|
throw new Error('No rowSeparator specified.');
|
|
85
68
|
}
|
|
86
69
|
_this.rowSeparator = config.rowSeparator;
|
|
87
70
|
_this.hasHeader = !!config.hasHeader;
|
|
71
|
+
_this.trimQualifier = !!config.trimQualifier;
|
|
88
72
|
_this.delimiter = config.delimiter;
|
|
89
73
|
_this.qualifier = config.qualifier;
|
|
90
74
|
return _this;
|
|
91
75
|
}
|
|
92
76
|
DelimitedProcessor.prototype.process = function (processStream) {
|
|
93
77
|
return __awaiter(this, void 0, void 0, function () {
|
|
94
|
-
var
|
|
95
|
-
var _this = this;
|
|
78
|
+
var _i, _a, file, filePath, fileReadStream, readStream, generator;
|
|
96
79
|
return __generator(this, function (_b) {
|
|
97
80
|
switch (_b.label) {
|
|
98
81
|
case 0:
|
|
99
|
-
_loop_1 = function (file) {
|
|
100
|
-
var filePath, fileReadStream, readStream, header, splitRegExp, processRow, generator;
|
|
101
|
-
return __generator(this, function (_a) {
|
|
102
|
-
switch (_a.label) {
|
|
103
|
-
case 0:
|
|
104
|
-
filePath = path_1.default.join(this_1.path, file);
|
|
105
|
-
fileReadStream = fs_1.default.createReadStream(filePath);
|
|
106
|
-
readStream = fileReadStream.pipe(split2(this_1.rowSeparator));
|
|
107
|
-
header = [];
|
|
108
|
-
splitRegExp = new RegExp(this_1.delimiter + "(?=(?:(?:[^" + this_1.qualifier + "]*" + this_1.qualifier + "){2})*[^" + this_1.qualifier + "]*$)");
|
|
109
|
-
processRow = function (row) {
|
|
110
|
-
var arr = [];
|
|
111
|
-
if (_this.qualifier) {
|
|
112
|
-
arr = row.split(splitRegExp);
|
|
113
|
-
}
|
|
114
|
-
else {
|
|
115
|
-
arr = row.split(_this.delimiter);
|
|
116
|
-
}
|
|
117
|
-
if (_this.hasHeader && !header.length) {
|
|
118
|
-
header = arr.map(function (x) { return x.trim(); });
|
|
119
|
-
}
|
|
120
|
-
else {
|
|
121
|
-
var obj = void 0;
|
|
122
|
-
if (header.length) {
|
|
123
|
-
obj = Object.fromEntries(header.map(function (x, i) { return [x, arr[i]]; }));
|
|
124
|
-
}
|
|
125
|
-
return { header: header, arr: arr, obj: obj, row: row };
|
|
126
|
-
}
|
|
127
|
-
};
|
|
128
|
-
generator = function () {
|
|
129
|
-
return __asyncGenerator(this, arguments, function () {
|
|
130
|
-
var readStream_1, readStream_1_1, row, result, e_1_1;
|
|
131
|
-
var e_1, _a;
|
|
132
|
-
return __generator(this, function (_b) {
|
|
133
|
-
switch (_b.label) {
|
|
134
|
-
case 0:
|
|
135
|
-
_b.trys.push([0, 7, 8, 13]);
|
|
136
|
-
readStream_1 = __asyncValues(readStream);
|
|
137
|
-
_b.label = 1;
|
|
138
|
-
case 1: return [4 /*yield*/, __await(readStream_1.next())];
|
|
139
|
-
case 2:
|
|
140
|
-
if (!(readStream_1_1 = _b.sent(), !readStream_1_1.done)) return [3 /*break*/, 6];
|
|
141
|
-
row = readStream_1_1.value;
|
|
142
|
-
result = processRow(row);
|
|
143
|
-
if (!result) return [3 /*break*/, 5];
|
|
144
|
-
return [4 /*yield*/, __await(result)];
|
|
145
|
-
case 3: return [4 /*yield*/, _b.sent()];
|
|
146
|
-
case 4:
|
|
147
|
-
_b.sent();
|
|
148
|
-
_b.label = 5;
|
|
149
|
-
case 5: return [3 /*break*/, 1];
|
|
150
|
-
case 6: return [3 /*break*/, 13];
|
|
151
|
-
case 7:
|
|
152
|
-
e_1_1 = _b.sent();
|
|
153
|
-
e_1 = { error: e_1_1 };
|
|
154
|
-
return [3 /*break*/, 13];
|
|
155
|
-
case 8:
|
|
156
|
-
_b.trys.push([8, , 11, 12]);
|
|
157
|
-
if (!(readStream_1_1 && !readStream_1_1.done && (_a = readStream_1.return))) return [3 /*break*/, 10];
|
|
158
|
-
return [4 /*yield*/, __await(_a.call(readStream_1))];
|
|
159
|
-
case 9:
|
|
160
|
-
_b.sent();
|
|
161
|
-
_b.label = 10;
|
|
162
|
-
case 10: return [3 /*break*/, 12];
|
|
163
|
-
case 11:
|
|
164
|
-
if (e_1) throw e_1.error;
|
|
165
|
-
return [7 /*endfinally*/];
|
|
166
|
-
case 12: return [7 /*endfinally*/];
|
|
167
|
-
case 13: return [2 /*return*/];
|
|
168
|
-
}
|
|
169
|
-
});
|
|
170
|
-
});
|
|
171
|
-
};
|
|
172
|
-
return [4 /*yield*/, processStream(generator(), file, filePath)];
|
|
173
|
-
case 1:
|
|
174
|
-
_a.sent();
|
|
175
|
-
fileReadStream.destroy();
|
|
176
|
-
return [2 /*return*/];
|
|
177
|
-
}
|
|
178
|
-
});
|
|
179
|
-
};
|
|
180
|
-
this_1 = this;
|
|
181
82
|
_i = 0, _a = this.files;
|
|
182
83
|
_b.label = 1;
|
|
183
84
|
case 1:
|
|
184
85
|
if (!(_i < _a.length)) return [3 /*break*/, 4];
|
|
185
86
|
file = _a[_i];
|
|
186
|
-
|
|
87
|
+
filePath = path_1.default.join(this.path, file);
|
|
88
|
+
fileReadStream = fs_1.default.createReadStream(filePath);
|
|
89
|
+
readStream = fileReadStream.pipe(split2(this.rowSeparator));
|
|
90
|
+
generator = utils_1.getDelimitedGenerator({
|
|
91
|
+
delimiter: this.delimiter,
|
|
92
|
+
hasHeader: this.hasHeader,
|
|
93
|
+
qualifier: this.qualifier,
|
|
94
|
+
readStream: readStream,
|
|
95
|
+
trimQualifier: this.trimQualifier,
|
|
96
|
+
});
|
|
97
|
+
return [4 /*yield*/, processStream(generator(), file, filePath)];
|
|
187
98
|
case 2:
|
|
188
99
|
_b.sent();
|
|
100
|
+
fileReadStream.destroy();
|
|
189
101
|
_b.label = 3;
|
|
190
102
|
case 3:
|
|
191
103
|
_i++;
|
|
@@ -3,6 +3,7 @@ import { DirectoryProcessor } from './base/directory-processor';
|
|
|
3
3
|
export declare class ExcelProcessor extends DirectoryProcessor {
|
|
4
4
|
protected hasHeader: boolean | number | undefined;
|
|
5
5
|
protected ignoreEmpty: boolean;
|
|
6
|
+
protected encoding?: string;
|
|
6
7
|
protected fillMergedCells: boolean;
|
|
7
8
|
protected sheets: (string | number)[] | sheetGetter;
|
|
8
9
|
constructor(config: IExcelProcessorConfig);
|
|
@@ -60,6 +60,7 @@ var ExcelProcessor = /** @class */ (function (_super) {
|
|
|
60
60
|
__extends(ExcelProcessor, _super);
|
|
61
61
|
function ExcelProcessor(config) {
|
|
62
62
|
var _this = _super.call(this, config) || this;
|
|
63
|
+
_this.encoding = config.encoding;
|
|
63
64
|
_this.hasHeader = config.hasHeader;
|
|
64
65
|
_this.ignoreEmpty = config.ignoreEmpty === false ? false : true;
|
|
65
66
|
_this.fillMergedCells = !!config.fillMergedCells;
|
|
@@ -106,6 +107,7 @@ var ExcelProcessor = /** @class */ (function (_super) {
|
|
|
106
107
|
withHeader: this.hasHeader,
|
|
107
108
|
ignoreEmpty: this.ignoreEmpty,
|
|
108
109
|
fillMergedCells: this.fillMergedCells,
|
|
110
|
+
encoding: this.encoding,
|
|
109
111
|
})];
|
|
110
112
|
case 7:
|
|
111
113
|
readStream = _b.sent();
|
|
@@ -1,13 +1,17 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { AxiosRequestConfig } from 'axios';
|
|
2
2
|
import { IDelimitedHttpProcessorConfig, IJsonHttpProcessorConfig, processStream } from '../types';
|
|
3
3
|
import { Processor } from './base/processor';
|
|
4
4
|
export declare class HttpProcessor extends Processor {
|
|
5
|
-
protected connection:
|
|
5
|
+
protected connection: AxiosRequestConfig;
|
|
6
6
|
protected nextRequest: (() => Promise<any>) | undefined;
|
|
7
7
|
protected jsonPath: RegExp | undefined;
|
|
8
8
|
protected rowSeparator: string | undefined;
|
|
9
9
|
protected dataFormat: 'json' | 'delimited';
|
|
10
|
+
protected hasHeader: boolean;
|
|
11
|
+
protected delimiter?: string;
|
|
12
|
+
protected qualifier?: string;
|
|
13
|
+
protected trimQualifier: boolean;
|
|
10
14
|
constructor(config: IJsonHttpProcessorConfig | IDelimitedHttpProcessorConfig);
|
|
11
|
-
protected processHttpData(processStream: processStream, options:
|
|
15
|
+
protected processHttpData(processStream: processStream, options: AxiosRequestConfig): Promise<void>;
|
|
12
16
|
process(processStream: processStream): Promise<void>;
|
|
13
17
|
}
|
|
@@ -12,6 +12,17 @@ var __extends = (this && this.__extends) || (function () {
|
|
|
12
12
|
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
|
|
13
13
|
};
|
|
14
14
|
})();
|
|
15
|
+
var __assign = (this && this.__assign) || function () {
|
|
16
|
+
__assign = Object.assign || function(t) {
|
|
17
|
+
for (var s, i = 1, n = arguments.length; i < n; i++) {
|
|
18
|
+
s = arguments[i];
|
|
19
|
+
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
|
|
20
|
+
t[p] = s[p];
|
|
21
|
+
}
|
|
22
|
+
return t;
|
|
23
|
+
};
|
|
24
|
+
return __assign.apply(this, arguments);
|
|
25
|
+
};
|
|
15
26
|
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
16
27
|
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
17
28
|
return new (P || (P = Promise))(function (resolve, reject) {
|
|
@@ -48,9 +59,12 @@ var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
|
48
59
|
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
49
60
|
}
|
|
50
61
|
};
|
|
62
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
63
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
64
|
+
};
|
|
51
65
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
52
66
|
exports.HttpProcessor = void 0;
|
|
53
|
-
var
|
|
67
|
+
var axios_1 = __importDefault(require("axios"));
|
|
54
68
|
var Pick_1 = require("stream-json/filters/Pick");
|
|
55
69
|
var Parser_1 = require("stream-json/Parser");
|
|
56
70
|
var StreamValues_1 = require("stream-json/streamers/StreamValues");
|
|
@@ -61,6 +75,8 @@ var HttpProcessor = /** @class */ (function (_super) {
|
|
|
61
75
|
__extends(HttpProcessor, _super);
|
|
62
76
|
function HttpProcessor(config) {
|
|
63
77
|
var _this = _super.call(this, config) || this;
|
|
78
|
+
_this.hasHeader = false;
|
|
79
|
+
_this.trimQualifier = false;
|
|
64
80
|
if (!config.connection) {
|
|
65
81
|
throw new Error("No connection specified.");
|
|
66
82
|
}
|
|
@@ -71,6 +87,10 @@ var HttpProcessor = /** @class */ (function (_super) {
|
|
|
71
87
|
throw new Error('No rowSeparator specified.');
|
|
72
88
|
}
|
|
73
89
|
_this.rowSeparator = config.rowSeparator;
|
|
90
|
+
_this.hasHeader = !!config.hasHeader;
|
|
91
|
+
_this.trimQualifier = !!config.trimQualifier;
|
|
92
|
+
_this.delimiter = config.delimiter;
|
|
93
|
+
_this.qualifier = config.qualifier;
|
|
74
94
|
}
|
|
75
95
|
else if (config.dataFormat === 'json') {
|
|
76
96
|
_this.jsonPath = config.jsonPath;
|
|
@@ -80,42 +100,51 @@ var HttpProcessor = /** @class */ (function (_super) {
|
|
|
80
100
|
}
|
|
81
101
|
HttpProcessor.prototype.processHttpData = function (processStream, options) {
|
|
82
102
|
return __awaiter(this, void 0, void 0, function () {
|
|
83
|
-
var requestStream, delimitedStream, requestStream, jsonStream, nextOptions;
|
|
103
|
+
var requestStream, delimitedStream, generator, requestStream, jsonStream, nextOptions;
|
|
84
104
|
return __generator(this, function (_a) {
|
|
85
105
|
switch (_a.label) {
|
|
86
106
|
case 0:
|
|
87
|
-
if (!(this.dataFormat === 'delimited')) return [3 /*break*/,
|
|
88
|
-
|
|
89
|
-
delimitedStream = requestStream.pipe(split2(this.rowSeparator));
|
|
90
|
-
return [4 /*yield*/, processStream(delimitedStream)];
|
|
107
|
+
if (!(this.dataFormat === 'delimited')) return [3 /*break*/, 3];
|
|
108
|
+
return [4 /*yield*/, axios_1.default(__assign(__assign({}, options), { responseType: 'stream' }))];
|
|
91
109
|
case 1:
|
|
92
|
-
_a.sent();
|
|
93
|
-
requestStream.
|
|
94
|
-
|
|
110
|
+
requestStream = _a.sent();
|
|
111
|
+
delimitedStream = requestStream.data.pipe(split2(this.rowSeparator));
|
|
112
|
+
generator = utils_1.getDelimitedGenerator({
|
|
113
|
+
readStream: delimitedStream,
|
|
114
|
+
delimiter: this.delimiter,
|
|
115
|
+
hasHeader: this.hasHeader,
|
|
116
|
+
qualifier: this.qualifier,
|
|
117
|
+
trimQualifier: this.trimQualifier,
|
|
118
|
+
});
|
|
119
|
+
return [4 /*yield*/, processStream(generator())];
|
|
95
120
|
case 2:
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
121
|
+
_a.sent();
|
|
122
|
+
return [3 /*break*/, 6];
|
|
123
|
+
case 3:
|
|
124
|
+
if (!(this.dataFormat === 'json')) return [3 /*break*/, 6];
|
|
125
|
+
return [4 /*yield*/, axios_1.default(__assign(__assign({}, options), { responseType: 'stream' }))];
|
|
126
|
+
case 4:
|
|
127
|
+
requestStream = _a.sent();
|
|
128
|
+
jsonStream = requestStream.data
|
|
99
129
|
.pipe(Parser_1.parser())
|
|
100
130
|
.pipe(Pick_1.pick({ filter: this.jsonPath || '' }))
|
|
101
131
|
.pipe(StreamValues_1.streamValues())
|
|
102
132
|
.pipe(utils_1.getValueFromJSONChunk());
|
|
103
133
|
return [4 /*yield*/, processStream(jsonStream)];
|
|
104
|
-
case
|
|
134
|
+
case 5:
|
|
105
135
|
_a.sent();
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
if (!this.nextRequest) return [3 /*break*/, 7];
|
|
136
|
+
_a.label = 6;
|
|
137
|
+
case 6:
|
|
138
|
+
if (!this.nextRequest) return [3 /*break*/, 9];
|
|
110
139
|
return [4 /*yield*/, this.nextRequest()];
|
|
111
|
-
case
|
|
140
|
+
case 7:
|
|
112
141
|
nextOptions = _a.sent();
|
|
113
|
-
if (!nextOptions) return [3 /*break*/,
|
|
142
|
+
if (!nextOptions) return [3 /*break*/, 9];
|
|
114
143
|
return [4 /*yield*/, this.processHttpData(processStream, nextOptions)];
|
|
115
|
-
case
|
|
144
|
+
case 8:
|
|
116
145
|
_a.sent();
|
|
117
|
-
_a.label =
|
|
118
|
-
case
|
|
146
|
+
_a.label = 9;
|
|
147
|
+
case 9: return [2 /*return*/];
|
|
119
148
|
}
|
|
120
149
|
});
|
|
121
150
|
});
|
package/lib/types.d.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import {
|
|
3
|
-
import { RequestPromiseOptions } from 'request-promise';
|
|
2
|
+
import { AxiosRequestConfig } from 'axios';
|
|
4
3
|
import { Readable } from 'stream';
|
|
5
4
|
import { IWorksheet } from 'xlstream/lib/types';
|
|
6
5
|
export interface IDbConnection {
|
|
@@ -41,6 +40,7 @@ export interface IExcelProcessorConfig extends IDirectoryProcessorConfig {
|
|
|
41
40
|
ignoreEmpty?: boolean;
|
|
42
41
|
sheets?: ((number | string)[] | sheetGetter);
|
|
43
42
|
fillMergedCells?: boolean;
|
|
43
|
+
encoding?: string;
|
|
44
44
|
}
|
|
45
45
|
export interface IProcessor {
|
|
46
46
|
process(processStream: processStream, ...args: any): Promise<void>;
|
|
@@ -69,14 +69,14 @@ export interface IPostgresDestinationConfig extends IDatabaseDestinationConfig {
|
|
|
69
69
|
export interface IMssqlDestinationConfig extends IDatabaseDestinationConfig {
|
|
70
70
|
}
|
|
71
71
|
export interface IHttpDestinationConfig extends IDestinationConfig {
|
|
72
|
-
request:
|
|
72
|
+
request: AxiosRequestConfig;
|
|
73
73
|
}
|
|
74
74
|
export interface IStdoutDestinationConfig extends IDestinationConfig {
|
|
75
75
|
asTable?: boolean;
|
|
76
76
|
}
|
|
77
77
|
export interface IHttpProcessorConfig extends IProcessorConfig {
|
|
78
|
-
connection:
|
|
79
|
-
nextRequest?: () => Promise<
|
|
78
|
+
connection: AxiosRequestConfig;
|
|
79
|
+
nextRequest?: () => Promise<AxiosRequestConfig | null | undefined>;
|
|
80
80
|
}
|
|
81
81
|
export interface IJsonHttpProcessorConfig extends IHttpProcessorConfig {
|
|
82
82
|
dataFormat: 'json';
|
|
@@ -85,6 +85,10 @@ export interface IJsonHttpProcessorConfig extends IHttpProcessorConfig {
|
|
|
85
85
|
export interface IDelimitedHttpProcessorConfig extends IHttpProcessorConfig {
|
|
86
86
|
dataFormat: 'delimited';
|
|
87
87
|
rowSeparator: string;
|
|
88
|
+
delimiter?: string;
|
|
89
|
+
hasHeader?: boolean;
|
|
90
|
+
trimQualifier?: boolean;
|
|
91
|
+
qualifier?: string;
|
|
88
92
|
}
|
|
89
93
|
export interface IDatabaseProcessorConfig extends IProcessorConfig {
|
|
90
94
|
connection: any;
|
|
@@ -97,6 +101,7 @@ export interface IDelimitedProcessorConfig extends IDirectoryProcessorConfig {
|
|
|
97
101
|
rowSeparator: string;
|
|
98
102
|
delimiter?: string;
|
|
99
103
|
hasHeader?: boolean;
|
|
104
|
+
trimQualifier?: boolean;
|
|
100
105
|
qualifier?: string;
|
|
101
106
|
}
|
|
102
107
|
export interface IMssqlProcessorConfig extends IDatabaseProcessorConfig {
|
package/lib/utils.d.ts
CHANGED
|
@@ -4,3 +4,15 @@ import { DbTypes, IDbConnection } from './types';
|
|
|
4
4
|
export declare function getDb(databaseConfig: IDbConnection, dbType: DbTypes): Promise<any>;
|
|
5
5
|
export declare function closeDbConnection(databaseConfig: IDbConnection): Promise<void>;
|
|
6
6
|
export declare function getValueFromJSONChunk(): Transform;
|
|
7
|
+
export declare function getDelimitedGenerator({ readStream, delimiter, qualifier, hasHeader, trimQualifier }: {
|
|
8
|
+
readStream: any;
|
|
9
|
+
delimiter?: string;
|
|
10
|
+
qualifier?: string;
|
|
11
|
+
hasHeader: boolean;
|
|
12
|
+
trimQualifier: boolean;
|
|
13
|
+
}): () => AsyncGenerator<{
|
|
14
|
+
header: string[];
|
|
15
|
+
arr: string[];
|
|
16
|
+
obj: any;
|
|
17
|
+
row: any;
|
|
18
|
+
}, void, unknown>;
|
package/lib/utils.js
CHANGED
|
@@ -65,8 +65,27 @@ var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
|
65
65
|
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
66
66
|
}
|
|
67
67
|
};
|
|
68
|
+
var __asyncValues = (this && this.__asyncValues) || function (o) {
|
|
69
|
+
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
|
|
70
|
+
var m = o[Symbol.asyncIterator], i;
|
|
71
|
+
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
|
|
72
|
+
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
|
|
73
|
+
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
|
|
74
|
+
};
|
|
75
|
+
var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); }
|
|
76
|
+
var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
|
|
77
|
+
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
|
|
78
|
+
var g = generator.apply(thisArg, _arguments || []), i, q = [];
|
|
79
|
+
return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i;
|
|
80
|
+
function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; }
|
|
81
|
+
function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
|
|
82
|
+
function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
|
|
83
|
+
function fulfill(value) { resume("next", value); }
|
|
84
|
+
function reject(value) { resume("throw", value); }
|
|
85
|
+
function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
|
|
86
|
+
};
|
|
68
87
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
69
|
-
exports.getValueFromJSONChunk = exports.closeDbConnection = exports.getDb = void 0;
|
|
88
|
+
exports.getDelimitedGenerator = exports.getValueFromJSONChunk = exports.closeDbConnection = exports.getDb = void 0;
|
|
70
89
|
var stream_1 = require("stream");
|
|
71
90
|
var cachedDbConnections = new Map();
|
|
72
91
|
function getDb(databaseConfig, dbType) {
|
|
@@ -149,3 +168,76 @@ function getValueFromJSONChunk() {
|
|
|
149
168
|
}
|
|
150
169
|
exports.getValueFromJSONChunk = getValueFromJSONChunk;
|
|
151
170
|
;
|
|
171
|
+
function getDelimitedGenerator(_a) {
|
|
172
|
+
var readStream = _a.readStream, delimiter = _a.delimiter, qualifier = _a.qualifier, hasHeader = _a.hasHeader, trimQualifier = _a.trimQualifier;
|
|
173
|
+
var header = [];
|
|
174
|
+
var splitRegExp = new RegExp(delimiter + "(?=(?:(?:[^" + qualifier + "]*" + qualifier + "){2})*[^" + qualifier + "]*$)");
|
|
175
|
+
var processRow = function (row) {
|
|
176
|
+
var arr = [];
|
|
177
|
+
if (qualifier) {
|
|
178
|
+
arr = row.split(splitRegExp);
|
|
179
|
+
if (trimQualifier) {
|
|
180
|
+
arr = arr.map(function (x) { return x[0] === qualifier && x[x.length - 1] === qualifier ? x.slice(1, -1) : x; });
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
else {
|
|
184
|
+
arr = row.split(delimiter);
|
|
185
|
+
}
|
|
186
|
+
if (hasHeader && !header.length) {
|
|
187
|
+
header = arr.map(function (x) { return x.trim(); });
|
|
188
|
+
}
|
|
189
|
+
else {
|
|
190
|
+
var obj = void 0;
|
|
191
|
+
if (header.length) {
|
|
192
|
+
obj = Object.fromEntries(header.map(function (x, i) { return [x, arr[i]]; }));
|
|
193
|
+
}
|
|
194
|
+
return { header: header, arr: arr, obj: obj, row: row };
|
|
195
|
+
}
|
|
196
|
+
};
|
|
197
|
+
var generator = function () {
|
|
198
|
+
return __asyncGenerator(this, arguments, function () {
|
|
199
|
+
var readStream_1, readStream_1_1, row, result, e_1_1;
|
|
200
|
+
var e_1, _a;
|
|
201
|
+
return __generator(this, function (_b) {
|
|
202
|
+
switch (_b.label) {
|
|
203
|
+
case 0:
|
|
204
|
+
_b.trys.push([0, 7, 8, 13]);
|
|
205
|
+
readStream_1 = __asyncValues(readStream);
|
|
206
|
+
_b.label = 1;
|
|
207
|
+
case 1: return [4 /*yield*/, __await(readStream_1.next())];
|
|
208
|
+
case 2:
|
|
209
|
+
if (!(readStream_1_1 = _b.sent(), !readStream_1_1.done)) return [3 /*break*/, 6];
|
|
210
|
+
row = readStream_1_1.value;
|
|
211
|
+
result = processRow(row);
|
|
212
|
+
if (!result) return [3 /*break*/, 5];
|
|
213
|
+
return [4 /*yield*/, __await(result)];
|
|
214
|
+
case 3: return [4 /*yield*/, _b.sent()];
|
|
215
|
+
case 4:
|
|
216
|
+
_b.sent();
|
|
217
|
+
_b.label = 5;
|
|
218
|
+
case 5: return [3 /*break*/, 1];
|
|
219
|
+
case 6: return [3 /*break*/, 13];
|
|
220
|
+
case 7:
|
|
221
|
+
e_1_1 = _b.sent();
|
|
222
|
+
e_1 = { error: e_1_1 };
|
|
223
|
+
return [3 /*break*/, 13];
|
|
224
|
+
case 8:
|
|
225
|
+
_b.trys.push([8, , 11, 12]);
|
|
226
|
+
if (!(readStream_1_1 && !readStream_1_1.done && (_a = readStream_1.return))) return [3 /*break*/, 10];
|
|
227
|
+
return [4 /*yield*/, __await(_a.call(readStream_1))];
|
|
228
|
+
case 9:
|
|
229
|
+
_b.sent();
|
|
230
|
+
_b.label = 10;
|
|
231
|
+
case 10: return [3 /*break*/, 12];
|
|
232
|
+
case 11:
|
|
233
|
+
if (e_1) throw e_1.error;
|
|
234
|
+
return [7 /*endfinally*/];
|
|
235
|
+
case 12: return [7 /*endfinally*/];
|
|
236
|
+
case 13: return [2 /*return*/];
|
|
237
|
+
}
|
|
238
|
+
});
|
|
239
|
+
});
|
|
240
|
+
};
|
|
241
|
+
return generator;
|
|
242
|
+
}
|
|
243
|
+
exports.getDelimitedGenerator = getDelimitedGenerator;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bellboy",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "6.2.0",
|
|
4
4
|
"description": "Highly performant JavaScript data stream ETL engine.",
|
|
5
5
|
"main": "lib/index",
|
|
6
6
|
"types": "lib/index",
|
|
@@ -27,6 +27,7 @@
|
|
|
27
27
|
"msnodesqlv8": "^2.1.0"
|
|
28
28
|
},
|
|
29
29
|
"devDependencies": {
|
|
30
|
+
"@types/axios": "^0.14.0",
|
|
30
31
|
"@types/jest": "^25.2.1",
|
|
31
32
|
"@types/mssql": "^6.0.7",
|
|
32
33
|
"@types/node": "^14.10.2",
|
|
@@ -45,16 +46,15 @@
|
|
|
45
46
|
},
|
|
46
47
|
"dependencies": {
|
|
47
48
|
"async-mqtt": "^2.2.1",
|
|
49
|
+
"axios": "^0.21.1",
|
|
48
50
|
"mssql": "^6.3.1",
|
|
49
51
|
"pg": "8.0.3",
|
|
50
52
|
"pg-promise": "10.5.3",
|
|
51
53
|
"pg-query-stream": "3.0.7",
|
|
52
|
-
"request": "^2.88.0",
|
|
53
|
-
"request-promise": "^4.2.4",
|
|
54
54
|
"shortid": "^2.2.15",
|
|
55
55
|
"split2": "^3.1.1",
|
|
56
56
|
"stream-json": "^1.7.1",
|
|
57
57
|
"tail": "^2.0.2",
|
|
58
|
-
"xlstream": "^2.
|
|
58
|
+
"xlstream": "^2.5.0"
|
|
59
59
|
}
|
|
60
60
|
}
|