bellboy 6.6.5 → 6.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/lib/processors/http-processor.d.ts +4 -3
- package/lib/processors/http-processor.js +25 -10
- package/lib/types.d.ts +4 -0
- package/lib/utils.d.ts +1 -0
- package/lib/utils.js +24 -1
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -323,7 +323,7 @@ Listens for messages and processes them one by one. It also handles backpressure
|
|
|
323
323
|
|
|
324
324
|
[Usage examples](tests/http-source.spec.ts)
|
|
325
325
|
|
|
326
|
-
Processes data received from a HTTP call. Can process `
|
|
326
|
+
Processes data received from a HTTP call. Can process `json`, `xml` as well as `delimited` data. Can handle pagination by using `nextRequest` function.
|
|
327
327
|
|
|
328
328
|
For delimited data produces rows described [here](#delimited-produced-row).
|
|
329
329
|
|
|
@@ -332,7 +332,7 @@ For delimited data produces rows described [here](#delimited-produced-row).
|
|
|
332
332
|
- [Processor options](#processor-options)
|
|
333
333
|
- **connection** `object` `required`\
|
|
334
334
|
Options from [axios](https://github.com/axios/axios) library.
|
|
335
|
-
- **dataFormat** `delimited | json` `required`
|
|
335
|
+
- **dataFormat** `delimited | json | xml` `required`
|
|
336
336
|
- **rowSeparator** `string` `required for delimited`
|
|
337
337
|
- **delimiter** `string` `only for delimited`\
|
|
338
338
|
A symbol separating fields of the row.
|
|
@@ -342,6 +342,8 @@ For delimited data produces rows described [here](#delimited-produced-row).
|
|
|
342
342
|
Symbol placed around a field to signify that it is the same field.
|
|
343
343
|
- **jsonPath** `RegExp | string`\
|
|
344
344
|
Path to the array to be streamed. This option is described in detail inside [JsonProcessor](#json-processor) section.
|
|
345
|
+
- **saxOptions** `object` `only for xml`\
|
|
346
|
+
Options for XML streaming as described in [sax-stream](https://github.com/melitele/sax-stream#api) library.
|
|
345
347
|
- **authorizationRequest** `object`
|
|
346
348
|
- **connection**\
|
|
347
349
|
Options from [axios](https://github.com/axios/axios) library.
|
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
import { AxiosRequestConfig } from 'axios';
|
|
2
|
-
import { AuthorizationRequest, IDelimitedHttpProcessorConfig, IJsonHttpProcessorConfig, processStream } from '../types';
|
|
2
|
+
import { AuthorizationRequest, IDelimitedHttpProcessorConfig, IJsonHttpProcessorConfig, IXmlHttpProcessorConfig, processStream } from '../types';
|
|
3
3
|
import { Processor } from './base/processor';
|
|
4
4
|
export declare class HttpProcessor extends Processor {
|
|
5
5
|
protected connection: AxiosRequestConfig;
|
|
6
6
|
protected nextRequest: (() => Promise<any>) | undefined;
|
|
7
7
|
protected jsonPath: RegExp | undefined;
|
|
8
8
|
protected rowSeparator: string | undefined;
|
|
9
|
-
protected dataFormat: 'json' | 'delimited';
|
|
9
|
+
protected dataFormat: 'json' | 'delimited' | 'xml';
|
|
10
10
|
protected hasHeader: boolean;
|
|
11
11
|
protected delimiter?: string;
|
|
12
12
|
protected qualifier?: string;
|
|
13
13
|
protected authorizationRequest?: AuthorizationRequest;
|
|
14
|
-
|
|
14
|
+
protected saxOptions?: any;
|
|
15
|
+
constructor(config: IJsonHttpProcessorConfig | IDelimitedHttpProcessorConfig | IXmlHttpProcessorConfig);
|
|
15
16
|
protected processHttpData(processStream: processStream, options: AxiosRequestConfig): Promise<void>;
|
|
16
17
|
process(processStream: processStream): Promise<void>;
|
|
17
18
|
}
|
|
@@ -71,7 +71,7 @@ var Parser_1 = require("stream-json/Parser");
|
|
|
71
71
|
var StreamValues_1 = require("stream-json/streamers/StreamValues");
|
|
72
72
|
var utils_1 = require("../utils");
|
|
73
73
|
var processor_1 = require("./base/processor");
|
|
74
|
-
var
|
|
74
|
+
var saxStream = require('sax-stream');
|
|
75
75
|
var HttpProcessor = /** @class */ (function (_super) {
|
|
76
76
|
__extends(HttpProcessor, _super);
|
|
77
77
|
function HttpProcessor(config) {
|
|
@@ -99,13 +99,16 @@ var HttpProcessor = /** @class */ (function (_super) {
|
|
|
99
99
|
_this.jsonPath = new RegExp(config.jsonPath);
|
|
100
100
|
}
|
|
101
101
|
}
|
|
102
|
+
else if (config.dataFormat === 'xml' && config.saxOptions) {
|
|
103
|
+
_this.saxOptions = config.saxOptions;
|
|
104
|
+
}
|
|
102
105
|
_this.nextRequest = config.nextRequest;
|
|
103
106
|
_this.authorizationRequest = config.authorizationRequest;
|
|
104
107
|
return _this;
|
|
105
108
|
}
|
|
106
109
|
HttpProcessor.prototype.processHttpData = function (processStream, options) {
|
|
107
110
|
return __awaiter(this, void 0, void 0, function () {
|
|
108
|
-
var requestStream, parser_1, delimitedStream, generator, requestStream, jsonStream, nextOptions;
|
|
111
|
+
var requestStream, parser_1, delimitedStream, generator, requestStream, jsonStream, requestStream, xmlStream, nextOptions;
|
|
109
112
|
return __generator(this, function (_a) {
|
|
110
113
|
switch (_a.label) {
|
|
111
114
|
case 0:
|
|
@@ -128,7 +131,7 @@ var HttpProcessor = /** @class */ (function (_super) {
|
|
|
128
131
|
return [4 /*yield*/, processStream(generator())];
|
|
129
132
|
case 2:
|
|
130
133
|
_a.sent();
|
|
131
|
-
return [3 /*break*/,
|
|
134
|
+
return [3 /*break*/, 9];
|
|
132
135
|
case 3:
|
|
133
136
|
if (!(this.dataFormat === 'json')) return [3 /*break*/, 6];
|
|
134
137
|
return [4 /*yield*/, axios_1.default(__assign(__assign({}, options), { responseType: 'stream' }))];
|
|
@@ -142,18 +145,30 @@ var HttpProcessor = /** @class */ (function (_super) {
|
|
|
142
145
|
return [4 /*yield*/, processStream(jsonStream)];
|
|
143
146
|
case 5:
|
|
144
147
|
_a.sent();
|
|
145
|
-
|
|
148
|
+
return [3 /*break*/, 9];
|
|
146
149
|
case 6:
|
|
147
|
-
if (!this.
|
|
148
|
-
return [4 /*yield*/,
|
|
150
|
+
if (!(this.dataFormat === 'xml')) return [3 /*break*/, 9];
|
|
151
|
+
return [4 /*yield*/, axios_1.default(__assign(__assign({}, options), { responseType: 'stream' }))];
|
|
149
152
|
case 7:
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
+
requestStream = _a.sent();
|
|
154
|
+
xmlStream = requestStream.data
|
|
155
|
+
.pipe(saxStream(this.saxOptions))
|
|
156
|
+
.pipe(utils_1.removeCircularReferencesFromChunk());
|
|
157
|
+
return [4 /*yield*/, processStream(xmlStream)];
|
|
153
158
|
case 8:
|
|
154
159
|
_a.sent();
|
|
155
160
|
_a.label = 9;
|
|
156
|
-
case 9:
|
|
161
|
+
case 9:
|
|
162
|
+
if (!this.nextRequest) return [3 /*break*/, 12];
|
|
163
|
+
return [4 /*yield*/, this.nextRequest()];
|
|
164
|
+
case 10:
|
|
165
|
+
nextOptions = _a.sent();
|
|
166
|
+
if (!nextOptions) return [3 /*break*/, 12];
|
|
167
|
+
return [4 /*yield*/, this.processHttpData(processStream, nextOptions)];
|
|
168
|
+
case 11:
|
|
169
|
+
_a.sent();
|
|
170
|
+
_a.label = 12;
|
|
171
|
+
case 12: return [2 /*return*/];
|
|
157
172
|
}
|
|
158
173
|
});
|
|
159
174
|
});
|
package/lib/types.d.ts
CHANGED
|
@@ -87,6 +87,10 @@ export interface IHttpProcessorConfig extends IProcessorConfig {
|
|
|
87
87
|
nextRequest?: () => Promise<AxiosRequestConfig | null | undefined>;
|
|
88
88
|
authorizationRequest?: AuthorizationRequest;
|
|
89
89
|
}
|
|
90
|
+
export interface IXmlHttpProcessorConfig extends IHttpProcessorConfig {
|
|
91
|
+
dataFormat: 'xml';
|
|
92
|
+
saxOptions?: any;
|
|
93
|
+
}
|
|
90
94
|
export interface IJsonHttpProcessorConfig extends IHttpProcessorConfig {
|
|
91
95
|
dataFormat: 'json';
|
|
92
96
|
jsonPath?: RegExp | string;
|
package/lib/utils.d.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { AuthorizationRequest, DbTypes, IDbConnection } from './types';
|
|
|
5
5
|
export declare function getDb(databaseConfig: IDbConnection, dbType: DbTypes): Promise<any>;
|
|
6
6
|
export declare function closeDbConnection(databaseConfig: IDbConnection): Promise<void>;
|
|
7
7
|
export declare function getValueFromJSONChunk(): Transform;
|
|
8
|
+
export declare function removeCircularReferencesFromChunk(): Transform;
|
|
8
9
|
export declare function getDelimitedGenerator({ readStream, hasHeader, }: {
|
|
9
10
|
readStream: any;
|
|
10
11
|
hasHeader: boolean;
|
package/lib/utils.js
CHANGED
|
@@ -88,7 +88,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
88
88
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
89
89
|
};
|
|
90
90
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
91
|
-
exports.applyHttpAuthorization = exports.getDelimitedGenerator = exports.getValueFromJSONChunk = exports.closeDbConnection = exports.getDb = void 0;
|
|
91
|
+
exports.applyHttpAuthorization = exports.getDelimitedGenerator = exports.removeCircularReferencesFromChunk = exports.getValueFromJSONChunk = exports.closeDbConnection = exports.getDb = void 0;
|
|
92
92
|
var axios_1 = __importDefault(require("axios"));
|
|
93
93
|
var stream_1 = require("stream");
|
|
94
94
|
var cachedDbConnections = new Map();
|
|
@@ -172,6 +172,29 @@ function getValueFromJSONChunk() {
|
|
|
172
172
|
}
|
|
173
173
|
exports.getValueFromJSONChunk = getValueFromJSONChunk;
|
|
174
174
|
;
|
|
175
|
+
function removeCircularReferencesFromChunk() {
|
|
176
|
+
return new stream_1.Transform({
|
|
177
|
+
objectMode: true,
|
|
178
|
+
transform: function (chunk, encoding, callback) {
|
|
179
|
+
function getCircularReplacer() {
|
|
180
|
+
var seen = new WeakSet();
|
|
181
|
+
return function (key, value) {
|
|
182
|
+
if (typeof value === "object" && value !== null) {
|
|
183
|
+
if (seen.has(value)) {
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
seen.add(value);
|
|
187
|
+
}
|
|
188
|
+
return value;
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
this.push(JSON.parse(JSON.stringify(chunk, getCircularReplacer())));
|
|
192
|
+
callback();
|
|
193
|
+
}
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
exports.removeCircularReferencesFromChunk = removeCircularReferencesFromChunk;
|
|
197
|
+
;
|
|
175
198
|
function getDelimitedGenerator(_a) {
|
|
176
199
|
var readStream = _a.readStream, hasHeader = _a.hasHeader;
|
|
177
200
|
var header = [];
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bellboy",
|
|
3
|
-
"version": "6.
|
|
3
|
+
"version": "6.7.1",
|
|
4
4
|
"description": "Highly performant JavaScript data stream ETL engine.",
|
|
5
5
|
"main": "lib/index",
|
|
6
6
|
"types": "lib/index",
|
|
@@ -52,6 +52,7 @@
|
|
|
52
52
|
"pg": "8.0.3",
|
|
53
53
|
"pg-promise": "10.5.3",
|
|
54
54
|
"pg-query-stream": "3.0.7",
|
|
55
|
+
"sax-stream": "^1.3.0",
|
|
55
56
|
"shortid": "^2.2.15",
|
|
56
57
|
"split2": "^3.1.1",
|
|
57
58
|
"stream-json": "^1.7.1",
|