bellboy 6.6.5 → 6.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -323,7 +323,7 @@ Listens for messages and processes them one by one. It also handles backpressure
323
323
 
324
324
  [Usage examples](tests/http-source.spec.ts)
325
325
 
326
- Processes data received from a HTTP call. Can process `JSON` as well as `delimited` data. Can handle pagination by using `nextRequest` function.
326
+ Processes data received from a HTTP call. Can process `json`, `xml` as well as `delimited` data. Can handle pagination by using `nextRequest` function.
327
327
 
328
328
  For delimited data produces rows described [here](#delimited-produced-row).
329
329
 
@@ -332,7 +332,7 @@ For delimited data produces rows described [here](#delimited-produced-row).
332
332
  - [Processor options](#processor-options)
333
333
  - **connection** `object` `required`\
334
334
  Options from [axios](https://github.com/axios/axios) library.
335
- - **dataFormat** `delimited | json` `required`
335
+ - **dataFormat** `delimited | json | xml` `required`
336
336
  - **rowSeparator** `string` `required for delimited`
337
337
  - **delimiter** `string` `only for delimited`\
338
338
  A symbol separating fields of the row.
@@ -342,6 +342,8 @@ For delimited data produces rows described [here](#delimited-produced-row).
342
342
  Symbol placed around a field to signify that it is the same field.
343
343
  - **jsonPath** `RegExp | string`\
344
344
  Path to the array to be streamed. This option is described in detail inside [JsonProcessor](#json-processor) section.
345
+ - **saxOptions** `object` `only for xml`\
346
+ Options for XML streaming as described in [sax-stream](https://github.com/melitele/sax-stream#api) library.
345
347
  - **authorizationRequest** `object`
346
348
  - **connection**\
347
349
  Options from [axios](https://github.com/axios/axios) library.
@@ -1,17 +1,18 @@
1
1
  import { AxiosRequestConfig } from 'axios';
2
- import { AuthorizationRequest, IDelimitedHttpProcessorConfig, IJsonHttpProcessorConfig, processStream } from '../types';
2
+ import { AuthorizationRequest, IDelimitedHttpProcessorConfig, IJsonHttpProcessorConfig, IXmlHttpProcessorConfig, processStream } from '../types';
3
3
  import { Processor } from './base/processor';
4
4
  export declare class HttpProcessor extends Processor {
5
5
  protected connection: AxiosRequestConfig;
6
6
  protected nextRequest: (() => Promise<any>) | undefined;
7
7
  protected jsonPath: RegExp | undefined;
8
8
  protected rowSeparator: string | undefined;
9
- protected dataFormat: 'json' | 'delimited';
9
+ protected dataFormat: 'json' | 'delimited' | 'xml';
10
10
  protected hasHeader: boolean;
11
11
  protected delimiter?: string;
12
12
  protected qualifier?: string;
13
13
  protected authorizationRequest?: AuthorizationRequest;
14
- constructor(config: IJsonHttpProcessorConfig | IDelimitedHttpProcessorConfig);
14
+ protected saxOptions?: any;
15
+ constructor(config: IJsonHttpProcessorConfig | IDelimitedHttpProcessorConfig | IXmlHttpProcessorConfig);
15
16
  protected processHttpData(processStream: processStream, options: AxiosRequestConfig): Promise<void>;
16
17
  process(processStream: processStream): Promise<void>;
17
18
  }
@@ -71,7 +71,7 @@ var Parser_1 = require("stream-json/Parser");
71
71
  var StreamValues_1 = require("stream-json/streamers/StreamValues");
72
72
  var utils_1 = require("../utils");
73
73
  var processor_1 = require("./base/processor");
74
- var split2 = require('split2');
74
+ var saxStream = require('sax-stream');
75
75
  var HttpProcessor = /** @class */ (function (_super) {
76
76
  __extends(HttpProcessor, _super);
77
77
  function HttpProcessor(config) {
@@ -99,13 +99,16 @@ var HttpProcessor = /** @class */ (function (_super) {
99
99
  _this.jsonPath = new RegExp(config.jsonPath);
100
100
  }
101
101
  }
102
+ else if (config.dataFormat === 'xml' && config.saxOptions) {
103
+ _this.saxOptions = config.saxOptions;
104
+ }
102
105
  _this.nextRequest = config.nextRequest;
103
106
  _this.authorizationRequest = config.authorizationRequest;
104
107
  return _this;
105
108
  }
106
109
  HttpProcessor.prototype.processHttpData = function (processStream, options) {
107
110
  return __awaiter(this, void 0, void 0, function () {
108
- var requestStream, parser_1, delimitedStream, generator, requestStream, jsonStream, nextOptions;
111
+ var requestStream, parser_1, delimitedStream, generator, requestStream, jsonStream, requestStream, xmlStream, nextOptions;
109
112
  return __generator(this, function (_a) {
110
113
  switch (_a.label) {
111
114
  case 0:
@@ -128,7 +131,7 @@ var HttpProcessor = /** @class */ (function (_super) {
128
131
  return [4 /*yield*/, processStream(generator())];
129
132
  case 2:
130
133
  _a.sent();
131
- return [3 /*break*/, 6];
134
+ return [3 /*break*/, 9];
132
135
  case 3:
133
136
  if (!(this.dataFormat === 'json')) return [3 /*break*/, 6];
134
137
  return [4 /*yield*/, axios_1.default(__assign(__assign({}, options), { responseType: 'stream' }))];
@@ -142,18 +145,28 @@ var HttpProcessor = /** @class */ (function (_super) {
142
145
  return [4 /*yield*/, processStream(jsonStream)];
143
146
  case 5:
144
147
  _a.sent();
145
- _a.label = 6;
148
+ return [3 /*break*/, 9];
146
149
  case 6:
147
- if (!this.nextRequest) return [3 /*break*/, 9];
148
- return [4 /*yield*/, this.nextRequest()];
150
+ if (!(this.dataFormat === 'xml')) return [3 /*break*/, 9];
151
+ return [4 /*yield*/, axios_1.default(__assign(__assign({}, options), { responseType: 'stream' }))];
149
152
  case 7:
150
- nextOptions = _a.sent();
151
- if (!nextOptions) return [3 /*break*/, 9];
152
- return [4 /*yield*/, this.processHttpData(processStream, nextOptions)];
153
+ requestStream = _a.sent();
154
+ xmlStream = requestStream.data.pipe(saxStream(this.saxOptions));
155
+ return [4 /*yield*/, processStream(xmlStream)];
153
156
  case 8:
154
157
  _a.sent();
155
158
  _a.label = 9;
156
- case 9: return [2 /*return*/];
159
+ case 9:
160
+ if (!this.nextRequest) return [3 /*break*/, 12];
161
+ return [4 /*yield*/, this.nextRequest()];
162
+ case 10:
163
+ nextOptions = _a.sent();
164
+ if (!nextOptions) return [3 /*break*/, 12];
165
+ return [4 /*yield*/, this.processHttpData(processStream, nextOptions)];
166
+ case 11:
167
+ _a.sent();
168
+ _a.label = 12;
169
+ case 12: return [2 /*return*/];
157
170
  }
158
171
  });
159
172
  });
package/lib/types.d.ts CHANGED
@@ -87,6 +87,10 @@ export interface IHttpProcessorConfig extends IProcessorConfig {
87
87
  nextRequest?: () => Promise<AxiosRequestConfig | null | undefined>;
88
88
  authorizationRequest?: AuthorizationRequest;
89
89
  }
90
+ export interface IXmlHttpProcessorConfig extends IHttpProcessorConfig {
91
+ dataFormat: 'xml';
92
+ saxOptions?: any;
93
+ }
90
94
  export interface IJsonHttpProcessorConfig extends IHttpProcessorConfig {
91
95
  dataFormat: 'json';
92
96
  jsonPath?: RegExp | string;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bellboy",
3
- "version": "6.6.5",
3
+ "version": "6.7.0",
4
4
  "description": "Highly performant JavaScript data stream ETL engine.",
5
5
  "main": "lib/index",
6
6
  "types": "lib/index",
@@ -52,6 +52,7 @@
52
52
  "pg": "8.0.3",
53
53
  "pg-promise": "10.5.3",
54
54
  "pg-query-stream": "3.0.7",
55
+ "sax-stream": "^1.3.0",
55
56
  "shortid": "^2.2.15",
56
57
  "split2": "^3.1.1",
57
58
  "stream-json": "^1.7.1",