bellboy 6.3.3 → 6.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -340,8 +340,19 @@ For delimited data produces rows described [here](#delimited-produced-row).
340
340
  If `true`, first row will be processed as a header.
341
341
  - **qualifier** `string` `only for delimited`\
342
342
  Symbol placed around a field to signify that it is the same field.
343
- - **jsonPath** `RegExp`\
343
+ - **jsonPath** `RegExp | string`\
344
344
  Path to the array to be streamed. This option is described in detail inside [JsonProcessor](#json-processor) section.
345
+ - **authorizationRequest** `object`
346
+ - **connection**\
347
+ Options from [axios](https://github.com/axios/axios) library.
348
+ - **applyTo**\
349
+ Where extracted field should be applied. Whether `header` or `query`.
350
+ - **sourceField**\
351
+ Name of the field from which value of authorization token will be extracted.
352
+ - **destinationField**\
353
+ Name of the field which will be applied to `header` or `query` using `applyTo` option.
354
+ - **prefix**\
355
+ Custom prefix to apply to the token.
345
356
  - **nextRequest** `async function(header)`\
346
357
  Function which must return `connection` for the next request or `null` if the next request is not needed.
347
358
 
@@ -435,10 +446,10 @@ Processes `JSON` files in the directory.
435
446
  #### Options
436
447
 
437
448
  - [Directory processor options](#directory-processor-options)
438
- - **jsonPath** `RegExp`\
449
+ - **jsonPath** `RegExp | string`\
439
450
  Path to the array to be streamed. Internally when JSON is streamed, current path is joined together using `.` as separator and then tested against provided regular expression. If not specified, a root array will be streamed. As an example, if you have this JSON object:\
440
451
  `{ "animals": { "dogs": [ "pug", "bulldog", "poodle" ] } }`\
441
- And want to stream `dogs` array, path you will need to use is `/animals.dogs.(\d+)/`.\
452
+ And want to stream `dogs` array, path you will need to use is `/animals.dogs.(\d+)/` if using RegExp as `jsonPath` and `animals.dogs.(\\d+)` if a string is used.\
442
453
  `(\d+)` is used here because each index of the array is a number.
443
454
 
444
455
  ### DelimitedProcessor <div id='delimited-processor'/>
@@ -572,6 +583,17 @@ Puts processed data one by one in `body` and executes specified HTTP request.
572
583
  - [General destination options](#destination-options)
573
584
  - **request** `required`\
574
585
  Options from [axios](https://github.com/axios/axios) library.
586
+ - **authorizationRequest** `object`
587
+ - **connection**\
588
+ Options from [axios](https://github.com/axios/axios) library.
589
+ - **applyTo**\
590
+ Where extracted field should be applied. Whether `header` or `query`.
591
+ - **sourceField**\
592
+ Name of the field from which value of authorization token will be extracted.
593
+ - **destinationField**\
594
+ Name of the field which will be applied to `header` or `query` using `applyTo` option.
595
+ - **prefix**\
596
+ Custom prefix to apply to the token.
575
597
 
576
598
  ### PostgresDestination <div id='postgres-destination'/>
577
599
 
@@ -1,8 +1,9 @@
1
1
  import { AxiosRequestConfig } from 'axios';
2
- import { IHttpDestinationConfig } from '../types';
2
+ import { AuthorizationRequest, IHttpDestinationConfig } from '../types';
3
3
  import { Destination } from './base/destination';
4
4
  export declare class HttpDestination extends Destination {
5
5
  protected request: AxiosRequestConfig;
6
+ protected authorizationRequest?: AuthorizationRequest;
6
7
  constructor(config: IHttpDestinationConfig);
7
8
  loadBatch(data: any[]): Promise<void>;
8
9
  }
@@ -65,12 +65,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
65
65
  Object.defineProperty(exports, "__esModule", { value: true });
66
66
  exports.HttpDestination = void 0;
67
67
  var axios_1 = __importDefault(require("axios"));
68
+ var utils_1 = require("../utils");
68
69
  var destination_1 = require("./base/destination");
69
70
  var HttpDestination = /** @class */ (function (_super) {
70
71
  __extends(HttpDestination, _super);
71
72
  function HttpDestination(config) {
72
73
  var _this = _super.call(this, config) || this;
73
74
  _this.request = config.request;
75
+ _this.authorizationRequest = config.authorizationRequest;
74
76
  return _this;
75
77
  }
76
78
  HttpDestination.prototype.loadBatch = function (data) {
@@ -78,19 +80,21 @@ var HttpDestination = /** @class */ (function (_super) {
78
80
  var i;
79
81
  return __generator(this, function (_a) {
80
82
  switch (_a.label) {
81
- case 0:
82
- i = 0;
83
- _a.label = 1;
83
+ case 0: return [4 /*yield*/, utils_1.applyHttpAuthorization(this.request, this.authorizationRequest)];
84
84
  case 1:
85
- if (!(i < data.length)) return [3 /*break*/, 4];
86
- return [4 /*yield*/, axios_1.default(__assign(__assign({}, this.request), { data: data[i] }))];
87
- case 2:
88
85
  _a.sent();
89
- _a.label = 3;
86
+ i = 0;
87
+ _a.label = 2;
88
+ case 2:
89
+ if (!(i < data.length)) return [3 /*break*/, 5];
90
+ return [4 /*yield*/, axios_1.default(__assign(__assign({}, this.request), { data: data[i] }))];
90
91
  case 3:
92
+ _a.sent();
93
+ _a.label = 4;
94
+ case 4:
91
95
  i++;
92
- return [3 /*break*/, 1];
93
- case 4: return [2 /*return*/];
96
+ return [3 /*break*/, 2];
97
+ case 5: return [2 /*return*/];
94
98
  }
95
99
  });
96
100
  });
@@ -90,6 +90,7 @@ var DelimitedProcessor = /** @class */ (function (_super) {
90
90
  record_delimiter: this.rowSeparator,
91
91
  raw: true,
92
92
  relax_quotes: true,
93
+ relax_column_count: true,
93
94
  });
94
95
  readStream = fileReadStream.pipe(parser);
95
96
  generator = utils_1.getDelimitedGenerator({
@@ -1,5 +1,5 @@
1
1
  import { AxiosRequestConfig } from 'axios';
2
- import { IDelimitedHttpProcessorConfig, IJsonHttpProcessorConfig, processStream } from '../types';
2
+ import { AuthorizationRequest, IDelimitedHttpProcessorConfig, IJsonHttpProcessorConfig, processStream } from '../types';
3
3
  import { Processor } from './base/processor';
4
4
  export declare class HttpProcessor extends Processor {
5
5
  protected connection: AxiosRequestConfig;
@@ -10,6 +10,7 @@ export declare class HttpProcessor extends Processor {
10
10
  protected hasHeader: boolean;
11
11
  protected delimiter?: string;
12
12
  protected qualifier?: string;
13
+ protected authorizationRequest?: AuthorizationRequest;
13
14
  constructor(config: IJsonHttpProcessorConfig | IDelimitedHttpProcessorConfig);
14
15
  protected processHttpData(processStream: processStream, options: AxiosRequestConfig): Promise<void>;
15
16
  process(processStream: processStream): Promise<void>;
@@ -91,10 +91,16 @@ var HttpProcessor = /** @class */ (function (_super) {
91
91
  _this.delimiter = config.delimiter;
92
92
  _this.qualifier = config.qualifier;
93
93
  }
94
- else if (config.dataFormat === 'json') {
95
- _this.jsonPath = config.jsonPath;
94
+ else if (config.dataFormat === 'json' && config.jsonPath) {
95
+ if (config.jsonPath instanceof RegExp) {
96
+ _this.jsonPath = config.jsonPath;
97
+ }
98
+ else {
99
+ _this.jsonPath = new RegExp(config.jsonPath);
100
+ }
96
101
  }
97
102
  _this.nextRequest = config.nextRequest;
103
+ _this.authorizationRequest = config.authorizationRequest;
98
104
  return _this;
99
105
  }
100
106
  HttpProcessor.prototype.processHttpData = function (processStream, options) {
@@ -156,8 +162,11 @@ var HttpProcessor = /** @class */ (function (_super) {
156
162
  return __awaiter(this, void 0, void 0, function () {
157
163
  return __generator(this, function (_a) {
158
164
  switch (_a.label) {
159
- case 0: return [4 /*yield*/, this.processHttpData(processStream, this.connection)];
165
+ case 0: return [4 /*yield*/, utils_1.applyHttpAuthorization(this.connection, this.authorizationRequest)];
160
166
  case 1:
167
+ _a.sent();
168
+ return [4 /*yield*/, this.processHttpData(processStream, this.connection)];
169
+ case 2:
161
170
  _a.sent();
162
171
  return [2 /*return*/];
163
172
  }
@@ -64,7 +64,14 @@ var JsonProcessor = /** @class */ (function (_super) {
64
64
  __extends(JsonProcessor, _super);
65
65
  function JsonProcessor(config) {
66
66
  var _this = _super.call(this, config) || this;
67
- _this.jsonPath = config.jsonPath;
67
+ if (config.jsonPath) {
68
+ if (config.jsonPath instanceof RegExp) {
69
+ _this.jsonPath = config.jsonPath;
70
+ }
71
+ else {
72
+ _this.jsonPath = new RegExp(config.jsonPath);
73
+ }
74
+ }
68
75
  return _this;
69
76
  }
70
77
  JsonProcessor.prototype.process = function (processStream) {
package/lib/types.d.ts CHANGED
@@ -2,6 +2,13 @@
2
2
  import { AxiosRequestConfig } from 'axios';
3
3
  import { Readable } from 'stream';
4
4
  import { IWorksheet } from 'xlstream/lib/types';
5
+ export interface AuthorizationRequest {
6
+ connection: AxiosRequestConfig;
7
+ applyTo: 'header' | 'query';
8
+ sourceField: string;
9
+ destinationField: string;
10
+ prefix?: string;
11
+ }
5
12
  export interface IDbConnection {
6
13
  user?: string;
7
14
  password?: string;
@@ -70,6 +77,7 @@ export interface IMssqlDestinationConfig extends IDatabaseDestinationConfig {
70
77
  }
71
78
  export interface IHttpDestinationConfig extends IDestinationConfig {
72
79
  request: AxiosRequestConfig;
80
+ authorizationRequest?: AuthorizationRequest;
73
81
  }
74
82
  export interface IStdoutDestinationConfig extends IDestinationConfig {
75
83
  asTable?: boolean;
@@ -77,10 +85,11 @@ export interface IStdoutDestinationConfig extends IDestinationConfig {
77
85
  export interface IHttpProcessorConfig extends IProcessorConfig {
78
86
  connection: AxiosRequestConfig;
79
87
  nextRequest?: () => Promise<AxiosRequestConfig | null | undefined>;
88
+ authorizationRequest?: AuthorizationRequest;
80
89
  }
81
90
  export interface IJsonHttpProcessorConfig extends IHttpProcessorConfig {
82
91
  dataFormat: 'json';
83
- jsonPath?: RegExp;
92
+ jsonPath?: RegExp | string;
84
93
  }
85
94
  export interface IDelimitedHttpProcessorConfig extends IHttpProcessorConfig {
86
95
  dataFormat: 'delimited';
@@ -94,7 +103,7 @@ export interface IDatabaseProcessorConfig extends IProcessorConfig {
94
103
  query: string;
95
104
  }
96
105
  export interface IJsonProcessorConfig extends IDirectoryProcessorConfig {
97
- jsonPath?: RegExp;
106
+ jsonPath?: RegExp | string;
98
107
  }
99
108
  export interface IDelimitedProcessorConfig extends IDirectoryProcessorConfig {
100
109
  rowSeparator: string;
package/lib/utils.d.ts CHANGED
@@ -1,6 +1,7 @@
1
1
  /// <reference types="node" />
2
+ import { AxiosRequestConfig } from 'axios';
2
3
  import { Transform } from 'stream';
3
- import { DbTypes, IDbConnection } from './types';
4
+ import { AuthorizationRequest, DbTypes, IDbConnection } from './types';
4
5
  export declare function getDb(databaseConfig: IDbConnection, dbType: DbTypes): Promise<any>;
5
6
  export declare function closeDbConnection(databaseConfig: IDbConnection): Promise<void>;
6
7
  export declare function getValueFromJSONChunk(): Transform;
@@ -13,3 +14,4 @@ export declare function getDelimitedGenerator({ readStream, hasHeader, }: {
13
14
  obj: any;
14
15
  row: string;
15
16
  }, void, unknown>;
17
+ export declare function applyHttpAuthorization(connection: AxiosRequestConfig, authorizationRequest?: AuthorizationRequest): Promise<void>;
package/lib/utils.js CHANGED
@@ -84,8 +84,12 @@ var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _ar
84
84
  function reject(value) { resume("throw", value); }
85
85
  function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
86
86
  };
87
+ var __importDefault = (this && this.__importDefault) || function (mod) {
88
+ return (mod && mod.__esModule) ? mod : { "default": mod };
89
+ };
87
90
  Object.defineProperty(exports, "__esModule", { value: true });
88
- exports.getDelimitedGenerator = exports.getValueFromJSONChunk = exports.closeDbConnection = exports.getDb = void 0;
91
+ exports.applyHttpAuthorization = exports.getDelimitedGenerator = exports.getValueFromJSONChunk = exports.closeDbConnection = exports.getDb = void 0;
92
+ var axios_1 = __importDefault(require("axios"));
89
93
  var stream_1 = require("stream");
90
94
  var cachedDbConnections = new Map();
91
95
  function getDb(databaseConfig, dbType) {
@@ -231,3 +235,24 @@ function getDelimitedGenerator(_a) {
231
235
  return generator;
232
236
  }
233
237
  exports.getDelimitedGenerator = getDelimitedGenerator;
238
+ function applyHttpAuthorization(connection, authorizationRequest) {
239
+ return __awaiter(this, void 0, void 0, function () {
240
+ var authorizationResponse, authorizationToken, applyTo;
241
+ var _a;
242
+ return __generator(this, function (_b) {
243
+ switch (_b.label) {
244
+ case 0:
245
+ if (!authorizationRequest) return [3 /*break*/, 2];
246
+ return [4 /*yield*/, axios_1.default(authorizationRequest.connection)];
247
+ case 1:
248
+ authorizationResponse = _b.sent();
249
+ authorizationToken = authorizationResponse.data[authorizationRequest.sourceField];
250
+ applyTo = authorizationRequest.applyTo === 'header' ? 'headers' : 'params';
251
+ connection[applyTo] = __assign(__assign({}, connection[applyTo]), (_a = {}, _a[authorizationRequest.destinationField] = authorizationRequest.prefix ? "" + authorizationRequest.prefix + authorizationToken : authorizationToken, _a));
252
+ _b.label = 2;
253
+ case 2: return [2 /*return*/];
254
+ }
255
+ });
256
+ });
257
+ }
258
+ exports.applyHttpAuthorization = applyHttpAuthorization;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bellboy",
3
- "version": "6.3.3",
3
+ "version": "6.5.1",
4
4
  "description": "Highly performant JavaScript data stream ETL engine.",
5
5
  "main": "lib/index",
6
6
  "types": "lib/index",