bellboy 8.1.2 → 8.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -350,6 +350,7 @@ For delimited data produces rows described [here](#delimited-produced-row).
350
350
  If `true`, first row will be processed as a header.
351
351
  - **qualifier** `string` `only for delimited`\
352
352
  Symbol placed around a field to signify that it is the same field.
353
+ - **encoding** `string` `only for delimited`
353
354
  - **jsonPath** `RegExp | string`\
354
355
  Path to the array to be streamed. This option is described in detail inside [JsonProcessor](#json-processor) section.
355
356
  - **saxOptions** `object` `only for xml`\
@@ -480,6 +481,7 @@ Processes files with delimited data in the directory.
480
481
  If `true`, first row will be processed as a header.
481
482
  - **qualifier** `string` \
482
483
  Symbol placed around a field to signify that it is the same field.
484
+ - **encoding** `string` `only for delimited`
483
485
 
484
486
  #### Produced row <div id='delimited-produced-row'/>
485
487
 
@@ -5,6 +5,7 @@ export declare class DelimitedProcessor extends DirectoryProcessor {
5
5
  protected hasHeader: boolean;
6
6
  protected delimiter?: string;
7
7
  protected qualifier?: string;
8
+ protected encoding?: string;
8
9
  constructor(config: IDelimitedProcessorConfig);
9
10
  process(processStream: processStream): Promise<void>;
10
11
  }
@@ -56,6 +56,7 @@ exports.DelimitedProcessor = void 0;
56
56
  var csv_parse_1 = require("csv-parse");
57
57
  var fs_1 = __importDefault(require("fs"));
58
58
  var path_1 = __importDefault(require("path"));
59
+ var iconv_lite_1 = __importDefault(require("iconv-lite"));
59
60
  var utils_1 = require("../utils");
60
61
  var directory_processor_1 = require("./base/directory-processor");
61
62
  var DelimitedProcessor = /** @class */ (function (_super) {
@@ -69,11 +70,12 @@ var DelimitedProcessor = /** @class */ (function (_super) {
69
70
  _this.hasHeader = !!config.hasHeader;
70
71
  _this.delimiter = config.delimiter;
71
72
  _this.qualifier = config.qualifier;
73
+ _this.encoding = config.encoding;
72
74
  return _this;
73
75
  }
74
76
  DelimitedProcessor.prototype.process = function (processStream) {
75
77
  return __awaiter(this, void 0, void 0, function () {
76
- var _i, _a, file, filePath, fileReadStream, parser, readStream, generator;
78
+ var _i, _a, file, filePath, fileReadStream, streamToParse, parser, readStream, generator;
77
79
  return __generator(this, function (_b) {
78
80
  switch (_b.label) {
79
81
  case 0:
@@ -84,6 +86,7 @@ var DelimitedProcessor = /** @class */ (function (_super) {
84
86
  file = _a[_i];
85
87
  filePath = path_1.default.join(this.path, file);
86
88
  fileReadStream = fs_1.default.createReadStream(filePath);
89
+ streamToParse = this.encoding ? fileReadStream.pipe(iconv_lite_1.default.decodeStream(this.encoding)) : fileReadStream;
87
90
  parser = csv_parse_1.parse({
88
91
  quote: this.qualifier,
89
92
  delimiter: this.delimiter,
@@ -92,7 +95,7 @@ var DelimitedProcessor = /** @class */ (function (_super) {
92
95
  relax_quotes: true,
93
96
  relax_column_count: true,
94
97
  });
95
- readStream = fileReadStream.pipe(parser);
98
+ readStream = streamToParse.pipe(parser);
96
99
  generator = utils_1.getDelimitedGenerator({
97
100
  hasHeader: this.hasHeader,
98
101
  readStream: readStream,
@@ -12,6 +12,7 @@ export declare class HttpProcessor extends Processor {
12
12
  protected qualifier?: string;
13
13
  protected authorizationRequest?: AuthorizationRequest;
14
14
  protected saxOptions?: any;
15
+ protected encoding?: string;
15
16
  constructor(config: IJsonHttpProcessorConfig | IDelimitedHttpProcessorConfig | IXmlHttpProcessorConfig);
16
17
  protected processHttpData(processStream: processStream, options: AxiosRequestConfig): Promise<void>;
17
18
  process(processStream: processStream): Promise<void>;
@@ -69,6 +69,7 @@ var csv_parse_1 = require("csv-parse");
69
69
  var Pick_1 = require("stream-json/filters/Pick");
70
70
  var Parser_1 = require("stream-json/Parser");
71
71
  var StreamValues_1 = require("stream-json/streamers/StreamValues");
72
+ var iconv_lite_1 = __importDefault(require("iconv-lite"));
72
73
  var utils_1 = require("../utils");
73
74
  var processor_1 = require("./base/processor");
74
75
  var saxStream = require('sax-stream');
@@ -90,6 +91,7 @@ var HttpProcessor = /** @class */ (function (_super) {
90
91
  _this.hasHeader = !!config.hasHeader;
91
92
  _this.delimiter = config.delimiter;
92
93
  _this.qualifier = config.qualifier;
94
+ _this.encoding = config.encoding;
93
95
  }
94
96
  else if (config.dataFormat === 'json' && config.jsonPath) {
95
97
  if (config.jsonPath instanceof RegExp) {
@@ -108,7 +110,7 @@ var HttpProcessor = /** @class */ (function (_super) {
108
110
  }
109
111
  HttpProcessor.prototype.processHttpData = function (processStream, options) {
110
112
  return __awaiter(this, void 0, void 0, function () {
111
- var requestStream, parser_1, delimitedStream, generator, requestStream, jsonStream, requestStream, xmlStream, nextOptions;
113
+ var requestStream, decodedStream, parser_1, delimitedStream, generator, requestStream, jsonStream, requestStream, xmlStream, nextOptions;
112
114
  return __generator(this, function (_a) {
113
115
  switch (_a.label) {
114
116
  case 0:
@@ -116,14 +118,16 @@ var HttpProcessor = /** @class */ (function (_super) {
116
118
  return [4 /*yield*/, axios_1.default(__assign(__assign({}, options), { responseType: 'stream' }))];
117
119
  case 1:
118
120
  requestStream = _a.sent();
121
+ decodedStream = this.encoding ? requestStream.data.pipe(iconv_lite_1.default.decodeStream(this.encoding)) : requestStream.data;
119
122
  parser_1 = csv_parse_1.parse({
120
123
  quote: this.qualifier,
121
124
  delimiter: this.delimiter,
122
125
  record_delimiter: this.rowSeparator,
123
126
  raw: true,
124
127
  relax_quotes: true,
128
+ relax_column_count: true,
125
129
  });
126
- delimitedStream = requestStream.data.pipe(parser_1);
130
+ delimitedStream = decodedStream.pipe(parser_1);
127
131
  generator = utils_1.getDelimitedGenerator({
128
132
  readStream: delimitedStream,
129
133
  hasHeader: this.hasHeader,
package/lib/types.d.ts CHANGED
@@ -122,6 +122,7 @@ export interface IDelimitedHttpProcessorConfig extends IHttpProcessorConfig {
122
122
  delimiter?: string;
123
123
  hasHeader?: boolean;
124
124
  qualifier?: string;
125
+ encoding?: string;
125
126
  }
126
127
  export interface IDatabaseProcessorConfig extends IProcessorConfig {
127
128
  query: string;
@@ -134,6 +135,7 @@ export interface IDelimitedProcessorConfig extends IDirectoryProcessorConfig {
134
135
  delimiter?: string;
135
136
  hasHeader?: boolean;
136
137
  qualifier?: string;
138
+ encoding?: string;
137
139
  }
138
140
  export interface IMssqlProcessorConfig extends IDatabaseProcessorConfig {
139
141
  connection: IMssqlDbConnection;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bellboy",
3
- "version": "8.1.2",
3
+ "version": "8.2.0",
4
4
  "description": "Highly performant JavaScript data stream ETL engine.",
5
5
  "main": "lib/index",
6
6
  "types": "lib/index",
@@ -54,6 +54,7 @@
54
54
  "async-mqtt": "^2.2.1",
55
55
  "axios": "^0.21.1",
56
56
  "csv": "^6.0.5",
57
+ "iconv-lite": "^0.6.3",
57
58
  "mssql": "8.1.0",
58
59
  "mysql2-stream-fix": "^3.9.1",
59
60
  "pg": "8.0.3",
@@ -66,4 +67,4 @@
66
67
  "tail": "^2.0.2",
67
68
  "xlstream": "^2.5.3"
68
69
  }
69
- }
70
+ }