bellboy 6.0.0 → 6.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -340,6 +340,8 @@ For delimited data produces rows described [here](#delimited-produced-row).
340
340
  If `true`, first row will be processed as a header.
341
341
  - **qualifier** `string` `only for delimited`\
342
342
  Symbol placed around a field to signify that it is the same field.
343
+ - **trimQualifier** `boolean` `only for delimited`\
344
+ Trim qualifier from produced row item.
343
345
  - **jsonPath** `RegExp`\
344
346
  Path to the array to be streamed. This option is described in detail inside [JsonProcessor](#json-processor) section.
345
347
  - **nextRequest** `async function(header)`\
@@ -394,6 +396,8 @@ Processes `XLSX` files in the directory.
394
396
  Whether to ignore empty rows or not, `true` by default.
395
397
  - **sheets** `(string | number)[] | async function(sheets)`\
396
398
  Array of sheet names and/or sheet indexes or async function, which accepts array of all sheets and must return another array of sheet names that needs to be processed. If not specified, first sheet will be processed.
399
+ - **encoding** `string`\
400
+ XLSX file encoding.
397
401
 
398
402
  ```javascript
399
403
  const processor = new bellboy.ExcelProcessor({
@@ -455,6 +459,8 @@ Processes files with delimited data in the directory.
455
459
  If `true`, first row will be processed as a header.
456
460
  - **qualifier** `string` \
457
461
  Symbol placed around a field to signify that it is the same field.
462
+ - **trimQualifier** `boolean` \
463
+ Trim qualifier from produced row item.
458
464
 
459
465
  #### Produced row <div id='delimited-produced-row'/>
460
466
 
@@ -5,6 +5,7 @@ export declare class DelimitedProcessor extends DirectoryProcessor {
5
5
  protected hasHeader: boolean;
6
6
  protected delimiter?: string;
7
7
  protected qualifier?: string;
8
+ protected trimQualifier: boolean;
8
9
  constructor(config: IDelimitedProcessorConfig);
9
10
  process(processStream: processStream): Promise<void>;
10
11
  }
@@ -62,11 +62,13 @@ var DelimitedProcessor = /** @class */ (function (_super) {
62
62
  __extends(DelimitedProcessor, _super);
63
63
  function DelimitedProcessor(config) {
64
64
  var _this = _super.call(this, config) || this;
65
+ _this.trimQualifier = false;
65
66
  if (!config.rowSeparator) {
66
67
  throw new Error('No rowSeparator specified.');
67
68
  }
68
69
  _this.rowSeparator = config.rowSeparator;
69
70
  _this.hasHeader = !!config.hasHeader;
71
+ _this.trimQualifier = !!config.trimQualifier;
70
72
  _this.delimiter = config.delimiter;
71
73
  _this.qualifier = config.qualifier;
72
74
  return _this;
@@ -90,6 +92,7 @@ var DelimitedProcessor = /** @class */ (function (_super) {
90
92
  hasHeader: this.hasHeader,
91
93
  qualifier: this.qualifier,
92
94
  readStream: readStream,
95
+ trimQualifier: this.trimQualifier,
93
96
  });
94
97
  return [4 /*yield*/, processStream(generator(), file, filePath)];
95
98
  case 2:
@@ -3,6 +3,7 @@ import { DirectoryProcessor } from './base/directory-processor';
3
3
  export declare class ExcelProcessor extends DirectoryProcessor {
4
4
  protected hasHeader: boolean | number | undefined;
5
5
  protected ignoreEmpty: boolean;
6
+ protected encoding?: string;
6
7
  protected fillMergedCells: boolean;
7
8
  protected sheets: (string | number)[] | sheetGetter;
8
9
  constructor(config: IExcelProcessorConfig);
@@ -60,6 +60,7 @@ var ExcelProcessor = /** @class */ (function (_super) {
60
60
  __extends(ExcelProcessor, _super);
61
61
  function ExcelProcessor(config) {
62
62
  var _this = _super.call(this, config) || this;
63
+ _this.encoding = config.encoding;
63
64
  _this.hasHeader = config.hasHeader;
64
65
  _this.ignoreEmpty = config.ignoreEmpty === false ? false : true;
65
66
  _this.fillMergedCells = !!config.fillMergedCells;
@@ -106,6 +107,7 @@ var ExcelProcessor = /** @class */ (function (_super) {
106
107
  withHeader: this.hasHeader,
107
108
  ignoreEmpty: this.ignoreEmpty,
108
109
  fillMergedCells: this.fillMergedCells,
110
+ encoding: this.encoding,
109
111
  })];
110
112
  case 7:
111
113
  readStream = _b.sent();
@@ -10,6 +10,7 @@ export declare class HttpProcessor extends Processor {
10
10
  protected hasHeader: boolean;
11
11
  protected delimiter?: string;
12
12
  protected qualifier?: string;
13
+ protected trimQualifier: boolean;
13
14
  constructor(config: IJsonHttpProcessorConfig | IDelimitedHttpProcessorConfig);
14
15
  protected processHttpData(processStream: processStream, options: AxiosRequestConfig): Promise<void>;
15
16
  process(processStream: processStream): Promise<void>;
@@ -76,6 +76,7 @@ var HttpProcessor = /** @class */ (function (_super) {
76
76
  function HttpProcessor(config) {
77
77
  var _this = _super.call(this, config) || this;
78
78
  _this.hasHeader = false;
79
+ _this.trimQualifier = false;
79
80
  if (!config.connection) {
80
81
  throw new Error("No connection specified.");
81
82
  }
@@ -87,6 +88,7 @@ var HttpProcessor = /** @class */ (function (_super) {
87
88
  }
88
89
  _this.rowSeparator = config.rowSeparator;
89
90
  _this.hasHeader = !!config.hasHeader;
91
+ _this.trimQualifier = !!config.trimQualifier;
90
92
  _this.delimiter = config.delimiter;
91
93
  _this.qualifier = config.qualifier;
92
94
  }
@@ -112,6 +114,7 @@ var HttpProcessor = /** @class */ (function (_super) {
112
114
  delimiter: this.delimiter,
113
115
  hasHeader: this.hasHeader,
114
116
  qualifier: this.qualifier,
117
+ trimQualifier: this.trimQualifier,
115
118
  });
116
119
  return [4 /*yield*/, processStream(generator())];
117
120
  case 2:
@@ -70,18 +70,26 @@ var MssqlProcessor = /** @class */ (function (_super) {
70
70
  objectMode: true,
71
71
  read: function () {
72
72
  return __awaiter(this, void 0, void 0, function () {
73
- var rowData;
73
+ var err_1, rowData;
74
74
  return __generator(this, function (_a) {
75
75
  switch (_a.label) {
76
76
  case 0:
77
- if (!(data.length === 0)) return [3 /*break*/, 3];
78
- if (!!done) return [3 /*break*/, 2];
79
- return [4 /*yield*/, processRows()];
77
+ if (!(data.length === 0)) return [3 /*break*/, 6];
78
+ if (!!done) return [3 /*break*/, 5];
79
+ _a.label = 1;
80
80
  case 1:
81
+ _a.trys.push([1, 3, , 4]);
82
+ return [4 /*yield*/, processRows()];
83
+ case 2:
81
84
  _a.sent();
82
- return [3 /*break*/, 3];
83
- case 2: return [2 /*return*/, this.push(null)];
85
+ return [3 /*break*/, 4];
84
86
  case 3:
87
+ err_1 = _a.sent();
88
+ this.emit('error', err_1);
89
+ return [3 /*break*/, 4];
90
+ case 4: return [3 /*break*/, 6];
91
+ case 5: return [2 /*return*/, this.push(null)];
92
+ case 6:
85
93
  rowData = data.shift();
86
94
  this.push(rowData);
87
95
  return [2 /*return*/];
package/lib/types.d.ts CHANGED
@@ -40,6 +40,7 @@ export interface IExcelProcessorConfig extends IDirectoryProcessorConfig {
40
40
  ignoreEmpty?: boolean;
41
41
  sheets?: ((number | string)[] | sheetGetter);
42
42
  fillMergedCells?: boolean;
43
+ encoding?: string;
43
44
  }
44
45
  export interface IProcessor {
45
46
  process(processStream: processStream, ...args: any): Promise<void>;
@@ -86,6 +87,7 @@ export interface IDelimitedHttpProcessorConfig extends IHttpProcessorConfig {
86
87
  rowSeparator: string;
87
88
  delimiter?: string;
88
89
  hasHeader?: boolean;
90
+ trimQualifier?: boolean;
89
91
  qualifier?: string;
90
92
  }
91
93
  export interface IDatabaseProcessorConfig extends IProcessorConfig {
@@ -99,6 +101,7 @@ export interface IDelimitedProcessorConfig extends IDirectoryProcessorConfig {
99
101
  rowSeparator: string;
100
102
  delimiter?: string;
101
103
  hasHeader?: boolean;
104
+ trimQualifier?: boolean;
102
105
  qualifier?: string;
103
106
  }
104
107
  export interface IMssqlProcessorConfig extends IDatabaseProcessorConfig {
package/lib/utils.d.ts CHANGED
@@ -4,11 +4,12 @@ import { DbTypes, IDbConnection } from './types';
4
4
  export declare function getDb(databaseConfig: IDbConnection, dbType: DbTypes): Promise<any>;
5
5
  export declare function closeDbConnection(databaseConfig: IDbConnection): Promise<void>;
6
6
  export declare function getValueFromJSONChunk(): Transform;
7
- export declare function getDelimitedGenerator({ readStream, delimiter, qualifier, hasHeader }: {
7
+ export declare function getDelimitedGenerator({ readStream, delimiter, qualifier, hasHeader, trimQualifier }: {
8
8
  readStream: any;
9
9
  delimiter?: string;
10
10
  qualifier?: string;
11
11
  hasHeader: boolean;
12
+ trimQualifier: boolean;
12
13
  }): () => AsyncGenerator<{
13
14
  header: string[];
14
15
  arr: string[];
package/lib/utils.js CHANGED
@@ -169,13 +169,16 @@ function getValueFromJSONChunk() {
169
169
  exports.getValueFromJSONChunk = getValueFromJSONChunk;
170
170
  ;
171
171
  function getDelimitedGenerator(_a) {
172
- var readStream = _a.readStream, delimiter = _a.delimiter, qualifier = _a.qualifier, hasHeader = _a.hasHeader;
172
+ var readStream = _a.readStream, delimiter = _a.delimiter, qualifier = _a.qualifier, hasHeader = _a.hasHeader, trimQualifier = _a.trimQualifier;
173
173
  var header = [];
174
174
  var splitRegExp = new RegExp(delimiter + "(?=(?:(?:[^" + qualifier + "]*" + qualifier + "){2})*[^" + qualifier + "]*$)");
175
175
  var processRow = function (row) {
176
176
  var arr = [];
177
177
  if (qualifier) {
178
178
  arr = row.split(splitRegExp);
179
+ if (trimQualifier) {
180
+ arr = arr.map(function (x) { return x[0] === qualifier && x[x.length - 1] === qualifier ? x.slice(1, -1) : x; });
181
+ }
179
182
  }
180
183
  else {
181
184
  arr = row.split(delimiter);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bellboy",
3
- "version": "6.0.0",
3
+ "version": "6.2.2",
4
4
  "description": "Highly performant JavaScript data stream ETL engine.",
5
5
  "main": "lib/index",
6
6
  "types": "lib/index",
@@ -55,6 +55,6 @@
55
55
  "split2": "^3.1.1",
56
56
  "stream-json": "^1.7.1",
57
57
  "tail": "^2.0.2",
58
- "xlstream": "^2.3.3"
58
+ "xlstream": "^2.5.1"
59
59
  }
60
60
  }