bellboy 6.2.1 → 6.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -340,8 +340,6 @@ For delimited data produces rows described [here](#delimited-produced-row).
340
340
  If `true`, first row will be processed as a header.
341
341
  - **qualifier** `string` `only for delimited`\
342
342
  Symbol placed around a field to signify that it is the same field.
343
- - **trimQualifier** `boolean` `only for delimited`\
344
- Trim qualifier from produced row item.
345
343
  - **jsonPath** `RegExp`\
346
344
  Path to the array to be streamed. This option is described in detail inside [JsonProcessor](#json-processor) section.
347
345
  - **nextRequest** `async function(header)`\
@@ -459,8 +457,6 @@ Processes files with delimited data in the directory.
459
457
  If `true`, first row will be processed as a header.
460
458
  - **qualifier** `string` \
461
459
  Symbol placed around a field to signify that it is the same field.
462
- - **trimQualifier** `boolean` \
463
- Trim qualifier from produced row item.
464
460
 
465
461
  #### Produced row <div id='delimited-produced-row'/>
466
462
 
@@ -12,6 +12,17 @@ var __extends = (this && this.__extends) || (function () {
12
12
  d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
13
13
  };
14
14
  })();
15
+ var __assign = (this && this.__assign) || function () {
16
+ __assign = Object.assign || function(t) {
17
+ for (var s, i = 1, n = arguments.length; i < n; i++) {
18
+ s = arguments[i];
19
+ for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
20
+ t[p] = s[p];
21
+ }
22
+ return t;
23
+ };
24
+ return __assign.apply(this, arguments);
25
+ };
15
26
  var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
16
27
  if (k2 === undefined) k2 = k;
17
28
  Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
@@ -69,7 +80,6 @@ var __generator = (this && this.__generator) || function (thisArg, body) {
69
80
  };
70
81
  Object.defineProperty(exports, "__esModule", { value: true });
71
82
  exports.MssqlDestination = void 0;
72
- var utils_1 = require("../utils");
73
83
  var database_destination_1 = require("./base/database-destination");
74
84
  var MssqlDestination = /** @class */ (function (_super) {
75
85
  __extends(MssqlDestination, _super);
@@ -80,7 +90,7 @@ var MssqlDestination = /** @class */ (function (_super) {
80
90
  }
81
91
  MssqlDestination.prototype.loadBatch = function (data) {
82
92
  return __awaiter(this, void 0, void 0, function () {
83
- var sql, _a, db, query, table, columns, transaction, _i, data_1, x, args, ii;
93
+ var sql, _a, pool, db, query, table, columns, _i, data_1, x, args, ii, transaction, err_1;
84
94
  var _b;
85
95
  return __generator(this, function (_c) {
86
96
  switch (_c.label) {
@@ -96,7 +106,8 @@ var MssqlDestination = /** @class */ (function (_super) {
96
106
  _c.label = 4;
97
107
  case 4:
98
108
  sql = _a;
99
- return [4 /*yield*/, utils_1.getDb(this.connection, 'mssql')];
109
+ pool = new sql.ConnectionPool(__assign({}, this.connection));
110
+ return [4 /*yield*/, pool.connect()];
100
111
  case 5:
101
112
  db = _c.sent();
102
113
  return [4 /*yield*/, db.request().query("SELECT TOP(0) * FROM " + this.table)];
@@ -104,10 +115,6 @@ var MssqlDestination = /** @class */ (function (_super) {
104
115
  query = _c.sent();
105
116
  table = query.recordset.toTable(this.table);
106
117
  columns = table.columns.map(function (x) { return x.name; });
107
- transaction = new sql.Transaction(db);
108
- return [4 /*yield*/, transaction.begin()];
109
- case 7:
110
- _c.sent();
111
118
  for (_i = 0, data_1 = data; _i < data_1.length; _i++) {
112
119
  x = data_1[_i];
113
120
  args = [];
@@ -116,11 +123,31 @@ var MssqlDestination = /** @class */ (function (_super) {
116
123
  }
117
124
  (_b = table.rows).add.apply(_b, args);
118
125
  }
119
- return [4 /*yield*/, transaction.request().bulk(table)];
126
+ transaction = new sql.Transaction(db);
127
+ _c.label = 7;
128
+ case 7:
129
+ _c.trys.push([7, 11, , 14]);
130
+ return [4 /*yield*/, transaction.begin()];
120
131
  case 8:
121
132
  _c.sent();
122
- return [4 /*yield*/, transaction.commit()];
133
+ return [4 /*yield*/, transaction.request().bulk(table)];
123
134
  case 9:
135
+ _c.sent();
136
+ return [4 /*yield*/, transaction.commit()];
137
+ case 10:
138
+ _c.sent();
139
+ return [3 /*break*/, 14];
140
+ case 11:
141
+ err_1 = _c.sent();
142
+ return [4 /*yield*/, transaction.rollback()];
143
+ case 12:
144
+ _c.sent();
145
+ return [4 /*yield*/, pool.close()];
146
+ case 13:
147
+ _c.sent();
148
+ throw err_1;
149
+ case 14: return [4 /*yield*/, pool.close()];
150
+ case 15:
124
151
  _c.sent();
125
152
  return [2 /*return*/];
126
153
  }
@@ -5,7 +5,6 @@ export declare class DelimitedProcessor extends DirectoryProcessor {
5
5
  protected hasHeader: boolean;
6
6
  protected delimiter?: string;
7
7
  protected qualifier?: string;
8
- protected trimQualifier: boolean;
9
8
  constructor(config: IDelimitedProcessorConfig);
10
9
  process(processStream: processStream): Promise<void>;
11
10
  }
@@ -53,29 +53,27 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
53
53
  };
54
54
  Object.defineProperty(exports, "__esModule", { value: true });
55
55
  exports.DelimitedProcessor = void 0;
56
+ var csv_parse_1 = require("csv-parse");
56
57
  var fs_1 = __importDefault(require("fs"));
57
58
  var path_1 = __importDefault(require("path"));
58
59
  var utils_1 = require("../utils");
59
60
  var directory_processor_1 = require("./base/directory-processor");
60
- var split2 = require('split2');
61
61
  var DelimitedProcessor = /** @class */ (function (_super) {
62
62
  __extends(DelimitedProcessor, _super);
63
63
  function DelimitedProcessor(config) {
64
64
  var _this = _super.call(this, config) || this;
65
- _this.trimQualifier = false;
66
65
  if (!config.rowSeparator) {
67
66
  throw new Error('No rowSeparator specified.');
68
67
  }
69
68
  _this.rowSeparator = config.rowSeparator;
70
69
  _this.hasHeader = !!config.hasHeader;
71
- _this.trimQualifier = !!config.trimQualifier;
72
70
  _this.delimiter = config.delimiter;
73
71
  _this.qualifier = config.qualifier;
74
72
  return _this;
75
73
  }
76
74
  DelimitedProcessor.prototype.process = function (processStream) {
77
75
  return __awaiter(this, void 0, void 0, function () {
78
- var _i, _a, file, filePath, fileReadStream, readStream, generator;
76
+ var _i, _a, file, filePath, fileReadStream, parser, readStream, generator;
79
77
  return __generator(this, function (_b) {
80
78
  switch (_b.label) {
81
79
  case 0:
@@ -86,13 +84,17 @@ var DelimitedProcessor = /** @class */ (function (_super) {
86
84
  file = _a[_i];
87
85
  filePath = path_1.default.join(this.path, file);
88
86
  fileReadStream = fs_1.default.createReadStream(filePath);
89
- readStream = fileReadStream.pipe(split2(this.rowSeparator));
90
- generator = utils_1.getDelimitedGenerator({
87
+ parser = csv_parse_1.parse({
88
+ quote: this.qualifier,
91
89
  delimiter: this.delimiter,
90
+ record_delimiter: this.rowSeparator,
91
+ raw: true,
92
+ relax_quotes: true,
93
+ });
94
+ readStream = fileReadStream.pipe(parser);
95
+ generator = utils_1.getDelimitedGenerator({
92
96
  hasHeader: this.hasHeader,
93
- qualifier: this.qualifier,
94
97
  readStream: readStream,
95
- trimQualifier: this.trimQualifier,
96
98
  });
97
99
  return [4 /*yield*/, processStream(generator(), file, filePath)];
98
100
  case 2:
@@ -10,7 +10,6 @@ export declare class HttpProcessor extends Processor {
10
10
  protected hasHeader: boolean;
11
11
  protected delimiter?: string;
12
12
  protected qualifier?: string;
13
- protected trimQualifier: boolean;
14
13
  constructor(config: IJsonHttpProcessorConfig | IDelimitedHttpProcessorConfig);
15
14
  protected processHttpData(processStream: processStream, options: AxiosRequestConfig): Promise<void>;
16
15
  process(processStream: processStream): Promise<void>;
@@ -65,6 +65,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
65
65
  Object.defineProperty(exports, "__esModule", { value: true });
66
66
  exports.HttpProcessor = void 0;
67
67
  var axios_1 = __importDefault(require("axios"));
68
+ var csv_parse_1 = require("csv-parse");
68
69
  var Pick_1 = require("stream-json/filters/Pick");
69
70
  var Parser_1 = require("stream-json/Parser");
70
71
  var StreamValues_1 = require("stream-json/streamers/StreamValues");
@@ -76,7 +77,6 @@ var HttpProcessor = /** @class */ (function (_super) {
76
77
  function HttpProcessor(config) {
77
78
  var _this = _super.call(this, config) || this;
78
79
  _this.hasHeader = false;
79
- _this.trimQualifier = false;
80
80
  if (!config.connection) {
81
81
  throw new Error("No connection specified.");
82
82
  }
@@ -88,7 +88,6 @@ var HttpProcessor = /** @class */ (function (_super) {
88
88
  }
89
89
  _this.rowSeparator = config.rowSeparator;
90
90
  _this.hasHeader = !!config.hasHeader;
91
- _this.trimQualifier = !!config.trimQualifier;
92
91
  _this.delimiter = config.delimiter;
93
92
  _this.qualifier = config.qualifier;
94
93
  }
@@ -100,7 +99,7 @@ var HttpProcessor = /** @class */ (function (_super) {
100
99
  }
101
100
  HttpProcessor.prototype.processHttpData = function (processStream, options) {
102
101
  return __awaiter(this, void 0, void 0, function () {
103
- var requestStream, delimitedStream, generator, requestStream, jsonStream, nextOptions;
102
+ var requestStream, parser_1, delimitedStream, generator, requestStream, jsonStream, nextOptions;
104
103
  return __generator(this, function (_a) {
105
104
  switch (_a.label) {
106
105
  case 0:
@@ -108,13 +107,17 @@ var HttpProcessor = /** @class */ (function (_super) {
108
107
  return [4 /*yield*/, axios_1.default(__assign(__assign({}, options), { responseType: 'stream' }))];
109
108
  case 1:
110
109
  requestStream = _a.sent();
111
- delimitedStream = requestStream.data.pipe(split2(this.rowSeparator));
110
+ parser_1 = csv_parse_1.parse({
111
+ quote: this.qualifier,
112
+ delimiter: this.delimiter,
113
+ record_delimiter: this.rowSeparator,
114
+ raw: true,
115
+ relax_quotes: true,
116
+ });
117
+ delimitedStream = requestStream.data.pipe(parser_1);
112
118
  generator = utils_1.getDelimitedGenerator({
113
119
  readStream: delimitedStream,
114
- delimiter: this.delimiter,
115
120
  hasHeader: this.hasHeader,
116
- qualifier: this.qualifier,
117
- trimQualifier: this.trimQualifier,
118
121
  });
119
122
  return [4 /*yield*/, processStream(generator())];
120
123
  case 2:
@@ -70,18 +70,26 @@ var MssqlProcessor = /** @class */ (function (_super) {
70
70
  objectMode: true,
71
71
  read: function () {
72
72
  return __awaiter(this, void 0, void 0, function () {
73
- var rowData;
73
+ var err_1, rowData;
74
74
  return __generator(this, function (_a) {
75
75
  switch (_a.label) {
76
76
  case 0:
77
- if (!(data.length === 0)) return [3 /*break*/, 3];
78
- if (!!done) return [3 /*break*/, 2];
79
- return [4 /*yield*/, processRows()];
77
+ if (!(data.length === 0)) return [3 /*break*/, 6];
78
+ if (!!done) return [3 /*break*/, 5];
79
+ _a.label = 1;
80
80
  case 1:
81
+ _a.trys.push([1, 3, , 4]);
82
+ return [4 /*yield*/, processRows()];
83
+ case 2:
81
84
  _a.sent();
82
- return [3 /*break*/, 3];
83
- case 2: return [2 /*return*/, this.push(null)];
85
+ return [3 /*break*/, 4];
84
86
  case 3:
87
+ err_1 = _a.sent();
88
+ this.emit('error', err_1);
89
+ return [3 /*break*/, 4];
90
+ case 4: return [3 /*break*/, 6];
91
+ case 5: return [2 /*return*/, this.push(null)];
92
+ case 6:
85
93
  rowData = data.shift();
86
94
  this.push(rowData);
87
95
  return [2 /*return*/];
package/lib/types.d.ts CHANGED
@@ -87,7 +87,6 @@ export interface IDelimitedHttpProcessorConfig extends IHttpProcessorConfig {
87
87
  rowSeparator: string;
88
88
  delimiter?: string;
89
89
  hasHeader?: boolean;
90
- trimQualifier?: boolean;
91
90
  qualifier?: string;
92
91
  }
93
92
  export interface IDatabaseProcessorConfig extends IProcessorConfig {
@@ -101,7 +100,6 @@ export interface IDelimitedProcessorConfig extends IDirectoryProcessorConfig {
101
100
  rowSeparator: string;
102
101
  delimiter?: string;
103
102
  hasHeader?: boolean;
104
- trimQualifier?: boolean;
105
103
  qualifier?: string;
106
104
  }
107
105
  export interface IMssqlProcessorConfig extends IDatabaseProcessorConfig {
package/lib/utils.d.ts CHANGED
@@ -4,15 +4,12 @@ import { DbTypes, IDbConnection } from './types';
4
4
  export declare function getDb(databaseConfig: IDbConnection, dbType: DbTypes): Promise<any>;
5
5
  export declare function closeDbConnection(databaseConfig: IDbConnection): Promise<void>;
6
6
  export declare function getValueFromJSONChunk(): Transform;
7
- export declare function getDelimitedGenerator({ readStream, delimiter, qualifier, hasHeader, trimQualifier }: {
7
+ export declare function getDelimitedGenerator({ readStream, hasHeader, }: {
8
8
  readStream: any;
9
- delimiter?: string;
10
- qualifier?: string;
11
9
  hasHeader: boolean;
12
- trimQualifier: boolean;
13
10
  }): () => AsyncGenerator<{
14
11
  header: string[];
15
12
  arr: string[];
16
13
  obj: any;
17
- row: any;
14
+ row: string;
18
15
  }, void, unknown>;
package/lib/utils.js CHANGED
@@ -169,29 +169,19 @@ function getValueFromJSONChunk() {
169
169
  exports.getValueFromJSONChunk = getValueFromJSONChunk;
170
170
  ;
171
171
  function getDelimitedGenerator(_a) {
172
- var readStream = _a.readStream, delimiter = _a.delimiter, qualifier = _a.qualifier, hasHeader = _a.hasHeader, trimQualifier = _a.trimQualifier;
172
+ var readStream = _a.readStream, hasHeader = _a.hasHeader;
173
173
  var header = [];
174
- var splitRegExp = new RegExp(delimiter + "(?=(?:(?:[^" + qualifier + "]*" + qualifier + "){2})*[^" + qualifier + "]*$)");
175
- var processRow = function (row) {
176
- var arr = [];
177
- if (qualifier) {
178
- arr = row.split(splitRegExp);
179
- if (trimQualifier) {
180
- arr = arr.map(function (x) { return x[0] === qualifier && x[x.length - 1] === qualifier ? x.slice(1, -1) : x; });
181
- }
182
- }
183
- else {
184
- arr = row.split(delimiter);
185
- }
174
+ var processRow = function (_a) {
175
+ var raw = _a.raw, record = _a.record;
186
176
  if (hasHeader && !header.length) {
187
- header = arr.map(function (x) { return x.trim(); });
177
+ header = record.map(function (x) { return x.trim(); });
188
178
  }
189
179
  else {
190
180
  var obj = void 0;
191
181
  if (header.length) {
192
- obj = Object.fromEntries(header.map(function (x, i) { return [x, arr[i]]; }));
182
+ obj = Object.fromEntries(header.map(function (x, i) { return [x, record[i]]; }));
193
183
  }
194
- return { header: header, arr: arr, obj: obj, row: row };
184
+ return { header: header, arr: record, obj: obj, row: raw };
195
185
  }
196
186
  };
197
187
  var generator = function () {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bellboy",
3
- "version": "6.2.1",
3
+ "version": "6.3.1",
4
4
  "description": "Highly performant JavaScript data stream ETL engine.",
5
5
  "main": "lib/index",
6
6
  "types": "lib/index",
@@ -47,6 +47,7 @@
47
47
  "dependencies": {
48
48
  "async-mqtt": "^2.2.1",
49
49
  "axios": "^0.21.1",
50
+ "csv": "^6.0.5",
50
51
  "mssql": "^6.3.1",
51
52
  "pg": "8.0.3",
52
53
  "pg-promise": "10.5.3",