bellboy 6.7.3 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -19
- package/lib/destinations/base/database-destination.d.ts +1 -2
- package/lib/destinations/base/database-destination.js +0 -1
- package/lib/destinations/mssql-destination.d.ts +2 -2
- package/lib/destinations/mssql-destination.js +40 -41
- package/lib/destinations/postgres-destination.d.ts +2 -1
- package/lib/destinations/postgres-destination.js +1 -0
- package/lib/job.d.ts +1 -0
- package/lib/job.js +13 -7
- package/lib/processors/base/database-processor.d.ts +0 -1
- package/lib/processors/base/database-processor.js +0 -4
- package/lib/processors/mssql-processor.d.ts +2 -1
- package/lib/processors/mssql-processor.js +3 -1
- package/lib/processors/postgres-processor.d.ts +2 -1
- package/lib/processors/postgres-processor.js +3 -1
- package/lib/types.d.ts +18 -5
- package/lib/utils.js +68 -44
- package/package.json +8 -2
package/README.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# bellboy
|
|
1
|
+
# bellboy  [](https://codecov.io/gh/Claviz/bellboy) 
|
|
2
2
|
|
|
3
3
|
Highly performant JavaScript data stream ETL engine.
|
|
4
4
|
|
|
@@ -14,6 +14,12 @@ Before install, make sure you are using [latest](https://nodejs.org/en/download/
|
|
|
14
14
|
npm install bellboy
|
|
15
15
|
```
|
|
16
16
|
|
|
17
|
+
If you will be using `bellboy` with the native [msnodesqlv8][msnodesqlv8-url] driver, add it as a dependency.
|
|
18
|
+
|
|
19
|
+
```
|
|
20
|
+
npm install msnodesqlv8
|
|
21
|
+
```
|
|
22
|
+
|
|
17
23
|
## Example
|
|
18
24
|
|
|
19
25
|
This example shows how `bellboy` can extract rows from the [Excel file](#excel-processor), modify it on the fly, load to the [Postgres database](#postgres-destination), move processed file to the other folder and process remaining files.
|
|
@@ -115,7 +121,7 @@ const job = new bellboy.Job(
|
|
|
115
121
|
|
|
116
122
|
Event listeners, which can be registered with `job.on` or `job.onAny` methods, allow you to listen to specific events in the job lifecycle and to interact with them.
|
|
117
123
|
|
|
118
|
-
-
|
|
124
|
+
- When multiple listeners are registered for a single event, those added by [reporters](#reporters) will be executed first, followed by the order of registration for the remaining listeners.
|
|
119
125
|
- Job always waits for the code inside a listener to complete.
|
|
120
126
|
- Any error thrown inside a listener will be ignored and warning message will be printed out.
|
|
121
127
|
- `job.stop()` method can be used inside a listener to stop job execution and throw an error if needed.
|
|
@@ -137,21 +143,24 @@ job.on("startProcessingStream", async (...args: any) => {
|
|
|
137
143
|
```
|
|
138
144
|
|
|
139
145
|
```ts
|
|
140
|
-
job.on(
|
|
141
|
-
|
|
146
|
+
job.on("startProcessingRow", async (row: any) => {
|
|
147
|
+
// Row has been received and is about to be processed inside `recordGenerator` method.
|
|
142
148
|
});
|
|
143
149
|
```
|
|
144
150
|
|
|
145
151
|
```ts
|
|
146
|
-
job.on(
|
|
147
|
-
|
|
152
|
+
job.on("rowGenerated", async (destinationIndex: number, generatedRow: any) => {
|
|
153
|
+
// Row has been generated using `recordGenerator` method.
|
|
148
154
|
});
|
|
149
155
|
```
|
|
150
156
|
|
|
151
157
|
```ts
|
|
152
|
-
job.on(
|
|
158
|
+
job.on(
|
|
159
|
+
"rowGenerationError",
|
|
160
|
+
async (destinationIndex: number, row: any, error: any) => {
|
|
153
161
|
// Record generation (`recordGenerator` method) has thrown an error.
|
|
154
|
-
}
|
|
162
|
+
}
|
|
163
|
+
);
|
|
155
164
|
```
|
|
156
165
|
|
|
157
166
|
```ts
|
|
@@ -297,8 +306,8 @@ Each processor in `bellboy` is a class which has a single responsibility of proc
|
|
|
297
306
|
- [ExcelProcessor](#excel-processor) processes **XLSX** file data from the file system.
|
|
298
307
|
- [JsonProcessor](#json-processor) processes **JSON** file data from the file system.
|
|
299
308
|
- [DelimitedProcessor](#delimited-processor) processes files with **delimited data** from the file system.
|
|
300
|
-
- [PostgresProcessor](#
|
|
301
|
-
- [MssqlProcessor](#
|
|
309
|
+
- [PostgresProcessor](#postgres-processor) processes data received from a **PostgreSQL** SELECT.
|
|
310
|
+
- [MssqlProcessor](#mssql-processor) processes data received from a **MSSQL** SELECT.
|
|
302
311
|
- [DynamicProcessor](#dynamic-processor) processes **dynamically generated** data.
|
|
303
312
|
- [TailProcessor](#tail-processor) processes **new lines** added to the file.
|
|
304
313
|
|
|
@@ -500,9 +509,9 @@ Watches for file changes and outputs last part of file as soon as new lines are
|
|
|
500
509
|
Name of the file the data came from.
|
|
501
510
|
- **data** `string`
|
|
502
511
|
|
|
503
|
-
###
|
|
512
|
+
### PostgresProcessor <div id='postgres-processor'/>
|
|
504
513
|
|
|
505
|
-
Processes `SELECT` query row by row.
|
|
514
|
+
Processes a PostgreSQL `SELECT` query row by row.
|
|
506
515
|
|
|
507
516
|
#### Options
|
|
508
517
|
|
|
@@ -512,16 +521,48 @@ Processes `SELECT` query row by row. There are two database processors - `Postgr
|
|
|
512
521
|
- **connection** `object` `required`
|
|
513
522
|
- **user**
|
|
514
523
|
- **password**
|
|
515
|
-
- **server**\
|
|
516
|
-
Used with `MssqlProcessor`.
|
|
517
524
|
- **host**
|
|
518
|
-
Used with `PostgresProcessor`.
|
|
519
525
|
- **port**
|
|
520
526
|
- **database**
|
|
521
|
-
- **schema
|
|
522
|
-
|
|
527
|
+
- **schema**
|
|
528
|
+
|
|
529
|
+
### MssqlProcessor <div id='mssql-processor'/>
|
|
530
|
+
|
|
531
|
+
Processes a MSSQL `SELECT` query row by row.
|
|
532
|
+
|
|
533
|
+
#### Options
|
|
534
|
+
|
|
535
|
+
- [Processor options](#processor-options)
|
|
536
|
+
- **query** `string` `required`\
|
|
537
|
+
Query to execute.
|
|
538
|
+
- **connection** `object` `required`
|
|
539
|
+
- **user**
|
|
540
|
+
- **password**
|
|
541
|
+
- **server**
|
|
542
|
+
- **port**
|
|
543
|
+
- **database**
|
|
523
544
|
- **driver**\
|
|
524
|
-
|
|
545
|
+
Optional [mssql][mssql-url] TDS driver; defaults to the pure JavaScript [Tedious][tedious-url] driver.
|
|
546
|
+
|
|
547
|
+
#### Usage
|
|
548
|
+
|
|
549
|
+
Here is an example of how to configure `MssqlProcessor` with a native TDS driver instead of the default pure JavasScript Tedious driver.
|
|
550
|
+
|
|
551
|
+
```javascript
|
|
552
|
+
const nativeDriver: ITdsDriver = await import('mssql/msnodesqlV8');
|
|
553
|
+
const connection: IMssqlDbConnection = {
|
|
554
|
+
user: 'user',
|
|
555
|
+
password: 'password',
|
|
556
|
+
server: 'server',
|
|
557
|
+
database: 'database',
|
|
558
|
+
driver: nativeDriver
|
|
559
|
+
};
|
|
560
|
+
const source = new MssqlProcessor({ connection, query: 'select * from orders' });
|
|
561
|
+
```
|
|
562
|
+
|
|
563
|
+
In previous versions of `bellboy`, `connection.driver` was a `string` parameter.
|
|
564
|
+
|
|
565
|
+
[More usage examples](tests/mssql-source.spec.ts)
|
|
525
566
|
|
|
526
567
|
### DynamicProcessor <div id='dynamic-processor'/>
|
|
527
568
|
|
|
@@ -633,6 +674,26 @@ Inserts data to MSSQL.
|
|
|
633
674
|
- **password**
|
|
634
675
|
- **server**
|
|
635
676
|
- **database**
|
|
677
|
+
- **driver** \
|
|
678
|
+
Optional [mssql][mssql-url] TDS driver; defaults to the pure JavaScript [Tedious][tedious-url] driver.
|
|
679
|
+
|
|
680
|
+
#### Usage
|
|
681
|
+
|
|
682
|
+
Here is an example of how to configure `MssqlDestination` with a native TDS driver instead of the default pure JavasScript Tedious driver.
|
|
683
|
+
|
|
684
|
+
```javascript
|
|
685
|
+
const nativeDriver: ITdsDriver = await import('mssql/msnodesqlV8');
|
|
686
|
+
const connection: IMssqlDbConnection = {
|
|
687
|
+
user: 'user',
|
|
688
|
+
password: 'password',
|
|
689
|
+
server: 'server',
|
|
690
|
+
database: 'database',
|
|
691
|
+
driver: nativeDriver
|
|
692
|
+
};
|
|
693
|
+
const sink = new MssqlDestination({ connection, table: 'orders', batchSize: 1000 });
|
|
694
|
+
```
|
|
695
|
+
|
|
696
|
+
[More usage examples](tests/mssql-destination.spec.ts)
|
|
636
697
|
|
|
637
698
|
## Extendability
|
|
638
699
|
|
|
@@ -676,7 +737,9 @@ class CustomDestination extends bellboy.Destination {
|
|
|
676
737
|
|
|
677
738
|
[Official stdout reporter](https://github.com/claviz/bellboy-stdout-reporter)
|
|
678
739
|
|
|
679
|
-
Reporter is a job wrapper which can operate with [job instance](#job) (for example, listen to events using job `on` method). To create a new reporter, you must extend `Reporter` class and implement `report` function, which will be executed during job instance initialization.
|
|
740
|
+
Reporter is a job wrapper which can operate with [job instance](#job) (for example, listen to events using job `on` method). To create a new reporter, you must extend `Reporter` class and implement `report` function, which will be executed during job instance initialization.
|
|
741
|
+
Reporter event listeners (`on`, `onAny`) are added before any other user-defined listeners.
|
|
742
|
+
This function accepts one parameter:
|
|
680
743
|
|
|
681
744
|
- **job** `Job` `required`\
|
|
682
745
|
[Job](#job) instance
|
|
@@ -694,3 +757,7 @@ class CustomReporter extends bellboy.Reporter {
|
|
|
694
757
|
## Testing
|
|
695
758
|
|
|
696
759
|
Tests can be run by using `docker-compose up --abort-on-container-exit --exit-code-from test --build test` command.
|
|
760
|
+
|
|
761
|
+
[mssql-url]: https://github.com/tediousjs/node-mssql
|
|
762
|
+
[tedious-url]: https://www.npmjs.com/package/tedious
|
|
763
|
+
[msnodesqlv8-url]: https://www.npmjs.com/package/msnodesqlv8
|
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import { IDatabaseDestinationConfig
|
|
1
|
+
import { IDatabaseDestinationConfig } from '../../types';
|
|
2
2
|
import { Destination } from './destination';
|
|
3
3
|
export declare abstract class DatabaseDestination extends Destination {
|
|
4
4
|
protected table: string;
|
|
5
|
-
protected connection: IDbConnection;
|
|
6
5
|
constructor(config: IDatabaseDestinationConfig);
|
|
7
6
|
loadBatch(data: any[]): Promise<void>;
|
|
8
7
|
}
|
|
@@ -56,7 +56,6 @@ var DatabaseDestination = /** @class */ (function (_super) {
|
|
|
56
56
|
function DatabaseDestination(config) {
|
|
57
57
|
var _this = _super.call(this, config) || this;
|
|
58
58
|
_this.table = config.table;
|
|
59
|
-
_this.connection = config.connection;
|
|
60
59
|
return _this;
|
|
61
60
|
}
|
|
62
61
|
DatabaseDestination.prototype.loadBatch = function (data) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { IMssqlDestinationConfig } from '../types';
|
|
1
|
+
import { IMssqlDestinationConfig, IMssqlDbConnection } from '../types';
|
|
2
2
|
import { DatabaseDestination } from './base/database-destination';
|
|
3
3
|
export declare class MssqlDestination extends DatabaseDestination {
|
|
4
|
-
|
|
4
|
+
protected connection: IMssqlDbConnection;
|
|
5
5
|
constructor(config: IMssqlDestinationConfig);
|
|
6
6
|
loadBatch(data: any[]): Promise<void>;
|
|
7
7
|
}
|
|
@@ -85,34 +85,33 @@ var MssqlDestination = /** @class */ (function (_super) {
|
|
|
85
85
|
__extends(MssqlDestination, _super);
|
|
86
86
|
function MssqlDestination(config) {
|
|
87
87
|
var _this = _super.call(this, config) || this;
|
|
88
|
-
_this.
|
|
88
|
+
_this.connection = config.connection;
|
|
89
89
|
return _this;
|
|
90
90
|
}
|
|
91
91
|
MssqlDestination.prototype.loadBatch = function (data) {
|
|
92
|
+
var _a;
|
|
92
93
|
return __awaiter(this, void 0, void 0, function () {
|
|
93
|
-
var sql,
|
|
94
|
-
var
|
|
95
|
-
return __generator(this, function (
|
|
96
|
-
switch (
|
|
94
|
+
var sql, _b, pool, db, query, table, columns, _i, data_1, x, args, ii, transaction, err_1;
|
|
95
|
+
var _c;
|
|
96
|
+
return __generator(this, function (_d) {
|
|
97
|
+
switch (_d.label) {
|
|
97
98
|
case 0:
|
|
98
|
-
if (!(this.driver
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
99
|
+
if (!((_a = this.connection.driver) !== null && _a !== void 0)) return [3 /*break*/, 1];
|
|
100
|
+
_b = _a;
|
|
101
|
+
return [3 /*break*/, 3];
|
|
102
|
+
case 1: return [4 /*yield*/, Promise.resolve().then(function () { return __importStar(require('mssql')); })];
|
|
103
|
+
case 2:
|
|
104
|
+
_b = _d.sent();
|
|
105
|
+
_d.label = 3;
|
|
104
106
|
case 3:
|
|
105
|
-
|
|
106
|
-
_c.label = 4;
|
|
107
|
-
case 4:
|
|
108
|
-
sql = _a;
|
|
107
|
+
sql = _b;
|
|
109
108
|
pool = new sql.ConnectionPool(__assign({}, this.connection));
|
|
110
109
|
return [4 /*yield*/, pool.connect()];
|
|
111
|
-
case
|
|
112
|
-
db =
|
|
110
|
+
case 4:
|
|
111
|
+
db = _d.sent();
|
|
113
112
|
return [4 /*yield*/, db.request().query("SELECT TOP(0) * FROM " + this.table)];
|
|
114
|
-
case
|
|
115
|
-
query =
|
|
113
|
+
case 5:
|
|
114
|
+
query = _d.sent();
|
|
116
115
|
table = query.recordset.toTable(this.table);
|
|
117
116
|
columns = table.columns.map(function (x) { return x.name; });
|
|
118
117
|
for (_i = 0, data_1 = data; _i < data_1.length; _i++) {
|
|
@@ -121,36 +120,36 @@ var MssqlDestination = /** @class */ (function (_super) {
|
|
|
121
120
|
for (ii = 0; ii < columns.length; ii++) {
|
|
122
121
|
args.push(x[columns[ii]]);
|
|
123
122
|
}
|
|
124
|
-
(
|
|
123
|
+
(_c = table.rows).add.apply(_c, args);
|
|
125
124
|
}
|
|
126
125
|
transaction = new sql.Transaction(db);
|
|
127
|
-
|
|
128
|
-
case
|
|
129
|
-
|
|
126
|
+
_d.label = 6;
|
|
127
|
+
case 6:
|
|
128
|
+
_d.trys.push([6, 10, , 14]);
|
|
130
129
|
return [4 /*yield*/, transaction.begin()];
|
|
131
|
-
case
|
|
132
|
-
|
|
130
|
+
case 7:
|
|
131
|
+
_d.sent();
|
|
133
132
|
return [4 /*yield*/, transaction.request().bulk(table)];
|
|
134
|
-
case
|
|
135
|
-
|
|
133
|
+
case 8:
|
|
134
|
+
_d.sent();
|
|
136
135
|
return [4 /*yield*/, transaction.commit()];
|
|
136
|
+
case 9:
|
|
137
|
+
_d.sent();
|
|
138
|
+
return [3 /*break*/, 14];
|
|
137
139
|
case 10:
|
|
138
|
-
|
|
139
|
-
return [3 /*break*/,
|
|
140
|
-
case 11:
|
|
141
|
-
err_1 = _c.sent();
|
|
142
|
-
if (!!transaction._aborted) return [3 /*break*/, 13];
|
|
140
|
+
err_1 = _d.sent();
|
|
141
|
+
if (!!transaction._aborted) return [3 /*break*/, 12];
|
|
143
142
|
return [4 /*yield*/, transaction.rollback()];
|
|
144
|
-
case
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
case
|
|
148
|
-
case
|
|
149
|
-
|
|
143
|
+
case 11:
|
|
144
|
+
_d.sent();
|
|
145
|
+
_d.label = 12;
|
|
146
|
+
case 12: return [4 /*yield*/, pool.close()];
|
|
147
|
+
case 13:
|
|
148
|
+
_d.sent();
|
|
150
149
|
throw err_1;
|
|
151
|
-
case
|
|
152
|
-
case
|
|
153
|
-
|
|
150
|
+
case 14: return [4 /*yield*/, pool.close()];
|
|
151
|
+
case 15:
|
|
152
|
+
_d.sent();
|
|
154
153
|
return [2 /*return*/];
|
|
155
154
|
}
|
|
156
155
|
});
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { IPostgresDestinationConfig } from '../types';
|
|
1
|
+
import { IPostgresDestinationConfig, IPostgresDbConnection } from '../types';
|
|
2
2
|
import { DatabaseDestination } from './base/database-destination';
|
|
3
3
|
export declare class PostgresDestination extends DatabaseDestination {
|
|
4
|
+
protected connection: IPostgresDbConnection;
|
|
4
5
|
protected upsertConstraints: string[] | undefined;
|
|
5
6
|
constructor(config: IPostgresDestinationConfig);
|
|
6
7
|
loadBatch(data: any[]): Promise<void>;
|
|
@@ -57,6 +57,7 @@ var PostgresDestination = /** @class */ (function (_super) {
|
|
|
57
57
|
__extends(PostgresDestination, _super);
|
|
58
58
|
function PostgresDestination(config) {
|
|
59
59
|
var _this = _super.call(this, config) || this;
|
|
60
|
+
_this.connection = config.connection;
|
|
60
61
|
_this.upsertConstraints = config.upsertConstraints;
|
|
61
62
|
return _this;
|
|
62
63
|
}
|
package/lib/job.d.ts
CHANGED
|
@@ -16,6 +16,7 @@ export declare class Job implements IJob {
|
|
|
16
16
|
protected stopped: boolean;
|
|
17
17
|
protected jobId: string;
|
|
18
18
|
protected jobName?: string;
|
|
19
|
+
private settingReporterEvents;
|
|
19
20
|
constructor(processor: IProcessor, destinations: IDestination[], config?: IJobConfig);
|
|
20
21
|
stop(message?: string): void;
|
|
21
22
|
run(): Promise<void>;
|
package/lib/job.js
CHANGED
|
@@ -63,6 +63,7 @@ var Job = /** @class */ (function () {
|
|
|
63
63
|
this.anyEvent = [];
|
|
64
64
|
this.stopped = false;
|
|
65
65
|
this.jobId = shortid_1.generate();
|
|
66
|
+
this.settingReporterEvents = false;
|
|
66
67
|
this.jobName = config === null || config === void 0 ? void 0 : config.jobName;
|
|
67
68
|
this.reporters = config.reporters || [];
|
|
68
69
|
}
|
|
@@ -74,6 +75,7 @@ var Job = /** @class */ (function () {
|
|
|
74
75
|
return __generator(this, function (_a) {
|
|
75
76
|
switch (_a.label) {
|
|
76
77
|
case 0:
|
|
78
|
+
this.settingReporterEvents = true;
|
|
77
79
|
i = 0;
|
|
78
80
|
_a.label = 1;
|
|
79
81
|
case 1:
|
|
@@ -86,6 +88,7 @@ var Job = /** @class */ (function () {
|
|
|
86
88
|
i++;
|
|
87
89
|
return [3 /*break*/, 1];
|
|
88
90
|
case 4:
|
|
91
|
+
this.settingReporterEvents = false;
|
|
89
92
|
this.stop = function (message) {
|
|
90
93
|
_this.stopped = true;
|
|
91
94
|
if (readStream && (readStream instanceof stream_1.Readable)) {
|
|
@@ -278,16 +281,19 @@ var Job = /** @class */ (function () {
|
|
|
278
281
|
});
|
|
279
282
|
};
|
|
280
283
|
Job.prototype.on = function (eventName, event, extendedEvent) {
|
|
281
|
-
var
|
|
282
|
-
if (!
|
|
283
|
-
this.events[eventName] = [
|
|
284
|
-
}
|
|
285
|
-
else {
|
|
286
|
-
this.events[eventName].push({ event: event, extendedEvent: extendedEvent, });
|
|
284
|
+
var eventEntry = { event: event, extendedEvent: extendedEvent };
|
|
285
|
+
if (!this.events[eventName]) {
|
|
286
|
+
this.events[eventName] = [];
|
|
287
287
|
}
|
|
288
|
+
this.settingReporterEvents
|
|
289
|
+
? this.events[eventName].unshift(eventEntry)
|
|
290
|
+
: this.events[eventName].push(eventEntry);
|
|
288
291
|
};
|
|
289
292
|
Job.prototype.onAny = function (event, extendedEvent) {
|
|
290
|
-
|
|
293
|
+
var eventEntry = { event: event, extendedEvent: extendedEvent };
|
|
294
|
+
this.settingReporterEvents
|
|
295
|
+
? this.anyEvent.unshift(eventEntry)
|
|
296
|
+
: this.anyEvent.push(eventEntry);
|
|
291
297
|
};
|
|
292
298
|
Job.prototype.flushRows = function (rows) {
|
|
293
299
|
return __awaiter(this, void 0, void 0, function () {
|
|
@@ -2,6 +2,5 @@ import { Processor } from './processor';
|
|
|
2
2
|
import { IDatabaseProcessorConfig } from '../../types';
|
|
3
3
|
export declare abstract class DatabaseProcessor extends Processor {
|
|
4
4
|
protected query: string;
|
|
5
|
-
protected connection: any;
|
|
6
5
|
constructor(config: IDatabaseProcessorConfig);
|
|
7
6
|
}
|
|
@@ -23,10 +23,6 @@ var DatabaseProcessor = /** @class */ (function (_super) {
|
|
|
23
23
|
throw new Error("No query specified.");
|
|
24
24
|
}
|
|
25
25
|
_this.query = config.query;
|
|
26
|
-
if (!config.connection) {
|
|
27
|
-
throw new Error("No connection specified.");
|
|
28
|
-
}
|
|
29
|
-
_this.connection = config.connection;
|
|
30
26
|
return _this;
|
|
31
27
|
}
|
|
32
28
|
return DatabaseProcessor;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { IMssqlProcessorConfig, processStream } from '../types';
|
|
1
|
+
import { IMssqlProcessorConfig, IMssqlDbConnection, processStream } from '../types';
|
|
2
2
|
import { DatabaseProcessor } from './base/database-processor';
|
|
3
3
|
export declare class MssqlProcessor extends DatabaseProcessor {
|
|
4
|
+
protected connection: IMssqlDbConnection;
|
|
4
5
|
constructor(config: IMssqlProcessorConfig);
|
|
5
6
|
process(processStream: processStream): Promise<void>;
|
|
6
7
|
}
|
|
@@ -56,7 +56,9 @@ var database_processor_1 = require("./base/database-processor");
|
|
|
56
56
|
var MssqlProcessor = /** @class */ (function (_super) {
|
|
57
57
|
__extends(MssqlProcessor, _super);
|
|
58
58
|
function MssqlProcessor(config) {
|
|
59
|
-
|
|
59
|
+
var _this = _super.call(this, config) || this;
|
|
60
|
+
_this.connection = config.connection;
|
|
61
|
+
return _this;
|
|
60
62
|
}
|
|
61
63
|
MssqlProcessor.prototype.process = function (processStream) {
|
|
62
64
|
return __awaiter(this, void 0, void 0, function () {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { IPostgresProcessorConfig, processStream } from '../types';
|
|
1
|
+
import { IPostgresProcessorConfig, IPostgresDbConnection, processStream } from '../types';
|
|
2
2
|
import { DatabaseProcessor } from './base/database-processor';
|
|
3
3
|
export declare class PostgresProcessor extends DatabaseProcessor {
|
|
4
|
+
protected connection: IPostgresDbConnection;
|
|
4
5
|
constructor(config: IPostgresProcessorConfig);
|
|
5
6
|
process(processStream: processStream): Promise<void>;
|
|
6
7
|
}
|
|
@@ -56,7 +56,9 @@ var QueryStream = require('pg-query-stream');
|
|
|
56
56
|
var PostgresProcessor = /** @class */ (function (_super) {
|
|
57
57
|
__extends(PostgresProcessor, _super);
|
|
58
58
|
function PostgresProcessor(config) {
|
|
59
|
-
|
|
59
|
+
var _this = _super.call(this, config) || this;
|
|
60
|
+
_this.connection = config.connection;
|
|
61
|
+
return _this;
|
|
60
62
|
}
|
|
61
63
|
PostgresProcessor.prototype.process = function (processStream) {
|
|
62
64
|
return __awaiter(this, void 0, void 0, function () {
|
package/lib/types.d.ts
CHANGED
|
@@ -9,15 +9,26 @@ export interface AuthorizationRequest {
|
|
|
9
9
|
destinationField: string;
|
|
10
10
|
prefix?: string;
|
|
11
11
|
}
|
|
12
|
-
export interface
|
|
12
|
+
export interface IPostgresDbConnection {
|
|
13
13
|
user?: string;
|
|
14
14
|
password?: string;
|
|
15
|
-
server?: string;
|
|
16
15
|
host?: string;
|
|
17
16
|
database?: string;
|
|
18
17
|
schema?: string;
|
|
19
|
-
driver?: 'tedious' | 'msnodesqlv8';
|
|
20
18
|
}
|
|
19
|
+
export interface ITdsDriver {
|
|
20
|
+
ConnectionPool: any;
|
|
21
|
+
Transaction: any;
|
|
22
|
+
}
|
|
23
|
+
export interface IMssqlDbConnection {
|
|
24
|
+
user?: string;
|
|
25
|
+
password?: string;
|
|
26
|
+
server?: string;
|
|
27
|
+
database?: string;
|
|
28
|
+
driver?: ITdsDriver;
|
|
29
|
+
options?: any;
|
|
30
|
+
}
|
|
31
|
+
export declare type IDbConnection = IPostgresDbConnection | IMssqlDbConnection;
|
|
21
32
|
export interface IReporter {
|
|
22
33
|
report(job: IJob): Promise<void> | void;
|
|
23
34
|
}
|
|
@@ -68,12 +79,13 @@ export interface IDestinationConfig {
|
|
|
68
79
|
}
|
|
69
80
|
export interface IDatabaseDestinationConfig extends IDestinationConfig {
|
|
70
81
|
table: string;
|
|
71
|
-
connection: IDbConnection;
|
|
72
82
|
}
|
|
73
83
|
export interface IPostgresDestinationConfig extends IDatabaseDestinationConfig {
|
|
84
|
+
connection: IPostgresDbConnection;
|
|
74
85
|
upsertConstraints?: string[];
|
|
75
86
|
}
|
|
76
87
|
export interface IMssqlDestinationConfig extends IDatabaseDestinationConfig {
|
|
88
|
+
connection: IMssqlDbConnection;
|
|
77
89
|
}
|
|
78
90
|
export interface IHttpDestinationConfig extends IDestinationConfig {
|
|
79
91
|
request: AxiosRequestConfig;
|
|
@@ -103,7 +115,6 @@ export interface IDelimitedHttpProcessorConfig extends IHttpProcessorConfig {
|
|
|
103
115
|
qualifier?: string;
|
|
104
116
|
}
|
|
105
117
|
export interface IDatabaseProcessorConfig extends IProcessorConfig {
|
|
106
|
-
connection: any;
|
|
107
118
|
query: string;
|
|
108
119
|
}
|
|
109
120
|
export interface IJsonProcessorConfig extends IDirectoryProcessorConfig {
|
|
@@ -116,8 +127,10 @@ export interface IDelimitedProcessorConfig extends IDirectoryProcessorConfig {
|
|
|
116
127
|
qualifier?: string;
|
|
117
128
|
}
|
|
118
129
|
export interface IMssqlProcessorConfig extends IDatabaseProcessorConfig {
|
|
130
|
+
connection: IMssqlDbConnection;
|
|
119
131
|
}
|
|
120
132
|
export interface IPostgresProcessorConfig extends IDatabaseProcessorConfig {
|
|
133
|
+
connection: IPostgresDbConnection;
|
|
121
134
|
}
|
|
122
135
|
export interface ITailProcessorConfig extends IDirectoryProcessorConfig {
|
|
123
136
|
fromBeginning?: boolean;
|
package/lib/utils.js
CHANGED
|
@@ -92,68 +92,94 @@ exports.applyHttpAuthorization = exports.getDelimitedGenerator = exports.removeC
|
|
|
92
92
|
var axios_1 = __importDefault(require("axios"));
|
|
93
93
|
var stream_1 = require("stream");
|
|
94
94
|
var cachedDbConnections = new Map();
|
|
95
|
+
function getCachedDbConnection(databaseConfig) {
|
|
96
|
+
var dbKey = getDbKey(databaseConfig);
|
|
97
|
+
return cachedDbConnections.get(dbKey);
|
|
98
|
+
}
|
|
99
|
+
function getDbKey(databaseConfig) {
|
|
100
|
+
if ('driver' in databaseConfig) {
|
|
101
|
+
var configWithoutDriver = __assign({}, databaseConfig);
|
|
102
|
+
delete configWithoutDriver.driver;
|
|
103
|
+
return JSON.stringify(configWithoutDriver);
|
|
104
|
+
}
|
|
105
|
+
return JSON.stringify(databaseConfig);
|
|
106
|
+
}
|
|
107
|
+
function setCachedDbConnection(databaseConfig, db, close) {
|
|
108
|
+
var dbKey = getDbKey(databaseConfig);
|
|
109
|
+
cachedDbConnections.set(dbKey, { db: db, close: close });
|
|
110
|
+
}
|
|
111
|
+
function unsetCachedDbConnection(databaseConfig) {
|
|
112
|
+
var dbKey = getDbKey(databaseConfig);
|
|
113
|
+
cachedDbConnections.delete(dbKey);
|
|
114
|
+
}
|
|
95
115
|
function getDb(databaseConfig, dbType) {
|
|
96
116
|
return __awaiter(this, void 0, void 0, function () {
|
|
97
|
-
var
|
|
98
|
-
return __generator(this, function (
|
|
99
|
-
switch (
|
|
117
|
+
var dbConnection;
|
|
118
|
+
return __generator(this, function (_a) {
|
|
119
|
+
switch (_a.label) {
|
|
100
120
|
case 0:
|
|
101
|
-
|
|
102
|
-
if (
|
|
103
|
-
|
|
104
|
-
if (dbConnection) {
|
|
105
|
-
return [2 /*return*/, dbConnection.db];
|
|
106
|
-
}
|
|
121
|
+
dbConnection = getCachedDbConnection(databaseConfig);
|
|
122
|
+
if (dbConnection) {
|
|
123
|
+
return [2 /*return*/, dbConnection.db];
|
|
107
124
|
}
|
|
108
|
-
if (!(dbType === 'mssql')) return [3 /*break*/,
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
case
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
125
|
+
if (!(dbType === 'mssql')) return [3 /*break*/, 2];
|
|
126
|
+
return [4 /*yield*/, getMssqlDb(databaseConfig)];
|
|
127
|
+
case 1: return [2 /*return*/, _a.sent()];
|
|
128
|
+
case 2: return [2 /*return*/, getPostgresDb(databaseConfig)];
|
|
129
|
+
}
|
|
130
|
+
});
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
exports.getDb = getDb;
|
|
134
|
+
function getMssqlDb(databaseConfig) {
|
|
135
|
+
var _a;
|
|
136
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
137
|
+
var driver, _b, pool, db;
|
|
138
|
+
return __generator(this, function (_c) {
|
|
139
|
+
switch (_c.label) {
|
|
140
|
+
case 0:
|
|
141
|
+
if (!((_a = databaseConfig.driver) !== null && _a !== void 0)) return [3 /*break*/, 1];
|
|
142
|
+
_b = _a;
|
|
143
|
+
return [3 /*break*/, 3];
|
|
144
|
+
case 1: return [4 /*yield*/, Promise.resolve().then(function () { return __importStar(require('mssql')); })];
|
|
145
|
+
case 2:
|
|
146
|
+
_b = _c.sent();
|
|
147
|
+
_c.label = 3;
|
|
115
148
|
case 3:
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
case 4:
|
|
119
|
-
sql = _a;
|
|
120
|
-
pool = new sql.ConnectionPool(__assign({}, databaseConfig));
|
|
149
|
+
driver = _b;
|
|
150
|
+
pool = getMssqlDbPool(driver, databaseConfig);
|
|
121
151
|
return [4 /*yield*/, pool.connect()];
|
|
122
|
-
case
|
|
123
|
-
db =
|
|
124
|
-
|
|
125
|
-
db: db,
|
|
126
|
-
close: pool.close.bind(pool),
|
|
127
|
-
});
|
|
128
|
-
return [2 /*return*/, db];
|
|
129
|
-
case 6:
|
|
130
|
-
pgp = require('pg-promise')({ schema: databaseConfig.schema || 'public', });
|
|
131
|
-
db = pgp(databaseConfig);
|
|
132
|
-
cachedDbConnections.set(dbKey, {
|
|
133
|
-
db: db,
|
|
134
|
-
close: pgp.end,
|
|
135
|
-
});
|
|
152
|
+
case 4:
|
|
153
|
+
db = _c.sent();
|
|
154
|
+
setCachedDbConnection(databaseConfig, db, pool.close.bind(pool));
|
|
136
155
|
return [2 /*return*/, db];
|
|
137
156
|
}
|
|
138
157
|
});
|
|
139
158
|
});
|
|
140
159
|
}
|
|
141
|
-
|
|
142
|
-
;
|
|
160
|
+
function getMssqlDbPool(driver, databaseConfig) {
|
|
161
|
+
var poolConfig = __assign({}, databaseConfig);
|
|
162
|
+
delete poolConfig.driver;
|
|
163
|
+
return new driver.ConnectionPool(__assign({}, databaseConfig));
|
|
164
|
+
}
|
|
165
|
+
function getPostgresDb(databaseConfig) {
|
|
166
|
+
var pgp = require('pg-promise')({ schema: databaseConfig.schema || 'public' });
|
|
167
|
+
var db = pgp(databaseConfig);
|
|
168
|
+
setCachedDbConnection(databaseConfig, db, pgp.end);
|
|
169
|
+
return db;
|
|
170
|
+
}
|
|
143
171
|
function closeDbConnection(databaseConfig) {
|
|
144
172
|
return __awaiter(this, void 0, void 0, function () {
|
|
145
|
-
var
|
|
173
|
+
var dbConnection;
|
|
146
174
|
return __generator(this, function (_a) {
|
|
147
175
|
switch (_a.label) {
|
|
148
176
|
case 0:
|
|
149
|
-
|
|
150
|
-
if (!cachedDbConnections.has(dbKey)) return [3 /*break*/, 2];
|
|
151
|
-
dbConnection = cachedDbConnections.get(dbKey);
|
|
177
|
+
dbConnection = getCachedDbConnection(databaseConfig);
|
|
152
178
|
if (!dbConnection) return [3 /*break*/, 2];
|
|
153
179
|
return [4 /*yield*/, dbConnection.close()];
|
|
154
180
|
case 1:
|
|
155
181
|
_a.sent();
|
|
156
|
-
|
|
182
|
+
unsetCachedDbConnection(databaseConfig);
|
|
157
183
|
_a.label = 2;
|
|
158
184
|
case 2: return [2 /*return*/];
|
|
159
185
|
}
|
|
@@ -173,7 +199,6 @@ function getValueFromJSONChunk() {
|
|
|
173
199
|
});
|
|
174
200
|
}
|
|
175
201
|
exports.getValueFromJSONChunk = getValueFromJSONChunk;
|
|
176
|
-
;
|
|
177
202
|
function removeCircularReferencesFromChunk() {
|
|
178
203
|
return new stream_1.Transform({
|
|
179
204
|
objectMode: true,
|
|
@@ -196,7 +221,6 @@ function removeCircularReferencesFromChunk() {
|
|
|
196
221
|
});
|
|
197
222
|
}
|
|
198
223
|
exports.removeCircularReferencesFromChunk = removeCircularReferencesFromChunk;
|
|
199
|
-
;
|
|
200
224
|
function getDelimitedGenerator(_a) {
|
|
201
225
|
var readStream = _a.readStream, hasHeader = _a.hasHeader;
|
|
202
226
|
var header = [];
|
package/package.json
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bellboy",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "8.0.0",
|
|
4
4
|
"description": "Highly performant JavaScript data stream ETL engine.",
|
|
5
5
|
"main": "lib/index",
|
|
6
6
|
"types": "lib/index",
|
|
7
7
|
"scripts": {
|
|
8
8
|
"build": "tsc -p .",
|
|
9
9
|
"test": "jest --coverage",
|
|
10
|
+
"lint": "tslint -c tslint.json -p tsconfig.json",
|
|
10
11
|
"codecov": "codecov"
|
|
11
12
|
},
|
|
12
13
|
"repository": {
|
|
@@ -23,9 +24,14 @@
|
|
|
23
24
|
"url": "https://github.com/Claviz/bellboy/issues"
|
|
24
25
|
},
|
|
25
26
|
"homepage": "https://github.com/Claviz/bellboy#readme",
|
|
26
|
-
"
|
|
27
|
+
"peerDependencies": {
|
|
27
28
|
"msnodesqlv8": "^2.1.0"
|
|
28
29
|
},
|
|
30
|
+
"peerDependenciesMeta": {
|
|
31
|
+
"msnodesqlv8": {
|
|
32
|
+
"optional": true
|
|
33
|
+
}
|
|
34
|
+
},
|
|
29
35
|
"devDependencies": {
|
|
30
36
|
"@types/axios": "^0.14.0",
|
|
31
37
|
"@types/jest": "^25.2.1",
|