bellboy 7.0.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -12
- package/lib/destinations/base/database-destination.d.ts +1 -2
- package/lib/destinations/base/database-destination.js +0 -1
- package/lib/destinations/mssql-destination.d.ts +2 -2
- package/lib/destinations/mssql-destination.js +40 -41
- package/lib/destinations/postgres-destination.d.ts +2 -1
- package/lib/destinations/postgres-destination.js +1 -0
- package/lib/processors/base/database-processor.d.ts +0 -1
- package/lib/processors/base/database-processor.js +0 -4
- package/lib/processors/mssql-processor.d.ts +2 -1
- package/lib/processors/mssql-processor.js +3 -1
- package/lib/processors/postgres-processor.d.ts +2 -1
- package/lib/processors/postgres-processor.js +3 -1
- package/lib/types.d.ts +18 -5
- package/lib/utils.js +68 -44
- package/package.json +8 -2
package/README.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# bellboy
|
|
1
|
+
# bellboy  [](https://codecov.io/gh/Claviz/bellboy) 
|
|
2
2
|
|
|
3
3
|
Highly performant JavaScript data stream ETL engine.
|
|
4
4
|
|
|
@@ -14,6 +14,12 @@ Before install, make sure you are using [latest](https://nodejs.org/en/download/
|
|
|
14
14
|
npm install bellboy
|
|
15
15
|
```
|
|
16
16
|
|
|
17
|
+
If you will be using `bellboy` with the native [msnodesqlv8][msnodesqlv8-url] driver, add it as a dependency.
|
|
18
|
+
|
|
19
|
+
```
|
|
20
|
+
npm install msnodesqlv8
|
|
21
|
+
```
|
|
22
|
+
|
|
17
23
|
## Example
|
|
18
24
|
|
|
19
25
|
This example shows how `bellboy` can extract rows from the [Excel file](#excel-processor), modify it on the fly, load to the [Postgres database](#postgres-destination), move processed file to the other folder and process remaining files.
|
|
@@ -115,7 +121,7 @@ const job = new bellboy.Job(
|
|
|
115
121
|
|
|
116
122
|
Event listeners, which can be registered with `job.on` or `job.onAny` methods, allow you to listen to specific events in the job lifecycle and to interact with them.
|
|
117
123
|
|
|
118
|
-
-
|
|
124
|
+
- When multiple listeners are registered for a single event, those added by [reporters](#reporters) will be executed first, followed by the order of registration for the remaining listeners.
|
|
119
125
|
- Job always waits for the code inside a listener to complete.
|
|
120
126
|
- Any error thrown inside a listener will be ignored and warning message will be printed out.
|
|
121
127
|
- `job.stop()` method can be used inside a listener to stop job execution and throw an error if needed.
|
|
@@ -300,8 +306,8 @@ Each processor in `bellboy` is a class which has a single responsibility of proc
|
|
|
300
306
|
- [ExcelProcessor](#excel-processor) processes **XLSX** file data from the file system.
|
|
301
307
|
- [JsonProcessor](#json-processor) processes **JSON** file data from the file system.
|
|
302
308
|
- [DelimitedProcessor](#delimited-processor) processes files with **delimited data** from the file system.
|
|
303
|
-
- [PostgresProcessor](#
|
|
304
|
-
- [MssqlProcessor](#
|
|
309
|
+
- [PostgresProcessor](#postgres-processor) processes data received from a **PostgreSQL** SELECT.
|
|
310
|
+
- [MssqlProcessor](#mssql-processor) processes data received from a **MSSQL** SELECT.
|
|
305
311
|
- [DynamicProcessor](#dynamic-processor) processes **dynamically generated** data.
|
|
306
312
|
- [TailProcessor](#tail-processor) processes **new lines** added to the file.
|
|
307
313
|
|
|
@@ -503,9 +509,9 @@ Watches for file changes and outputs last part of file as soon as new lines are
|
|
|
503
509
|
Name of the file the data came from.
|
|
504
510
|
- **data** `string`
|
|
505
511
|
|
|
506
|
-
###
|
|
512
|
+
### PostgresProcessor <div id='postgres-processor'/>
|
|
507
513
|
|
|
508
|
-
Processes `SELECT` query row by row.
|
|
514
|
+
Processes a PostgreSQL `SELECT` query row by row.
|
|
509
515
|
|
|
510
516
|
#### Options
|
|
511
517
|
|
|
@@ -515,16 +521,48 @@ Processes `SELECT` query row by row. There are two database processors - `Postgr
|
|
|
515
521
|
- **connection** `object` `required`
|
|
516
522
|
- **user**
|
|
517
523
|
- **password**
|
|
518
|
-
- **server**\
|
|
519
|
-
Used with `MssqlProcessor`.
|
|
520
524
|
- **host**
|
|
521
|
-
Used with `PostgresProcessor`.
|
|
522
525
|
- **port**
|
|
523
526
|
- **database**
|
|
524
|
-
- **schema
|
|
525
|
-
|
|
527
|
+
- **schema**
|
|
528
|
+
|
|
529
|
+
### MssqlProcessor <div id='mssql-processor'/>
|
|
530
|
+
|
|
531
|
+
Processes a MSSQL `SELECT` query row by row.
|
|
532
|
+
|
|
533
|
+
#### Options
|
|
534
|
+
|
|
535
|
+
- [Processor options](#processor-options)
|
|
536
|
+
- **query** `string` `required`\
|
|
537
|
+
Query to execute.
|
|
538
|
+
- **connection** `object` `required`
|
|
539
|
+
- **user**
|
|
540
|
+
- **password**
|
|
541
|
+
- **server**
|
|
542
|
+
- **port**
|
|
543
|
+
- **database**
|
|
526
544
|
- **driver**\
|
|
527
|
-
|
|
545
|
+
Optional [mssql][mssql-url] TDS driver; defaults to the pure JavaScript [Tedious][tedious-url] driver.
|
|
546
|
+
|
|
547
|
+
#### Usage
|
|
548
|
+
|
|
549
|
+
Here is an example of how to configure `MssqlProcessor` with a native TDS driver instead of the default pure JavasScript Tedious driver.
|
|
550
|
+
|
|
551
|
+
```javascript
|
|
552
|
+
const nativeDriver: ITdsDriver = await import('mssql/msnodesqlV8');
|
|
553
|
+
const connection: IMssqlDbConnection = {
|
|
554
|
+
user: 'user',
|
|
555
|
+
password: 'password',
|
|
556
|
+
server: 'server',
|
|
557
|
+
database: 'database',
|
|
558
|
+
driver: nativeDriver
|
|
559
|
+
};
|
|
560
|
+
const source = new MssqlProcessor({ connection, query: 'select * from orders' });
|
|
561
|
+
```
|
|
562
|
+
|
|
563
|
+
In previous versions of `bellboy`, `connection.driver` was a `string` parameter.
|
|
564
|
+
|
|
565
|
+
[More usage examples](tests/mssql-source.spec.ts)
|
|
528
566
|
|
|
529
567
|
### DynamicProcessor <div id='dynamic-processor'/>
|
|
530
568
|
|
|
@@ -636,6 +674,26 @@ Inserts data to MSSQL.
|
|
|
636
674
|
- **password**
|
|
637
675
|
- **server**
|
|
638
676
|
- **database**
|
|
677
|
+
- **driver** \
|
|
678
|
+
Optional [mssql][mssql-url] TDS driver; defaults to the pure JavaScript [Tedious][tedious-url] driver.
|
|
679
|
+
|
|
680
|
+
#### Usage
|
|
681
|
+
|
|
682
|
+
Here is an example of how to configure `MssqlDestination` with a native TDS driver instead of the default pure JavasScript Tedious driver.
|
|
683
|
+
|
|
684
|
+
```javascript
|
|
685
|
+
const nativeDriver: ITdsDriver = await import('mssql/msnodesqlV8');
|
|
686
|
+
const connection: IMssqlDbConnection = {
|
|
687
|
+
user: 'user',
|
|
688
|
+
password: 'password',
|
|
689
|
+
server: 'server',
|
|
690
|
+
database: 'database',
|
|
691
|
+
driver: nativeDriver
|
|
692
|
+
};
|
|
693
|
+
const sink = new MssqlDestination({ connection, table: 'orders', batchSize: 1000 });
|
|
694
|
+
```
|
|
695
|
+
|
|
696
|
+
[More usage examples](tests/mssql-destination.spec.ts)
|
|
639
697
|
|
|
640
698
|
## Extendability
|
|
641
699
|
|
|
@@ -699,3 +757,7 @@ class CustomReporter extends bellboy.Reporter {
|
|
|
699
757
|
## Testing
|
|
700
758
|
|
|
701
759
|
Tests can be run by using `docker-compose up --abort-on-container-exit --exit-code-from test --build test` command.
|
|
760
|
+
|
|
761
|
+
[mssql-url]: https://github.com/tediousjs/node-mssql
|
|
762
|
+
[tedious-url]: https://www.npmjs.com/package/tedious
|
|
763
|
+
[msnodesqlv8-url]: https://www.npmjs.com/package/msnodesqlv8
|
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import { IDatabaseDestinationConfig
|
|
1
|
+
import { IDatabaseDestinationConfig } from '../../types';
|
|
2
2
|
import { Destination } from './destination';
|
|
3
3
|
export declare abstract class DatabaseDestination extends Destination {
|
|
4
4
|
protected table: string;
|
|
5
|
-
protected connection: IDbConnection;
|
|
6
5
|
constructor(config: IDatabaseDestinationConfig);
|
|
7
6
|
loadBatch(data: any[]): Promise<void>;
|
|
8
7
|
}
|
|
@@ -56,7 +56,6 @@ var DatabaseDestination = /** @class */ (function (_super) {
|
|
|
56
56
|
function DatabaseDestination(config) {
|
|
57
57
|
var _this = _super.call(this, config) || this;
|
|
58
58
|
_this.table = config.table;
|
|
59
|
-
_this.connection = config.connection;
|
|
60
59
|
return _this;
|
|
61
60
|
}
|
|
62
61
|
DatabaseDestination.prototype.loadBatch = function (data) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { IMssqlDestinationConfig } from '../types';
|
|
1
|
+
import { IMssqlDestinationConfig, IMssqlDbConnection } from '../types';
|
|
2
2
|
import { DatabaseDestination } from './base/database-destination';
|
|
3
3
|
export declare class MssqlDestination extends DatabaseDestination {
|
|
4
|
-
|
|
4
|
+
protected connection: IMssqlDbConnection;
|
|
5
5
|
constructor(config: IMssqlDestinationConfig);
|
|
6
6
|
loadBatch(data: any[]): Promise<void>;
|
|
7
7
|
}
|
|
@@ -85,34 +85,33 @@ var MssqlDestination = /** @class */ (function (_super) {
|
|
|
85
85
|
__extends(MssqlDestination, _super);
|
|
86
86
|
function MssqlDestination(config) {
|
|
87
87
|
var _this = _super.call(this, config) || this;
|
|
88
|
-
_this.
|
|
88
|
+
_this.connection = config.connection;
|
|
89
89
|
return _this;
|
|
90
90
|
}
|
|
91
91
|
MssqlDestination.prototype.loadBatch = function (data) {
|
|
92
|
+
var _a;
|
|
92
93
|
return __awaiter(this, void 0, void 0, function () {
|
|
93
|
-
var sql,
|
|
94
|
-
var
|
|
95
|
-
return __generator(this, function (
|
|
96
|
-
switch (
|
|
94
|
+
var sql, _b, pool, db, query, table, columns, _i, data_1, x, args, ii, transaction, err_1;
|
|
95
|
+
var _c;
|
|
96
|
+
return __generator(this, function (_d) {
|
|
97
|
+
switch (_d.label) {
|
|
97
98
|
case 0:
|
|
98
|
-
if (!(this.driver
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
99
|
+
if (!((_a = this.connection.driver) !== null && _a !== void 0)) return [3 /*break*/, 1];
|
|
100
|
+
_b = _a;
|
|
101
|
+
return [3 /*break*/, 3];
|
|
102
|
+
case 1: return [4 /*yield*/, Promise.resolve().then(function () { return __importStar(require('mssql')); })];
|
|
103
|
+
case 2:
|
|
104
|
+
_b = _d.sent();
|
|
105
|
+
_d.label = 3;
|
|
104
106
|
case 3:
|
|
105
|
-
|
|
106
|
-
_c.label = 4;
|
|
107
|
-
case 4:
|
|
108
|
-
sql = _a;
|
|
107
|
+
sql = _b;
|
|
109
108
|
pool = new sql.ConnectionPool(__assign({}, this.connection));
|
|
110
109
|
return [4 /*yield*/, pool.connect()];
|
|
111
|
-
case
|
|
112
|
-
db =
|
|
110
|
+
case 4:
|
|
111
|
+
db = _d.sent();
|
|
113
112
|
return [4 /*yield*/, db.request().query("SELECT TOP(0) * FROM " + this.table)];
|
|
114
|
-
case
|
|
115
|
-
query =
|
|
113
|
+
case 5:
|
|
114
|
+
query = _d.sent();
|
|
116
115
|
table = query.recordset.toTable(this.table);
|
|
117
116
|
columns = table.columns.map(function (x) { return x.name; });
|
|
118
117
|
for (_i = 0, data_1 = data; _i < data_1.length; _i++) {
|
|
@@ -121,36 +120,36 @@ var MssqlDestination = /** @class */ (function (_super) {
|
|
|
121
120
|
for (ii = 0; ii < columns.length; ii++) {
|
|
122
121
|
args.push(x[columns[ii]]);
|
|
123
122
|
}
|
|
124
|
-
(
|
|
123
|
+
(_c = table.rows).add.apply(_c, args);
|
|
125
124
|
}
|
|
126
125
|
transaction = new sql.Transaction(db);
|
|
127
|
-
|
|
128
|
-
case
|
|
129
|
-
|
|
126
|
+
_d.label = 6;
|
|
127
|
+
case 6:
|
|
128
|
+
_d.trys.push([6, 10, , 14]);
|
|
130
129
|
return [4 /*yield*/, transaction.begin()];
|
|
131
|
-
case
|
|
132
|
-
|
|
130
|
+
case 7:
|
|
131
|
+
_d.sent();
|
|
133
132
|
return [4 /*yield*/, transaction.request().bulk(table)];
|
|
134
|
-
case
|
|
135
|
-
|
|
133
|
+
case 8:
|
|
134
|
+
_d.sent();
|
|
136
135
|
return [4 /*yield*/, transaction.commit()];
|
|
136
|
+
case 9:
|
|
137
|
+
_d.sent();
|
|
138
|
+
return [3 /*break*/, 14];
|
|
137
139
|
case 10:
|
|
138
|
-
|
|
139
|
-
return [3 /*break*/,
|
|
140
|
-
case 11:
|
|
141
|
-
err_1 = _c.sent();
|
|
142
|
-
if (!!transaction._aborted) return [3 /*break*/, 13];
|
|
140
|
+
err_1 = _d.sent();
|
|
141
|
+
if (!!transaction._aborted) return [3 /*break*/, 12];
|
|
143
142
|
return [4 /*yield*/, transaction.rollback()];
|
|
144
|
-
case
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
case
|
|
148
|
-
case
|
|
149
|
-
|
|
143
|
+
case 11:
|
|
144
|
+
_d.sent();
|
|
145
|
+
_d.label = 12;
|
|
146
|
+
case 12: return [4 /*yield*/, pool.close()];
|
|
147
|
+
case 13:
|
|
148
|
+
_d.sent();
|
|
150
149
|
throw err_1;
|
|
151
|
-
case
|
|
152
|
-
case
|
|
153
|
-
|
|
150
|
+
case 14: return [4 /*yield*/, pool.close()];
|
|
151
|
+
case 15:
|
|
152
|
+
_d.sent();
|
|
154
153
|
return [2 /*return*/];
|
|
155
154
|
}
|
|
156
155
|
});
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { IPostgresDestinationConfig } from '../types';
|
|
1
|
+
import { IPostgresDestinationConfig, IPostgresDbConnection } from '../types';
|
|
2
2
|
import { DatabaseDestination } from './base/database-destination';
|
|
3
3
|
export declare class PostgresDestination extends DatabaseDestination {
|
|
4
|
+
protected connection: IPostgresDbConnection;
|
|
4
5
|
protected upsertConstraints: string[] | undefined;
|
|
5
6
|
constructor(config: IPostgresDestinationConfig);
|
|
6
7
|
loadBatch(data: any[]): Promise<void>;
|
|
@@ -57,6 +57,7 @@ var PostgresDestination = /** @class */ (function (_super) {
|
|
|
57
57
|
__extends(PostgresDestination, _super);
|
|
58
58
|
function PostgresDestination(config) {
|
|
59
59
|
var _this = _super.call(this, config) || this;
|
|
60
|
+
_this.connection = config.connection;
|
|
60
61
|
_this.upsertConstraints = config.upsertConstraints;
|
|
61
62
|
return _this;
|
|
62
63
|
}
|
|
@@ -2,6 +2,5 @@ import { Processor } from './processor';
|
|
|
2
2
|
import { IDatabaseProcessorConfig } from '../../types';
|
|
3
3
|
export declare abstract class DatabaseProcessor extends Processor {
|
|
4
4
|
protected query: string;
|
|
5
|
-
protected connection: any;
|
|
6
5
|
constructor(config: IDatabaseProcessorConfig);
|
|
7
6
|
}
|
|
@@ -23,10 +23,6 @@ var DatabaseProcessor = /** @class */ (function (_super) {
|
|
|
23
23
|
throw new Error("No query specified.");
|
|
24
24
|
}
|
|
25
25
|
_this.query = config.query;
|
|
26
|
-
if (!config.connection) {
|
|
27
|
-
throw new Error("No connection specified.");
|
|
28
|
-
}
|
|
29
|
-
_this.connection = config.connection;
|
|
30
26
|
return _this;
|
|
31
27
|
}
|
|
32
28
|
return DatabaseProcessor;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { IMssqlProcessorConfig, processStream } from '../types';
|
|
1
|
+
import { IMssqlProcessorConfig, IMssqlDbConnection, processStream } from '../types';
|
|
2
2
|
import { DatabaseProcessor } from './base/database-processor';
|
|
3
3
|
export declare class MssqlProcessor extends DatabaseProcessor {
|
|
4
|
+
protected connection: IMssqlDbConnection;
|
|
4
5
|
constructor(config: IMssqlProcessorConfig);
|
|
5
6
|
process(processStream: processStream): Promise<void>;
|
|
6
7
|
}
|
|
@@ -56,7 +56,9 @@ var database_processor_1 = require("./base/database-processor");
|
|
|
56
56
|
var MssqlProcessor = /** @class */ (function (_super) {
|
|
57
57
|
__extends(MssqlProcessor, _super);
|
|
58
58
|
function MssqlProcessor(config) {
|
|
59
|
-
|
|
59
|
+
var _this = _super.call(this, config) || this;
|
|
60
|
+
_this.connection = config.connection;
|
|
61
|
+
return _this;
|
|
60
62
|
}
|
|
61
63
|
MssqlProcessor.prototype.process = function (processStream) {
|
|
62
64
|
return __awaiter(this, void 0, void 0, function () {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { IPostgresProcessorConfig, processStream } from '../types';
|
|
1
|
+
import { IPostgresProcessorConfig, IPostgresDbConnection, processStream } from '../types';
|
|
2
2
|
import { DatabaseProcessor } from './base/database-processor';
|
|
3
3
|
export declare class PostgresProcessor extends DatabaseProcessor {
|
|
4
|
+
protected connection: IPostgresDbConnection;
|
|
4
5
|
constructor(config: IPostgresProcessorConfig);
|
|
5
6
|
process(processStream: processStream): Promise<void>;
|
|
6
7
|
}
|
|
@@ -56,7 +56,9 @@ var QueryStream = require('pg-query-stream');
|
|
|
56
56
|
var PostgresProcessor = /** @class */ (function (_super) {
|
|
57
57
|
__extends(PostgresProcessor, _super);
|
|
58
58
|
function PostgresProcessor(config) {
|
|
59
|
-
|
|
59
|
+
var _this = _super.call(this, config) || this;
|
|
60
|
+
_this.connection = config.connection;
|
|
61
|
+
return _this;
|
|
60
62
|
}
|
|
61
63
|
PostgresProcessor.prototype.process = function (processStream) {
|
|
62
64
|
return __awaiter(this, void 0, void 0, function () {
|
package/lib/types.d.ts
CHANGED
|
@@ -9,15 +9,26 @@ export interface AuthorizationRequest {
|
|
|
9
9
|
destinationField: string;
|
|
10
10
|
prefix?: string;
|
|
11
11
|
}
|
|
12
|
-
export interface
|
|
12
|
+
export interface IPostgresDbConnection {
|
|
13
13
|
user?: string;
|
|
14
14
|
password?: string;
|
|
15
|
-
server?: string;
|
|
16
15
|
host?: string;
|
|
17
16
|
database?: string;
|
|
18
17
|
schema?: string;
|
|
19
|
-
driver?: 'tedious' | 'msnodesqlv8';
|
|
20
18
|
}
|
|
19
|
+
export interface ITdsDriver {
|
|
20
|
+
ConnectionPool: any;
|
|
21
|
+
Transaction: any;
|
|
22
|
+
}
|
|
23
|
+
export interface IMssqlDbConnection {
|
|
24
|
+
user?: string;
|
|
25
|
+
password?: string;
|
|
26
|
+
server?: string;
|
|
27
|
+
database?: string;
|
|
28
|
+
driver?: ITdsDriver;
|
|
29
|
+
options?: any;
|
|
30
|
+
}
|
|
31
|
+
export declare type IDbConnection = IPostgresDbConnection | IMssqlDbConnection;
|
|
21
32
|
export interface IReporter {
|
|
22
33
|
report(job: IJob): Promise<void> | void;
|
|
23
34
|
}
|
|
@@ -68,12 +79,13 @@ export interface IDestinationConfig {
|
|
|
68
79
|
}
|
|
69
80
|
export interface IDatabaseDestinationConfig extends IDestinationConfig {
|
|
70
81
|
table: string;
|
|
71
|
-
connection: IDbConnection;
|
|
72
82
|
}
|
|
73
83
|
export interface IPostgresDestinationConfig extends IDatabaseDestinationConfig {
|
|
84
|
+
connection: IPostgresDbConnection;
|
|
74
85
|
upsertConstraints?: string[];
|
|
75
86
|
}
|
|
76
87
|
export interface IMssqlDestinationConfig extends IDatabaseDestinationConfig {
|
|
88
|
+
connection: IMssqlDbConnection;
|
|
77
89
|
}
|
|
78
90
|
export interface IHttpDestinationConfig extends IDestinationConfig {
|
|
79
91
|
request: AxiosRequestConfig;
|
|
@@ -103,7 +115,6 @@ export interface IDelimitedHttpProcessorConfig extends IHttpProcessorConfig {
|
|
|
103
115
|
qualifier?: string;
|
|
104
116
|
}
|
|
105
117
|
export interface IDatabaseProcessorConfig extends IProcessorConfig {
|
|
106
|
-
connection: any;
|
|
107
118
|
query: string;
|
|
108
119
|
}
|
|
109
120
|
export interface IJsonProcessorConfig extends IDirectoryProcessorConfig {
|
|
@@ -116,8 +127,10 @@ export interface IDelimitedProcessorConfig extends IDirectoryProcessorConfig {
|
|
|
116
127
|
qualifier?: string;
|
|
117
128
|
}
|
|
118
129
|
export interface IMssqlProcessorConfig extends IDatabaseProcessorConfig {
|
|
130
|
+
connection: IMssqlDbConnection;
|
|
119
131
|
}
|
|
120
132
|
export interface IPostgresProcessorConfig extends IDatabaseProcessorConfig {
|
|
133
|
+
connection: IPostgresDbConnection;
|
|
121
134
|
}
|
|
122
135
|
export interface ITailProcessorConfig extends IDirectoryProcessorConfig {
|
|
123
136
|
fromBeginning?: boolean;
|
package/lib/utils.js
CHANGED
|
@@ -92,68 +92,94 @@ exports.applyHttpAuthorization = exports.getDelimitedGenerator = exports.removeC
|
|
|
92
92
|
var axios_1 = __importDefault(require("axios"));
|
|
93
93
|
var stream_1 = require("stream");
|
|
94
94
|
var cachedDbConnections = new Map();
|
|
95
|
+
function getCachedDbConnection(databaseConfig) {
|
|
96
|
+
var dbKey = getDbKey(databaseConfig);
|
|
97
|
+
return cachedDbConnections.get(dbKey);
|
|
98
|
+
}
|
|
99
|
+
function getDbKey(databaseConfig) {
|
|
100
|
+
if ('driver' in databaseConfig) {
|
|
101
|
+
var configWithoutDriver = __assign({}, databaseConfig);
|
|
102
|
+
delete configWithoutDriver.driver;
|
|
103
|
+
return JSON.stringify(configWithoutDriver);
|
|
104
|
+
}
|
|
105
|
+
return JSON.stringify(databaseConfig);
|
|
106
|
+
}
|
|
107
|
+
function setCachedDbConnection(databaseConfig, db, close) {
|
|
108
|
+
var dbKey = getDbKey(databaseConfig);
|
|
109
|
+
cachedDbConnections.set(dbKey, { db: db, close: close });
|
|
110
|
+
}
|
|
111
|
+
function unsetCachedDbConnection(databaseConfig) {
|
|
112
|
+
var dbKey = getDbKey(databaseConfig);
|
|
113
|
+
cachedDbConnections.delete(dbKey);
|
|
114
|
+
}
|
|
95
115
|
function getDb(databaseConfig, dbType) {
|
|
96
116
|
return __awaiter(this, void 0, void 0, function () {
|
|
97
|
-
var
|
|
98
|
-
return __generator(this, function (
|
|
99
|
-
switch (
|
|
117
|
+
var dbConnection;
|
|
118
|
+
return __generator(this, function (_a) {
|
|
119
|
+
switch (_a.label) {
|
|
100
120
|
case 0:
|
|
101
|
-
|
|
102
|
-
if (
|
|
103
|
-
|
|
104
|
-
if (dbConnection) {
|
|
105
|
-
return [2 /*return*/, dbConnection.db];
|
|
106
|
-
}
|
|
121
|
+
dbConnection = getCachedDbConnection(databaseConfig);
|
|
122
|
+
if (dbConnection) {
|
|
123
|
+
return [2 /*return*/, dbConnection.db];
|
|
107
124
|
}
|
|
108
|
-
if (!(dbType === 'mssql')) return [3 /*break*/,
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
case
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
125
|
+
if (!(dbType === 'mssql')) return [3 /*break*/, 2];
|
|
126
|
+
return [4 /*yield*/, getMssqlDb(databaseConfig)];
|
|
127
|
+
case 1: return [2 /*return*/, _a.sent()];
|
|
128
|
+
case 2: return [2 /*return*/, getPostgresDb(databaseConfig)];
|
|
129
|
+
}
|
|
130
|
+
});
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
exports.getDb = getDb;
|
|
134
|
+
function getMssqlDb(databaseConfig) {
|
|
135
|
+
var _a;
|
|
136
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
137
|
+
var driver, _b, pool, db;
|
|
138
|
+
return __generator(this, function (_c) {
|
|
139
|
+
switch (_c.label) {
|
|
140
|
+
case 0:
|
|
141
|
+
if (!((_a = databaseConfig.driver) !== null && _a !== void 0)) return [3 /*break*/, 1];
|
|
142
|
+
_b = _a;
|
|
143
|
+
return [3 /*break*/, 3];
|
|
144
|
+
case 1: return [4 /*yield*/, Promise.resolve().then(function () { return __importStar(require('mssql')); })];
|
|
145
|
+
case 2:
|
|
146
|
+
_b = _c.sent();
|
|
147
|
+
_c.label = 3;
|
|
115
148
|
case 3:
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
case 4:
|
|
119
|
-
sql = _a;
|
|
120
|
-
pool = new sql.ConnectionPool(__assign({}, databaseConfig));
|
|
149
|
+
driver = _b;
|
|
150
|
+
pool = getMssqlDbPool(driver, databaseConfig);
|
|
121
151
|
return [4 /*yield*/, pool.connect()];
|
|
122
|
-
case
|
|
123
|
-
db =
|
|
124
|
-
|
|
125
|
-
db: db,
|
|
126
|
-
close: pool.close.bind(pool),
|
|
127
|
-
});
|
|
128
|
-
return [2 /*return*/, db];
|
|
129
|
-
case 6:
|
|
130
|
-
pgp = require('pg-promise')({ schema: databaseConfig.schema || 'public', });
|
|
131
|
-
db = pgp(databaseConfig);
|
|
132
|
-
cachedDbConnections.set(dbKey, {
|
|
133
|
-
db: db,
|
|
134
|
-
close: pgp.end,
|
|
135
|
-
});
|
|
152
|
+
case 4:
|
|
153
|
+
db = _c.sent();
|
|
154
|
+
setCachedDbConnection(databaseConfig, db, pool.close.bind(pool));
|
|
136
155
|
return [2 /*return*/, db];
|
|
137
156
|
}
|
|
138
157
|
});
|
|
139
158
|
});
|
|
140
159
|
}
|
|
141
|
-
|
|
142
|
-
;
|
|
160
|
+
function getMssqlDbPool(driver, databaseConfig) {
|
|
161
|
+
var poolConfig = __assign({}, databaseConfig);
|
|
162
|
+
delete poolConfig.driver;
|
|
163
|
+
return new driver.ConnectionPool(__assign({}, databaseConfig));
|
|
164
|
+
}
|
|
165
|
+
function getPostgresDb(databaseConfig) {
|
|
166
|
+
var pgp = require('pg-promise')({ schema: databaseConfig.schema || 'public' });
|
|
167
|
+
var db = pgp(databaseConfig);
|
|
168
|
+
setCachedDbConnection(databaseConfig, db, pgp.end);
|
|
169
|
+
return db;
|
|
170
|
+
}
|
|
143
171
|
function closeDbConnection(databaseConfig) {
|
|
144
172
|
return __awaiter(this, void 0, void 0, function () {
|
|
145
|
-
var
|
|
173
|
+
var dbConnection;
|
|
146
174
|
return __generator(this, function (_a) {
|
|
147
175
|
switch (_a.label) {
|
|
148
176
|
case 0:
|
|
149
|
-
|
|
150
|
-
if (!cachedDbConnections.has(dbKey)) return [3 /*break*/, 2];
|
|
151
|
-
dbConnection = cachedDbConnections.get(dbKey);
|
|
177
|
+
dbConnection = getCachedDbConnection(databaseConfig);
|
|
152
178
|
if (!dbConnection) return [3 /*break*/, 2];
|
|
153
179
|
return [4 /*yield*/, dbConnection.close()];
|
|
154
180
|
case 1:
|
|
155
181
|
_a.sent();
|
|
156
|
-
|
|
182
|
+
unsetCachedDbConnection(databaseConfig);
|
|
157
183
|
_a.label = 2;
|
|
158
184
|
case 2: return [2 /*return*/];
|
|
159
185
|
}
|
|
@@ -173,7 +199,6 @@ function getValueFromJSONChunk() {
|
|
|
173
199
|
});
|
|
174
200
|
}
|
|
175
201
|
exports.getValueFromJSONChunk = getValueFromJSONChunk;
|
|
176
|
-
;
|
|
177
202
|
function removeCircularReferencesFromChunk() {
|
|
178
203
|
return new stream_1.Transform({
|
|
179
204
|
objectMode: true,
|
|
@@ -196,7 +221,6 @@ function removeCircularReferencesFromChunk() {
|
|
|
196
221
|
});
|
|
197
222
|
}
|
|
198
223
|
exports.removeCircularReferencesFromChunk = removeCircularReferencesFromChunk;
|
|
199
|
-
;
|
|
200
224
|
function getDelimitedGenerator(_a) {
|
|
201
225
|
var readStream = _a.readStream, hasHeader = _a.hasHeader;
|
|
202
226
|
var header = [];
|
package/package.json
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bellboy",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "8.0.0",
|
|
4
4
|
"description": "Highly performant JavaScript data stream ETL engine.",
|
|
5
5
|
"main": "lib/index",
|
|
6
6
|
"types": "lib/index",
|
|
7
7
|
"scripts": {
|
|
8
8
|
"build": "tsc -p .",
|
|
9
9
|
"test": "jest --coverage",
|
|
10
|
+
"lint": "tslint -c tslint.json -p tsconfig.json",
|
|
10
11
|
"codecov": "codecov"
|
|
11
12
|
},
|
|
12
13
|
"repository": {
|
|
@@ -23,9 +24,14 @@
|
|
|
23
24
|
"url": "https://github.com/Claviz/bellboy/issues"
|
|
24
25
|
},
|
|
25
26
|
"homepage": "https://github.com/Claviz/bellboy#readme",
|
|
26
|
-
"
|
|
27
|
+
"peerDependencies": {
|
|
27
28
|
"msnodesqlv8": "^2.1.0"
|
|
28
29
|
},
|
|
30
|
+
"peerDependenciesMeta": {
|
|
31
|
+
"msnodesqlv8": {
|
|
32
|
+
"optional": true
|
|
33
|
+
}
|
|
34
|
+
},
|
|
29
35
|
"devDependencies": {
|
|
30
36
|
"@types/axios": "^0.14.0",
|
|
31
37
|
"@types/jest": "^25.2.1",
|