bellboy 7.0.0 → 8.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +116 -12
- package/lib/destinations/base/database-destination.d.ts +1 -2
- package/lib/destinations/base/database-destination.js +0 -1
- package/lib/destinations/mssql-destination.d.ts +2 -2
- package/lib/destinations/mssql-destination.js +40 -41
- package/lib/destinations/mysql-destination.d.ts +7 -0
- package/lib/destinations/mysql-destination.js +114 -0
- package/lib/destinations/postgres-destination.d.ts +2 -1
- package/lib/destinations/postgres-destination.js +1 -0
- package/lib/index.d.ts +2 -0
- package/lib/index.js +5 -1
- package/lib/processors/base/database-processor.d.ts +0 -1
- package/lib/processors/base/database-processor.js +0 -4
- package/lib/processors/mssql-processor.d.ts +2 -1
- package/lib/processors/mssql-processor.js +3 -1
- package/lib/processors/mysql-processor.d.ts +7 -0
- package/lib/processors/mysql-processor.js +109 -0
- package/lib/processors/postgres-processor.d.ts +2 -1
- package/lib/processors/postgres-processor.js +3 -1
- package/lib/types.d.ts +31 -6
- package/lib/utils.js +83 -41
- package/package.json +9 -2
package/README.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# bellboy
|
|
1
|
+
# bellboy  [](https://codecov.io/gh/Claviz/bellboy) 
|
|
2
2
|
|
|
3
3
|
Highly performant JavaScript data stream ETL engine.
|
|
4
4
|
|
|
@@ -14,6 +14,12 @@ Before install, make sure you are using [latest](https://nodejs.org/en/download/
|
|
|
14
14
|
npm install bellboy
|
|
15
15
|
```
|
|
16
16
|
|
|
17
|
+
If you will be using `bellboy` with the native [msnodesqlv8][msnodesqlv8-url] driver, add it as a dependency.
|
|
18
|
+
|
|
19
|
+
```
|
|
20
|
+
npm install msnodesqlv8
|
|
21
|
+
```
|
|
22
|
+
|
|
17
23
|
## Example
|
|
18
24
|
|
|
19
25
|
This example shows how `bellboy` can extract rows from the [Excel file](#excel-processor), modify it on the fly, load to the [Postgres database](#postgres-destination), move processed file to the other folder and process remaining files.
|
|
@@ -115,7 +121,7 @@ const job = new bellboy.Job(
|
|
|
115
121
|
|
|
116
122
|
Event listeners, which can be registered with `job.on` or `job.onAny` methods, allow you to listen to specific events in the job lifecycle and to interact with them.
|
|
117
123
|
|
|
118
|
-
-
|
|
124
|
+
- When multiple listeners are registered for a single event, those added by [reporters](#reporters) will be executed first, followed by the order of registration for the remaining listeners.
|
|
119
125
|
- Job always waits for the code inside a listener to complete.
|
|
120
126
|
- Any error thrown inside a listener will be ignored and warning message will be printed out.
|
|
121
127
|
- `job.stop()` method can be used inside a listener to stop job execution and throw an error if needed.
|
|
@@ -300,8 +306,9 @@ Each processor in `bellboy` is a class which has a single responsibility of proc
|
|
|
300
306
|
- [ExcelProcessor](#excel-processor) processes **XLSX** file data from the file system.
|
|
301
307
|
- [JsonProcessor](#json-processor) processes **JSON** file data from the file system.
|
|
302
308
|
- [DelimitedProcessor](#delimited-processor) processes files with **delimited data** from the file system.
|
|
303
|
-
- [PostgresProcessor](#
|
|
304
|
-
- [
|
|
309
|
+
- [PostgresProcessor](#postgres-processor) processes data received from a **PostgreSQL** SELECT.
|
|
310
|
+
- [MySqlProcessor](#mysql-processor) processes data received from a **MySQL** SELECT.
|
|
311
|
+
- [MssqlProcessor](#mssql-processor) processes data received from a **MSSQL** SELECT.
|
|
305
312
|
- [DynamicProcessor](#dynamic-processor) processes **dynamically generated** data.
|
|
306
313
|
- [TailProcessor](#tail-processor) processes **new lines** added to the file.
|
|
307
314
|
|
|
@@ -503,9 +510,26 @@ Watches for file changes and outputs last part of file as soon as new lines are
|
|
|
503
510
|
Name of the file the data came from.
|
|
504
511
|
- **data** `string`
|
|
505
512
|
|
|
506
|
-
###
|
|
513
|
+
### PostgresProcessor <div id='postgres-processor'/>
|
|
514
|
+
|
|
515
|
+
Processes a PostgreSQL `SELECT` query row by row.
|
|
516
|
+
|
|
517
|
+
#### Options
|
|
518
|
+
|
|
519
|
+
- [Processor options](#processor-options)
|
|
520
|
+
- **query** `string` `required`\
|
|
521
|
+
Query to execute.
|
|
522
|
+
- **connection** `object` `required`
|
|
523
|
+
- **user**
|
|
524
|
+
- **password**
|
|
525
|
+
- **host**
|
|
526
|
+
- **port**
|
|
527
|
+
- **database**
|
|
528
|
+
- **schema**
|
|
529
|
+
|
|
530
|
+
### MySqlProcessor <div id='mysql-processor'/>
|
|
507
531
|
|
|
508
|
-
Processes `SELECT` query row by row.
|
|
532
|
+
Processes a MySQL `SELECT` query row by row.
|
|
509
533
|
|
|
510
534
|
#### Options
|
|
511
535
|
|
|
@@ -515,16 +539,50 @@ Processes `SELECT` query row by row. There are two database processors - `Postgr
|
|
|
515
539
|
- **connection** `object` `required`
|
|
516
540
|
- **user**
|
|
517
541
|
- **password**
|
|
518
|
-
- **server**\
|
|
519
|
-
Used with `MssqlProcessor`.
|
|
520
542
|
- **host**
|
|
521
|
-
Used with `PostgresProcessor`.
|
|
522
543
|
- **port**
|
|
523
544
|
- **database**
|
|
524
|
-
|
|
525
|
-
|
|
545
|
+
|
|
546
|
+
### MssqlProcessor <div id='mssql-processor'/>
|
|
547
|
+
|
|
548
|
+
Processes a MSSQL `SELECT` query row by row.
|
|
549
|
+
|
|
550
|
+
#### Options
|
|
551
|
+
|
|
552
|
+
- [Processor options](#processor-options)
|
|
553
|
+
- **query** `string` `required`\
|
|
554
|
+
Query to execute.
|
|
555
|
+
- **connection** `object` `required`
|
|
556
|
+
- **user**
|
|
557
|
+
- **password**
|
|
558
|
+
- **server**
|
|
559
|
+
- **port**
|
|
560
|
+
- **database**
|
|
526
561
|
- **driver**\
|
|
527
|
-
|
|
562
|
+
Optional [mssql][mssql-url] TDS driver; defaults to the pure JavaScript [Tedious][tedious-url] driver.
|
|
563
|
+
|
|
564
|
+
#### Usage
|
|
565
|
+
|
|
566
|
+
Here is an example of how to configure `MssqlProcessor` with a native TDS driver instead of the default pure JavasScript Tedious driver.
|
|
567
|
+
|
|
568
|
+
```javascript
|
|
569
|
+
const nativeDriver: ITdsDriver = await import("mssql/msnodesqlV8");
|
|
570
|
+
const connection: IMssqlDbConnection = {
|
|
571
|
+
user: "user",
|
|
572
|
+
password: "password",
|
|
573
|
+
server: "server",
|
|
574
|
+
database: "database",
|
|
575
|
+
driver: nativeDriver,
|
|
576
|
+
};
|
|
577
|
+
const source = new MssqlProcessor({
|
|
578
|
+
connection,
|
|
579
|
+
query: "select * from orders",
|
|
580
|
+
});
|
|
581
|
+
```
|
|
582
|
+
|
|
583
|
+
In previous versions of `bellboy`, `connection.driver` was a `string` parameter.
|
|
584
|
+
|
|
585
|
+
[More usage examples](tests/mssql-source.spec.ts)
|
|
528
586
|
|
|
529
587
|
### DynamicProcessor <div id='dynamic-processor'/>
|
|
530
588
|
|
|
@@ -554,6 +612,7 @@ Every [job](#job) can have as many destinations (outputs) as needed. For example
|
|
|
554
612
|
- [StdoutDestination](#stdout-destination) logs data to **console**.
|
|
555
613
|
- [HttpDestination](#http-destination) executes **HTTP** request calls.
|
|
556
614
|
- [PostgresDestination](#postgres-destination) inserts/upserts data to **PostgreSQL** database.
|
|
615
|
+
- [MySqlDestination](#mysql-destination) inserts/upserts data to **MySQL** database.
|
|
557
616
|
- [MssqlDestination](#mssql-destination) inserts data to **MSSQL** database.
|
|
558
617
|
|
|
559
618
|
### Options <div id='destination-options'/>
|
|
@@ -620,6 +679,23 @@ Inserts data to PostgreSQL.
|
|
|
620
679
|
- **database**
|
|
621
680
|
- **schema**
|
|
622
681
|
|
|
682
|
+
### MySqlDestination <div id='mysql-destination'/>
|
|
683
|
+
|
|
684
|
+
[Usage examples](tests/mysql-destination.spec.ts)
|
|
685
|
+
|
|
686
|
+
Inserts data to MySQL.
|
|
687
|
+
|
|
688
|
+
#### Options
|
|
689
|
+
|
|
690
|
+
- [General destination options](#destination-options)
|
|
691
|
+
- **table** `string` `required`\
|
|
692
|
+
Table name.
|
|
693
|
+
- **connection** `object` `required`
|
|
694
|
+
- **user**
|
|
695
|
+
- **password**
|
|
696
|
+
- **host**
|
|
697
|
+
- **database**
|
|
698
|
+
|
|
623
699
|
### MssqlDestination <div id='mssql-destination'/>
|
|
624
700
|
|
|
625
701
|
[Usage examples](tests/mssql-destination.spec.ts)
|
|
@@ -636,6 +712,30 @@ Inserts data to MSSQL.
|
|
|
636
712
|
- **password**
|
|
637
713
|
- **server**
|
|
638
714
|
- **database**
|
|
715
|
+
- **driver** \
|
|
716
|
+
Optional [mssql][mssql-url] TDS driver; defaults to the pure JavaScript [Tedious][tedious-url] driver.
|
|
717
|
+
|
|
718
|
+
#### Usage
|
|
719
|
+
|
|
720
|
+
Here is an example of how to configure `MssqlDestination` with a native TDS driver instead of the default pure JavasScript Tedious driver.
|
|
721
|
+
|
|
722
|
+
```javascript
|
|
723
|
+
const nativeDriver: ITdsDriver = await import("mssql/msnodesqlV8");
|
|
724
|
+
const connection: IMssqlDbConnection = {
|
|
725
|
+
user: "user",
|
|
726
|
+
password: "password",
|
|
727
|
+
server: "server",
|
|
728
|
+
database: "database",
|
|
729
|
+
driver: nativeDriver,
|
|
730
|
+
};
|
|
731
|
+
const sink = new MssqlDestination({
|
|
732
|
+
connection,
|
|
733
|
+
table: "orders",
|
|
734
|
+
batchSize: 1000,
|
|
735
|
+
});
|
|
736
|
+
```
|
|
737
|
+
|
|
738
|
+
[More usage examples](tests/mssql-destination.spec.ts)
|
|
639
739
|
|
|
640
740
|
## Extendability
|
|
641
741
|
|
|
@@ -699,3 +799,7 @@ class CustomReporter extends bellboy.Reporter {
|
|
|
699
799
|
## Testing
|
|
700
800
|
|
|
701
801
|
Tests can be run by using `docker-compose up --abort-on-container-exit --exit-code-from test --build test` command.
|
|
802
|
+
|
|
803
|
+
[mssql-url]: https://github.com/tediousjs/node-mssql
|
|
804
|
+
[tedious-url]: https://www.npmjs.com/package/tedious
|
|
805
|
+
[msnodesqlv8-url]: https://www.npmjs.com/package/msnodesqlv8
|
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import { IDatabaseDestinationConfig
|
|
1
|
+
import { IDatabaseDestinationConfig } from '../../types';
|
|
2
2
|
import { Destination } from './destination';
|
|
3
3
|
export declare abstract class DatabaseDestination extends Destination {
|
|
4
4
|
protected table: string;
|
|
5
|
-
protected connection: IDbConnection;
|
|
6
5
|
constructor(config: IDatabaseDestinationConfig);
|
|
7
6
|
loadBatch(data: any[]): Promise<void>;
|
|
8
7
|
}
|
|
@@ -56,7 +56,6 @@ var DatabaseDestination = /** @class */ (function (_super) {
|
|
|
56
56
|
function DatabaseDestination(config) {
|
|
57
57
|
var _this = _super.call(this, config) || this;
|
|
58
58
|
_this.table = config.table;
|
|
59
|
-
_this.connection = config.connection;
|
|
60
59
|
return _this;
|
|
61
60
|
}
|
|
62
61
|
DatabaseDestination.prototype.loadBatch = function (data) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { IMssqlDestinationConfig } from '../types';
|
|
1
|
+
import { IMssqlDestinationConfig, IMssqlDbConnection } from '../types';
|
|
2
2
|
import { DatabaseDestination } from './base/database-destination';
|
|
3
3
|
export declare class MssqlDestination extends DatabaseDestination {
|
|
4
|
-
|
|
4
|
+
protected connection: IMssqlDbConnection;
|
|
5
5
|
constructor(config: IMssqlDestinationConfig);
|
|
6
6
|
loadBatch(data: any[]): Promise<void>;
|
|
7
7
|
}
|
|
@@ -85,34 +85,33 @@ var MssqlDestination = /** @class */ (function (_super) {
|
|
|
85
85
|
__extends(MssqlDestination, _super);
|
|
86
86
|
function MssqlDestination(config) {
|
|
87
87
|
var _this = _super.call(this, config) || this;
|
|
88
|
-
_this.
|
|
88
|
+
_this.connection = config.connection;
|
|
89
89
|
return _this;
|
|
90
90
|
}
|
|
91
91
|
MssqlDestination.prototype.loadBatch = function (data) {
|
|
92
|
+
var _a;
|
|
92
93
|
return __awaiter(this, void 0, void 0, function () {
|
|
93
|
-
var sql,
|
|
94
|
-
var
|
|
95
|
-
return __generator(this, function (
|
|
96
|
-
switch (
|
|
94
|
+
var sql, _b, pool, db, query, table, columns, _i, data_1, x, args, ii, transaction, err_1;
|
|
95
|
+
var _c;
|
|
96
|
+
return __generator(this, function (_d) {
|
|
97
|
+
switch (_d.label) {
|
|
97
98
|
case 0:
|
|
98
|
-
if (!(this.driver
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
99
|
+
if (!((_a = this.connection.driver) !== null && _a !== void 0)) return [3 /*break*/, 1];
|
|
100
|
+
_b = _a;
|
|
101
|
+
return [3 /*break*/, 3];
|
|
102
|
+
case 1: return [4 /*yield*/, Promise.resolve().then(function () { return __importStar(require('mssql')); })];
|
|
103
|
+
case 2:
|
|
104
|
+
_b = _d.sent();
|
|
105
|
+
_d.label = 3;
|
|
104
106
|
case 3:
|
|
105
|
-
|
|
106
|
-
_c.label = 4;
|
|
107
|
-
case 4:
|
|
108
|
-
sql = _a;
|
|
107
|
+
sql = _b;
|
|
109
108
|
pool = new sql.ConnectionPool(__assign({}, this.connection));
|
|
110
109
|
return [4 /*yield*/, pool.connect()];
|
|
111
|
-
case
|
|
112
|
-
db =
|
|
110
|
+
case 4:
|
|
111
|
+
db = _d.sent();
|
|
113
112
|
return [4 /*yield*/, db.request().query("SELECT TOP(0) * FROM " + this.table)];
|
|
114
|
-
case
|
|
115
|
-
query =
|
|
113
|
+
case 5:
|
|
114
|
+
query = _d.sent();
|
|
116
115
|
table = query.recordset.toTable(this.table);
|
|
117
116
|
columns = table.columns.map(function (x) { return x.name; });
|
|
118
117
|
for (_i = 0, data_1 = data; _i < data_1.length; _i++) {
|
|
@@ -121,36 +120,36 @@ var MssqlDestination = /** @class */ (function (_super) {
|
|
|
121
120
|
for (ii = 0; ii < columns.length; ii++) {
|
|
122
121
|
args.push(x[columns[ii]]);
|
|
123
122
|
}
|
|
124
|
-
(
|
|
123
|
+
(_c = table.rows).add.apply(_c, args);
|
|
125
124
|
}
|
|
126
125
|
transaction = new sql.Transaction(db);
|
|
127
|
-
|
|
128
|
-
case
|
|
129
|
-
|
|
126
|
+
_d.label = 6;
|
|
127
|
+
case 6:
|
|
128
|
+
_d.trys.push([6, 10, , 14]);
|
|
130
129
|
return [4 /*yield*/, transaction.begin()];
|
|
131
|
-
case
|
|
132
|
-
|
|
130
|
+
case 7:
|
|
131
|
+
_d.sent();
|
|
133
132
|
return [4 /*yield*/, transaction.request().bulk(table)];
|
|
134
|
-
case
|
|
135
|
-
|
|
133
|
+
case 8:
|
|
134
|
+
_d.sent();
|
|
136
135
|
return [4 /*yield*/, transaction.commit()];
|
|
136
|
+
case 9:
|
|
137
|
+
_d.sent();
|
|
138
|
+
return [3 /*break*/, 14];
|
|
137
139
|
case 10:
|
|
138
|
-
|
|
139
|
-
return [3 /*break*/,
|
|
140
|
-
case 11:
|
|
141
|
-
err_1 = _c.sent();
|
|
142
|
-
if (!!transaction._aborted) return [3 /*break*/, 13];
|
|
140
|
+
err_1 = _d.sent();
|
|
141
|
+
if (!!transaction._aborted) return [3 /*break*/, 12];
|
|
143
142
|
return [4 /*yield*/, transaction.rollback()];
|
|
144
|
-
case
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
case
|
|
148
|
-
case
|
|
149
|
-
|
|
143
|
+
case 11:
|
|
144
|
+
_d.sent();
|
|
145
|
+
_d.label = 12;
|
|
146
|
+
case 12: return [4 /*yield*/, pool.close()];
|
|
147
|
+
case 13:
|
|
148
|
+
_d.sent();
|
|
150
149
|
throw err_1;
|
|
151
|
-
case
|
|
152
|
-
case
|
|
153
|
-
|
|
150
|
+
case 14: return [4 /*yield*/, pool.close()];
|
|
151
|
+
case 15:
|
|
152
|
+
_d.sent();
|
|
154
153
|
return [2 /*return*/];
|
|
155
154
|
}
|
|
156
155
|
});
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { IMySqlDestinationConfig, IMySqlDbConnection } from '../types';
|
|
2
|
+
import { DatabaseDestination } from './base/database-destination';
|
|
3
|
+
export declare class MySqlDestination extends DatabaseDestination {
|
|
4
|
+
protected connection: IMySqlDbConnection;
|
|
5
|
+
constructor(config: IMySqlDestinationConfig);
|
|
6
|
+
loadBatch(data: any[]): Promise<void>;
|
|
7
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __extends = (this && this.__extends) || (function () {
|
|
3
|
+
var extendStatics = function (d, b) {
|
|
4
|
+
extendStatics = Object.setPrototypeOf ||
|
|
5
|
+
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
|
|
6
|
+
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
|
|
7
|
+
return extendStatics(d, b);
|
|
8
|
+
};
|
|
9
|
+
return function (d, b) {
|
|
10
|
+
extendStatics(d, b);
|
|
11
|
+
function __() { this.constructor = d; }
|
|
12
|
+
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
|
|
13
|
+
};
|
|
14
|
+
})();
|
|
15
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
16
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
17
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
18
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
19
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
20
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
21
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
22
|
+
});
|
|
23
|
+
};
|
|
24
|
+
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
25
|
+
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
26
|
+
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
27
|
+
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
28
|
+
function step(op) {
|
|
29
|
+
if (f) throw new TypeError("Generator is already executing.");
|
|
30
|
+
while (_) try {
|
|
31
|
+
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
32
|
+
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
33
|
+
switch (op[0]) {
|
|
34
|
+
case 0: case 1: t = op; break;
|
|
35
|
+
case 4: _.label++; return { value: op[1], done: false };
|
|
36
|
+
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
37
|
+
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
38
|
+
default:
|
|
39
|
+
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
40
|
+
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
41
|
+
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
42
|
+
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
43
|
+
if (t[2]) _.ops.pop();
|
|
44
|
+
_.trys.pop(); continue;
|
|
45
|
+
}
|
|
46
|
+
op = body.call(thisArg, _);
|
|
47
|
+
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
48
|
+
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
49
|
+
}
|
|
50
|
+
};
|
|
51
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
52
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
53
|
+
};
|
|
54
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
55
|
+
exports.MySqlDestination = void 0;
|
|
56
|
+
var database_destination_1 = require("./base/database-destination");
|
|
57
|
+
var promise_1 = __importDefault(require("mysql2/promise"));
|
|
58
|
+
var MySqlDestination = /** @class */ (function (_super) {
|
|
59
|
+
__extends(MySqlDestination, _super);
|
|
60
|
+
function MySqlDestination(config) {
|
|
61
|
+
var _this = _super.call(this, config) || this;
|
|
62
|
+
_this.connection = config.connection;
|
|
63
|
+
return _this;
|
|
64
|
+
}
|
|
65
|
+
MySqlDestination.prototype.loadBatch = function (data) {
|
|
66
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
67
|
+
var pool, dbConnection, columns, columnNames, insertQuery, insertData, err_1;
|
|
68
|
+
return __generator(this, function (_a) {
|
|
69
|
+
switch (_a.label) {
|
|
70
|
+
case 0:
|
|
71
|
+
pool = promise_1.default.createPool(this.connection);
|
|
72
|
+
return [4 /*yield*/, pool.getConnection()];
|
|
73
|
+
case 1:
|
|
74
|
+
dbConnection = _a.sent();
|
|
75
|
+
return [4 /*yield*/, dbConnection.query("DESCRIBE " + this.table)];
|
|
76
|
+
case 2:
|
|
77
|
+
columns = (_a.sent())[0];
|
|
78
|
+
columnNames = columns.map(function (col) { return "`" + col.Field + "`"; });
|
|
79
|
+
insertQuery = "INSERT INTO " + this.table + " (" + columnNames.join(', ') + ") VALUES ?";
|
|
80
|
+
return [4 /*yield*/, dbConnection.beginTransaction()];
|
|
81
|
+
case 3:
|
|
82
|
+
_a.sent();
|
|
83
|
+
_a.label = 4;
|
|
84
|
+
case 4:
|
|
85
|
+
_a.trys.push([4, 8, 10, 12]);
|
|
86
|
+
insertData = data.map(function (row) { return columnNames.map(function (column) { return row[column.replace(/`/g, '')]; }); });
|
|
87
|
+
return [4 /*yield*/, dbConnection.beginTransaction()];
|
|
88
|
+
case 5:
|
|
89
|
+
_a.sent();
|
|
90
|
+
return [4 /*yield*/, dbConnection.query(insertQuery, [insertData])];
|
|
91
|
+
case 6:
|
|
92
|
+
_a.sent();
|
|
93
|
+
return [4 /*yield*/, dbConnection.commit()];
|
|
94
|
+
case 7:
|
|
95
|
+
_a.sent();
|
|
96
|
+
return [3 /*break*/, 12];
|
|
97
|
+
case 8:
|
|
98
|
+
err_1 = _a.sent();
|
|
99
|
+
return [4 /*yield*/, dbConnection.rollback()];
|
|
100
|
+
case 9:
|
|
101
|
+
_a.sent();
|
|
102
|
+
throw err_1;
|
|
103
|
+
case 10: return [4 /*yield*/, pool.end()];
|
|
104
|
+
case 11:
|
|
105
|
+
_a.sent();
|
|
106
|
+
return [7 /*endfinally*/];
|
|
107
|
+
case 12: return [2 /*return*/];
|
|
108
|
+
}
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
};
|
|
112
|
+
return MySqlDestination;
|
|
113
|
+
}(database_destination_1.DatabaseDestination));
|
|
114
|
+
exports.MySqlDestination = MySqlDestination;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { IPostgresDestinationConfig } from '../types';
|
|
1
|
+
import { IPostgresDestinationConfig, IPostgresDbConnection } from '../types';
|
|
2
2
|
import { DatabaseDestination } from './base/database-destination';
|
|
3
3
|
export declare class PostgresDestination extends DatabaseDestination {
|
|
4
|
+
protected connection: IPostgresDbConnection;
|
|
4
5
|
protected upsertConstraints: string[] | undefined;
|
|
5
6
|
constructor(config: IPostgresDestinationConfig);
|
|
6
7
|
loadBatch(data: any[]): Promise<void>;
|
|
@@ -57,6 +57,7 @@ var PostgresDestination = /** @class */ (function (_super) {
|
|
|
57
57
|
__extends(PostgresDestination, _super);
|
|
58
58
|
function PostgresDestination(config) {
|
|
59
59
|
var _this = _super.call(this, config) || this;
|
|
60
|
+
_this.connection = config.connection;
|
|
60
61
|
_this.upsertConstraints = config.upsertConstraints;
|
|
61
62
|
return _this;
|
|
62
63
|
}
|
package/lib/index.d.ts
CHANGED
|
@@ -4,6 +4,7 @@ export { DatabaseProcessor } from './processors/base/database-processor';
|
|
|
4
4
|
export { DirectoryProcessor } from './processors/base/directory-processor';
|
|
5
5
|
export { ExcelProcessor } from './processors/excel-processor';
|
|
6
6
|
export { PostgresProcessor } from './processors/postgres-processor';
|
|
7
|
+
export { MySqlProcessor } from './processors/mysql-processor';
|
|
7
8
|
export { MssqlProcessor } from './processors/mssql-processor';
|
|
8
9
|
export { HttpProcessor } from './processors/http-processor';
|
|
9
10
|
export { DynamicProcessor } from './processors/dynamic-processor';
|
|
@@ -17,6 +18,7 @@ export { HttpDestination } from './destinations/http-destination';
|
|
|
17
18
|
export { MssqlDestination } from './destinations/mssql-destination';
|
|
18
19
|
export { PostgresDestination } from './destinations/postgres-destination';
|
|
19
20
|
export { StdoutDestination } from './destinations/stdout-destination';
|
|
21
|
+
export { MySqlDestination } from './destinations/mysql-destination';
|
|
20
22
|
export { getDb } from './utils';
|
|
21
23
|
export { Reporter } from './reporters/base/reporter';
|
|
22
24
|
export { IBellboyEvent } from './types';
|
package/lib/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Reporter = exports.getDb = exports.StdoutDestination = exports.PostgresDestination = exports.MssqlDestination = exports.HttpDestination = exports.DatabaseDestination = exports.Destination = exports.DelimitedProcessor = exports.TailProcessor = exports.MqttProcessor = exports.JsonProcessor = exports.DynamicProcessor = exports.HttpProcessor = exports.MssqlProcessor = exports.PostgresProcessor = exports.ExcelProcessor = exports.DirectoryProcessor = exports.DatabaseProcessor = exports.Processor = exports.Job = void 0;
|
|
3
|
+
exports.Reporter = exports.getDb = exports.MySqlDestination = exports.StdoutDestination = exports.PostgresDestination = exports.MssqlDestination = exports.HttpDestination = exports.DatabaseDestination = exports.Destination = exports.DelimitedProcessor = exports.TailProcessor = exports.MqttProcessor = exports.JsonProcessor = exports.DynamicProcessor = exports.HttpProcessor = exports.MssqlProcessor = exports.MySqlProcessor = exports.PostgresProcessor = exports.ExcelProcessor = exports.DirectoryProcessor = exports.DatabaseProcessor = exports.Processor = exports.Job = void 0;
|
|
4
4
|
var job_1 = require("./job");
|
|
5
5
|
Object.defineProperty(exports, "Job", { enumerable: true, get: function () { return job_1.Job; } });
|
|
6
6
|
var processor_1 = require("./processors/base/processor");
|
|
@@ -13,6 +13,8 @@ var excel_processor_1 = require("./processors/excel-processor");
|
|
|
13
13
|
Object.defineProperty(exports, "ExcelProcessor", { enumerable: true, get: function () { return excel_processor_1.ExcelProcessor; } });
|
|
14
14
|
var postgres_processor_1 = require("./processors/postgres-processor");
|
|
15
15
|
Object.defineProperty(exports, "PostgresProcessor", { enumerable: true, get: function () { return postgres_processor_1.PostgresProcessor; } });
|
|
16
|
+
var mysql_processor_1 = require("./processors/mysql-processor");
|
|
17
|
+
Object.defineProperty(exports, "MySqlProcessor", { enumerable: true, get: function () { return mysql_processor_1.MySqlProcessor; } });
|
|
16
18
|
var mssql_processor_1 = require("./processors/mssql-processor");
|
|
17
19
|
Object.defineProperty(exports, "MssqlProcessor", { enumerable: true, get: function () { return mssql_processor_1.MssqlProcessor; } });
|
|
18
20
|
var http_processor_1 = require("./processors/http-processor");
|
|
@@ -39,6 +41,8 @@ var postgres_destination_1 = require("./destinations/postgres-destination");
|
|
|
39
41
|
Object.defineProperty(exports, "PostgresDestination", { enumerable: true, get: function () { return postgres_destination_1.PostgresDestination; } });
|
|
40
42
|
var stdout_destination_1 = require("./destinations/stdout-destination");
|
|
41
43
|
Object.defineProperty(exports, "StdoutDestination", { enumerable: true, get: function () { return stdout_destination_1.StdoutDestination; } });
|
|
44
|
+
var mysql_destination_1 = require("./destinations/mysql-destination");
|
|
45
|
+
Object.defineProperty(exports, "MySqlDestination", { enumerable: true, get: function () { return mysql_destination_1.MySqlDestination; } });
|
|
42
46
|
var utils_1 = require("./utils");
|
|
43
47
|
Object.defineProperty(exports, "getDb", { enumerable: true, get: function () { return utils_1.getDb; } });
|
|
44
48
|
var reporter_1 = require("./reporters/base/reporter");
|
|
@@ -2,6 +2,5 @@ import { Processor } from './processor';
|
|
|
2
2
|
import { IDatabaseProcessorConfig } from '../../types';
|
|
3
3
|
export declare abstract class DatabaseProcessor extends Processor {
|
|
4
4
|
protected query: string;
|
|
5
|
-
protected connection: any;
|
|
6
5
|
constructor(config: IDatabaseProcessorConfig);
|
|
7
6
|
}
|
|
@@ -23,10 +23,6 @@ var DatabaseProcessor = /** @class */ (function (_super) {
|
|
|
23
23
|
throw new Error("No query specified.");
|
|
24
24
|
}
|
|
25
25
|
_this.query = config.query;
|
|
26
|
-
if (!config.connection) {
|
|
27
|
-
throw new Error("No connection specified.");
|
|
28
|
-
}
|
|
29
|
-
_this.connection = config.connection;
|
|
30
26
|
return _this;
|
|
31
27
|
}
|
|
32
28
|
return DatabaseProcessor;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { IMssqlProcessorConfig, processStream } from '../types';
|
|
1
|
+
import { IMssqlProcessorConfig, IMssqlDbConnection, processStream } from '../types';
|
|
2
2
|
import { DatabaseProcessor } from './base/database-processor';
|
|
3
3
|
export declare class MssqlProcessor extends DatabaseProcessor {
|
|
4
|
+
protected connection: IMssqlDbConnection;
|
|
4
5
|
constructor(config: IMssqlProcessorConfig);
|
|
5
6
|
process(processStream: processStream): Promise<void>;
|
|
6
7
|
}
|
|
@@ -56,7 +56,9 @@ var database_processor_1 = require("./base/database-processor");
|
|
|
56
56
|
var MssqlProcessor = /** @class */ (function (_super) {
|
|
57
57
|
__extends(MssqlProcessor, _super);
|
|
58
58
|
function MssqlProcessor(config) {
|
|
59
|
-
|
|
59
|
+
var _this = _super.call(this, config) || this;
|
|
60
|
+
_this.connection = config.connection;
|
|
61
|
+
return _this;
|
|
60
62
|
}
|
|
61
63
|
MssqlProcessor.prototype.process = function (processStream) {
|
|
62
64
|
return __awaiter(this, void 0, void 0, function () {
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { IMySqlDbConnection, IMySqlProcessorConfig, processStream } from '../types';
|
|
2
|
+
import { DatabaseProcessor } from './base/database-processor';
|
|
3
|
+
export declare class MySqlProcessor extends DatabaseProcessor {
|
|
4
|
+
protected connection: IMySqlDbConnection;
|
|
5
|
+
constructor(config: IMySqlProcessorConfig);
|
|
6
|
+
process(processStream: processStream): Promise<void>;
|
|
7
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __extends = (this && this.__extends) || (function () {
|
|
3
|
+
var extendStatics = function (d, b) {
|
|
4
|
+
extendStatics = Object.setPrototypeOf ||
|
|
5
|
+
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
|
|
6
|
+
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
|
|
7
|
+
return extendStatics(d, b);
|
|
8
|
+
};
|
|
9
|
+
return function (d, b) {
|
|
10
|
+
extendStatics(d, b);
|
|
11
|
+
function __() { this.constructor = d; }
|
|
12
|
+
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
|
|
13
|
+
};
|
|
14
|
+
})();
|
|
15
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
16
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
17
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
18
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
19
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
20
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
21
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
22
|
+
});
|
|
23
|
+
};
|
|
24
|
+
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
25
|
+
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
26
|
+
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
27
|
+
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
28
|
+
function step(op) {
|
|
29
|
+
if (f) throw new TypeError("Generator is already executing.");
|
|
30
|
+
while (_) try {
|
|
31
|
+
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
32
|
+
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
33
|
+
switch (op[0]) {
|
|
34
|
+
case 0: case 1: t = op; break;
|
|
35
|
+
case 4: _.label++; return { value: op[1], done: false };
|
|
36
|
+
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
37
|
+
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
38
|
+
default:
|
|
39
|
+
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
40
|
+
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
41
|
+
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
42
|
+
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
43
|
+
if (t[2]) _.ops.pop();
|
|
44
|
+
_.trys.pop(); continue;
|
|
45
|
+
}
|
|
46
|
+
op = body.call(thisArg, _);
|
|
47
|
+
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
48
|
+
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
49
|
+
}
|
|
50
|
+
};
|
|
51
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
52
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
53
|
+
};
|
|
54
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
55
|
+
exports.MySqlProcessor = void 0;
|
|
56
|
+
var stream_1 = require("stream");
|
|
57
|
+
var database_processor_1 = require("./base/database-processor");
|
|
58
|
+
var promise_1 = __importDefault(require("mysql2/promise"));
|
|
59
|
+
var MySqlProcessor = /** @class */ (function (_super) {
|
|
60
|
+
__extends(MySqlProcessor, _super);
|
|
61
|
+
function MySqlProcessor(config) {
|
|
62
|
+
var _this = _super.call(this, config) || this;
|
|
63
|
+
_this.connection = config.connection;
|
|
64
|
+
return _this;
|
|
65
|
+
}
|
|
66
|
+
MySqlProcessor.prototype.process = function (processStream) {
|
|
67
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
68
|
+
var dbConnection, query, stream;
|
|
69
|
+
return __generator(this, function (_a) {
|
|
70
|
+
switch (_a.label) {
|
|
71
|
+
case 0:
|
|
72
|
+
dbConnection = promise_1.default.createPool(this.connection);
|
|
73
|
+
query = dbConnection.pool.query(this.query);
|
|
74
|
+
stream = new stream_1.Readable({ objectMode: true });
|
|
75
|
+
stream._read = function () {
|
|
76
|
+
//dbConnection && dbConnection.resume();
|
|
77
|
+
};
|
|
78
|
+
query.on('result', function (row) {
|
|
79
|
+
if (!stream.push(row)) {
|
|
80
|
+
dbConnection.pause();
|
|
81
|
+
}
|
|
82
|
+
stream.emit('result', row);
|
|
83
|
+
});
|
|
84
|
+
query.on('error', function (err) {
|
|
85
|
+
stream.emit('error', err);
|
|
86
|
+
});
|
|
87
|
+
query.on('end', function () {
|
|
88
|
+
stream.push(null);
|
|
89
|
+
setImmediate(function () {
|
|
90
|
+
stream.emit('close');
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
query.on('fields', function (fields) {
|
|
94
|
+
stream.emit('fields', fields);
|
|
95
|
+
});
|
|
96
|
+
return [4 /*yield*/, processStream(stream)];
|
|
97
|
+
case 1:
|
|
98
|
+
_a.sent();
|
|
99
|
+
return [4 /*yield*/, dbConnection.end()];
|
|
100
|
+
case 2:
|
|
101
|
+
_a.sent();
|
|
102
|
+
return [2 /*return*/];
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
});
|
|
106
|
+
};
|
|
107
|
+
return MySqlProcessor;
|
|
108
|
+
}(database_processor_1.DatabaseProcessor));
|
|
109
|
+
exports.MySqlProcessor = MySqlProcessor;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { IPostgresProcessorConfig, processStream } from '../types';
|
|
1
|
+
import { IPostgresProcessorConfig, IPostgresDbConnection, processStream } from '../types';
|
|
2
2
|
import { DatabaseProcessor } from './base/database-processor';
|
|
3
3
|
export declare class PostgresProcessor extends DatabaseProcessor {
|
|
4
|
+
protected connection: IPostgresDbConnection;
|
|
4
5
|
constructor(config: IPostgresProcessorConfig);
|
|
5
6
|
process(processStream: processStream): Promise<void>;
|
|
6
7
|
}
|
|
@@ -56,7 +56,9 @@ var QueryStream = require('pg-query-stream');
|
|
|
56
56
|
var PostgresProcessor = /** @class */ (function (_super) {
|
|
57
57
|
__extends(PostgresProcessor, _super);
|
|
58
58
|
function PostgresProcessor(config) {
|
|
59
|
-
|
|
59
|
+
var _this = _super.call(this, config) || this;
|
|
60
|
+
_this.connection = config.connection;
|
|
61
|
+
return _this;
|
|
60
62
|
}
|
|
61
63
|
PostgresProcessor.prototype.process = function (processStream) {
|
|
62
64
|
return __awaiter(this, void 0, void 0, function () {
|
package/lib/types.d.ts
CHANGED
|
@@ -9,15 +9,32 @@ export interface AuthorizationRequest {
|
|
|
9
9
|
destinationField: string;
|
|
10
10
|
prefix?: string;
|
|
11
11
|
}
|
|
12
|
-
export interface
|
|
12
|
+
export interface IPostgresDbConnection {
|
|
13
13
|
user?: string;
|
|
14
14
|
password?: string;
|
|
15
|
-
server?: string;
|
|
16
15
|
host?: string;
|
|
17
16
|
database?: string;
|
|
18
17
|
schema?: string;
|
|
19
|
-
driver?: 'tedious' | 'msnodesqlv8';
|
|
20
18
|
}
|
|
19
|
+
export interface ITdsDriver {
|
|
20
|
+
ConnectionPool: any;
|
|
21
|
+
Transaction: any;
|
|
22
|
+
}
|
|
23
|
+
export interface IMssqlDbConnection {
|
|
24
|
+
user?: string;
|
|
25
|
+
password?: string;
|
|
26
|
+
server?: string;
|
|
27
|
+
database?: string;
|
|
28
|
+
driver?: ITdsDriver;
|
|
29
|
+
options?: any;
|
|
30
|
+
}
|
|
31
|
+
export interface IMySqlDbConnection {
|
|
32
|
+
user?: string;
|
|
33
|
+
password?: string;
|
|
34
|
+
host?: string;
|
|
35
|
+
database?: string;
|
|
36
|
+
}
|
|
37
|
+
export declare type IDbConnection = IPostgresDbConnection | IMssqlDbConnection | IMySqlDbConnection;
|
|
21
38
|
export interface IReporter {
|
|
22
39
|
report(job: IJob): Promise<void> | void;
|
|
23
40
|
}
|
|
@@ -68,12 +85,16 @@ export interface IDestinationConfig {
|
|
|
68
85
|
}
|
|
69
86
|
export interface IDatabaseDestinationConfig extends IDestinationConfig {
|
|
70
87
|
table: string;
|
|
71
|
-
connection: IDbConnection;
|
|
72
88
|
}
|
|
73
89
|
export interface IPostgresDestinationConfig extends IDatabaseDestinationConfig {
|
|
90
|
+
connection: IPostgresDbConnection;
|
|
74
91
|
upsertConstraints?: string[];
|
|
75
92
|
}
|
|
76
93
|
export interface IMssqlDestinationConfig extends IDatabaseDestinationConfig {
|
|
94
|
+
connection: IMssqlDbConnection;
|
|
95
|
+
}
|
|
96
|
+
export interface IMySqlDestinationConfig extends IDatabaseDestinationConfig {
|
|
97
|
+
connection: IMssqlDbConnection;
|
|
77
98
|
}
|
|
78
99
|
export interface IHttpDestinationConfig extends IDestinationConfig {
|
|
79
100
|
request: AxiosRequestConfig;
|
|
@@ -103,7 +124,6 @@ export interface IDelimitedHttpProcessorConfig extends IHttpProcessorConfig {
|
|
|
103
124
|
qualifier?: string;
|
|
104
125
|
}
|
|
105
126
|
export interface IDatabaseProcessorConfig extends IProcessorConfig {
|
|
106
|
-
connection: any;
|
|
107
127
|
query: string;
|
|
108
128
|
}
|
|
109
129
|
export interface IJsonProcessorConfig extends IDirectoryProcessorConfig {
|
|
@@ -116,8 +136,13 @@ export interface IDelimitedProcessorConfig extends IDirectoryProcessorConfig {
|
|
|
116
136
|
qualifier?: string;
|
|
117
137
|
}
|
|
118
138
|
export interface IMssqlProcessorConfig extends IDatabaseProcessorConfig {
|
|
139
|
+
connection: IMssqlDbConnection;
|
|
119
140
|
}
|
|
120
141
|
export interface IPostgresProcessorConfig extends IDatabaseProcessorConfig {
|
|
142
|
+
connection: IPostgresDbConnection;
|
|
143
|
+
}
|
|
144
|
+
export interface IMySqlProcessorConfig extends IDatabaseProcessorConfig {
|
|
145
|
+
connection: IMySqlDbConnection;
|
|
121
146
|
}
|
|
122
147
|
export interface ITailProcessorConfig extends IDirectoryProcessorConfig {
|
|
123
148
|
fromBeginning?: boolean;
|
|
@@ -135,7 +160,7 @@ export interface IBellboyEvent {
|
|
|
135
160
|
timestamp: number;
|
|
136
161
|
jobStopped: boolean;
|
|
137
162
|
}
|
|
138
|
-
export declare type DbTypes = 'postgres' | 'mssql';
|
|
163
|
+
export declare type DbTypes = 'postgres' | 'mssql' | 'mysql';
|
|
139
164
|
export declare type event = (...args: any) => Promise<any>;
|
|
140
165
|
export declare type extendedEvent = (bellboyEvent: IBellboyEvent) => Promise<any>;
|
|
141
166
|
export declare type sheetGetter = (sheets: IWorksheet[]) => Promise<string[]>;
|
package/lib/utils.js
CHANGED
|
@@ -90,70 +90,114 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
90
90
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
91
91
|
exports.applyHttpAuthorization = exports.getDelimitedGenerator = exports.removeCircularReferencesFromChunk = exports.getValueFromJSONChunk = exports.closeDbConnection = exports.getDb = void 0;
|
|
92
92
|
var axios_1 = __importDefault(require("axios"));
|
|
93
|
+
var promise_1 = __importDefault(require("mysql2/promise"));
|
|
93
94
|
var stream_1 = require("stream");
|
|
94
95
|
var cachedDbConnections = new Map();
|
|
96
|
+
function getCachedDbConnection(databaseConfig) {
|
|
97
|
+
var dbKey = getDbKey(databaseConfig);
|
|
98
|
+
return cachedDbConnections.get(dbKey);
|
|
99
|
+
}
|
|
100
|
+
function getDbKey(databaseConfig) {
|
|
101
|
+
if ('driver' in databaseConfig) {
|
|
102
|
+
var configWithoutDriver = __assign({}, databaseConfig);
|
|
103
|
+
delete configWithoutDriver.driver;
|
|
104
|
+
return JSON.stringify(configWithoutDriver);
|
|
105
|
+
}
|
|
106
|
+
return JSON.stringify(databaseConfig);
|
|
107
|
+
}
|
|
108
|
+
function setCachedDbConnection(databaseConfig, db, close) {
|
|
109
|
+
var dbKey = getDbKey(databaseConfig);
|
|
110
|
+
cachedDbConnections.set(dbKey, { db: db, close: close });
|
|
111
|
+
}
|
|
112
|
+
function unsetCachedDbConnection(databaseConfig) {
|
|
113
|
+
var dbKey = getDbKey(databaseConfig);
|
|
114
|
+
cachedDbConnections.delete(dbKey);
|
|
115
|
+
}
|
|
95
116
|
function getDb(databaseConfig, dbType) {
|
|
96
117
|
return __awaiter(this, void 0, void 0, function () {
|
|
97
|
-
var
|
|
118
|
+
var dbConnection, _a;
|
|
98
119
|
return __generator(this, function (_b) {
|
|
99
120
|
switch (_b.label) {
|
|
100
121
|
case 0:
|
|
101
|
-
|
|
102
|
-
if (
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
122
|
+
dbConnection = getCachedDbConnection(databaseConfig);
|
|
123
|
+
if (dbConnection) {
|
|
124
|
+
return [2 /*return*/, dbConnection.db];
|
|
125
|
+
}
|
|
126
|
+
_a = dbType;
|
|
127
|
+
switch (_a) {
|
|
128
|
+
case 'mssql': return [3 /*break*/, 1];
|
|
129
|
+
case 'mysql': return [3 /*break*/, 3];
|
|
130
|
+
case 'postgres': return [3 /*break*/, 4];
|
|
107
131
|
}
|
|
108
|
-
if (!(dbType === 'mssql')) return [3 /*break*/, 6];
|
|
109
|
-
if (!(databaseConfig.driver === 'msnodesqlv8')) return [3 /*break*/, 2];
|
|
110
|
-
return [4 /*yield*/, Promise.resolve().then(function () { return __importStar(require('mssql/msnodesqlv8')); })];
|
|
111
|
-
case 1:
|
|
112
|
-
_a = _b.sent();
|
|
113
132
|
return [3 /*break*/, 4];
|
|
114
|
-
case
|
|
133
|
+
case 1: return [4 /*yield*/, getMssqlDb(databaseConfig)];
|
|
134
|
+
case 2: return [2 /*return*/, _b.sent()];
|
|
135
|
+
case 3: return [2 /*return*/, getMySqlDb(databaseConfig)];
|
|
136
|
+
case 4: return [2 /*return*/, getPostgresDb(databaseConfig)];
|
|
137
|
+
}
|
|
138
|
+
});
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
exports.getDb = getDb;
|
|
142
|
+
function getMySqlDb(databaseConfig) {
|
|
143
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
144
|
+
var dbConnection;
|
|
145
|
+
return __generator(this, function (_a) {
|
|
146
|
+
dbConnection = promise_1.default.createPool(databaseConfig);
|
|
147
|
+
setCachedDbConnection(databaseConfig, dbConnection, dbConnection.end.bind(dbConnection));
|
|
148
|
+
return [2 /*return*/, dbConnection];
|
|
149
|
+
});
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
function getMssqlDb(databaseConfig) {
|
|
153
|
+
var _a;
|
|
154
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
155
|
+
var driver, _b, pool, db;
|
|
156
|
+
return __generator(this, function (_c) {
|
|
157
|
+
switch (_c.label) {
|
|
158
|
+
case 0:
|
|
159
|
+
if (!((_a = databaseConfig.driver) !== null && _a !== void 0)) return [3 /*break*/, 1];
|
|
160
|
+
_b = _a;
|
|
161
|
+
return [3 /*break*/, 3];
|
|
162
|
+
case 1: return [4 /*yield*/, Promise.resolve().then(function () { return __importStar(require('mssql')); })];
|
|
163
|
+
case 2:
|
|
164
|
+
_b = _c.sent();
|
|
165
|
+
_c.label = 3;
|
|
115
166
|
case 3:
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
case 4:
|
|
119
|
-
sql = _a;
|
|
120
|
-
pool = new sql.ConnectionPool(__assign({}, databaseConfig));
|
|
167
|
+
driver = _b;
|
|
168
|
+
pool = getMssqlDbPool(driver, databaseConfig);
|
|
121
169
|
return [4 /*yield*/, pool.connect()];
|
|
122
|
-
case
|
|
123
|
-
db =
|
|
124
|
-
|
|
125
|
-
db: db,
|
|
126
|
-
close: pool.close.bind(pool),
|
|
127
|
-
});
|
|
128
|
-
return [2 /*return*/, db];
|
|
129
|
-
case 6:
|
|
130
|
-
pgp = require('pg-promise')({ schema: databaseConfig.schema || 'public', });
|
|
131
|
-
db = pgp(databaseConfig);
|
|
132
|
-
cachedDbConnections.set(dbKey, {
|
|
133
|
-
db: db,
|
|
134
|
-
close: pgp.end,
|
|
135
|
-
});
|
|
170
|
+
case 4:
|
|
171
|
+
db = _c.sent();
|
|
172
|
+
setCachedDbConnection(databaseConfig, db, pool.close.bind(pool));
|
|
136
173
|
return [2 /*return*/, db];
|
|
137
174
|
}
|
|
138
175
|
});
|
|
139
176
|
});
|
|
140
177
|
}
|
|
141
|
-
|
|
142
|
-
;
|
|
178
|
+
function getMssqlDbPool(driver, databaseConfig) {
|
|
179
|
+
var poolConfig = __assign({}, databaseConfig);
|
|
180
|
+
delete poolConfig.driver;
|
|
181
|
+
return new driver.ConnectionPool(__assign({}, databaseConfig));
|
|
182
|
+
}
|
|
183
|
+
function getPostgresDb(databaseConfig) {
|
|
184
|
+
var pgp = require('pg-promise')({ schema: databaseConfig.schema || 'public' });
|
|
185
|
+
var db = pgp(databaseConfig);
|
|
186
|
+
setCachedDbConnection(databaseConfig, db, pgp.end);
|
|
187
|
+
return db;
|
|
188
|
+
}
|
|
143
189
|
function closeDbConnection(databaseConfig) {
|
|
144
190
|
return __awaiter(this, void 0, void 0, function () {
|
|
145
|
-
var
|
|
191
|
+
var dbConnection;
|
|
146
192
|
return __generator(this, function (_a) {
|
|
147
193
|
switch (_a.label) {
|
|
148
194
|
case 0:
|
|
149
|
-
|
|
150
|
-
if (!cachedDbConnections.has(dbKey)) return [3 /*break*/, 2];
|
|
151
|
-
dbConnection = cachedDbConnections.get(dbKey);
|
|
195
|
+
dbConnection = getCachedDbConnection(databaseConfig);
|
|
152
196
|
if (!dbConnection) return [3 /*break*/, 2];
|
|
153
197
|
return [4 /*yield*/, dbConnection.close()];
|
|
154
198
|
case 1:
|
|
155
199
|
_a.sent();
|
|
156
|
-
|
|
200
|
+
unsetCachedDbConnection(databaseConfig);
|
|
157
201
|
_a.label = 2;
|
|
158
202
|
case 2: return [2 /*return*/];
|
|
159
203
|
}
|
|
@@ -173,7 +217,6 @@ function getValueFromJSONChunk() {
|
|
|
173
217
|
});
|
|
174
218
|
}
|
|
175
219
|
exports.getValueFromJSONChunk = getValueFromJSONChunk;
|
|
176
|
-
;
|
|
177
220
|
function removeCircularReferencesFromChunk() {
|
|
178
221
|
return new stream_1.Transform({
|
|
179
222
|
objectMode: true,
|
|
@@ -196,7 +239,6 @@ function removeCircularReferencesFromChunk() {
|
|
|
196
239
|
});
|
|
197
240
|
}
|
|
198
241
|
exports.removeCircularReferencesFromChunk = removeCircularReferencesFromChunk;
|
|
199
|
-
;
|
|
200
242
|
function getDelimitedGenerator(_a) {
|
|
201
243
|
var readStream = _a.readStream, hasHeader = _a.hasHeader;
|
|
202
244
|
var header = [];
|
package/package.json
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bellboy",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "8.1.0",
|
|
4
4
|
"description": "Highly performant JavaScript data stream ETL engine.",
|
|
5
5
|
"main": "lib/index",
|
|
6
6
|
"types": "lib/index",
|
|
7
7
|
"scripts": {
|
|
8
8
|
"build": "tsc -p .",
|
|
9
9
|
"test": "jest --coverage",
|
|
10
|
+
"lint": "tslint -c tslint.json -p tsconfig.json",
|
|
10
11
|
"codecov": "codecov"
|
|
11
12
|
},
|
|
12
13
|
"repository": {
|
|
@@ -23,9 +24,14 @@
|
|
|
23
24
|
"url": "https://github.com/Claviz/bellboy/issues"
|
|
24
25
|
},
|
|
25
26
|
"homepage": "https://github.com/Claviz/bellboy#readme",
|
|
26
|
-
"
|
|
27
|
+
"peerDependencies": {
|
|
27
28
|
"msnodesqlv8": "^2.1.0"
|
|
28
29
|
},
|
|
30
|
+
"peerDependenciesMeta": {
|
|
31
|
+
"msnodesqlv8": {
|
|
32
|
+
"optional": true
|
|
33
|
+
}
|
|
34
|
+
},
|
|
29
35
|
"devDependencies": {
|
|
30
36
|
"@types/axios": "^0.14.0",
|
|
31
37
|
"@types/jest": "^25.2.1",
|
|
@@ -49,6 +55,7 @@
|
|
|
49
55
|
"axios": "^0.21.1",
|
|
50
56
|
"csv": "^6.0.5",
|
|
51
57
|
"mssql": "8.1.0",
|
|
58
|
+
"mysql2": "^3.6.5",
|
|
52
59
|
"pg": "8.0.3",
|
|
53
60
|
"pg-promise": "10.5.3",
|
|
54
61
|
"pg-query-stream": "3.0.7",
|