bellboy 8.0.0 → 8.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +58 -16
- package/lib/destinations/mysql-destination.d.ts +7 -0
- package/lib/destinations/mysql-destination.js +114 -0
- package/lib/index.d.ts +2 -0
- package/lib/index.js +5 -1
- package/lib/job.js +7 -5
- package/lib/processors/mysql-processor.d.ts +7 -0
- package/lib/processors/mysql-processor.js +109 -0
- package/lib/types.d.ts +14 -2
- package/lib/utils.js +25 -7
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -307,6 +307,7 @@ Each processor in `bellboy` is a class which has a single responsibility of proc
|
|
|
307
307
|
- [JsonProcessor](#json-processor) processes **JSON** file data from the file system.
|
|
308
308
|
- [DelimitedProcessor](#delimited-processor) processes files with **delimited data** from the file system.
|
|
309
309
|
- [PostgresProcessor](#postgres-processor) processes data received from a **PostgreSQL** SELECT.
|
|
310
|
+
- [MySqlProcessor](#mysql-processor) processes data received from a **MySQL** SELECT.
|
|
310
311
|
- [MssqlProcessor](#mssql-processor) processes data received from a **MSSQL** SELECT.
|
|
311
312
|
- [DynamicProcessor](#dynamic-processor) processes **dynamically generated** data.
|
|
312
313
|
- [TailProcessor](#tail-processor) processes **new lines** added to the file.
|
|
@@ -526,6 +527,22 @@ Processes a PostgreSQL `SELECT` query row by row.
|
|
|
526
527
|
- **database**
|
|
527
528
|
- **schema**
|
|
528
529
|
|
|
530
|
+
### MySqlProcessor <div id='mysql-processor'/>
|
|
531
|
+
|
|
532
|
+
Processes a MySQL `SELECT` query row by row.
|
|
533
|
+
|
|
534
|
+
#### Options
|
|
535
|
+
|
|
536
|
+
- [Processor options](#processor-options)
|
|
537
|
+
- **query** `string` `required`\
|
|
538
|
+
Query to execute.
|
|
539
|
+
- **connection** `object` `required`
|
|
540
|
+
- **user**
|
|
541
|
+
- **password**
|
|
542
|
+
- **host**
|
|
543
|
+
- **port**
|
|
544
|
+
- **database**
|
|
545
|
+
|
|
529
546
|
### MssqlProcessor <div id='mssql-processor'/>
|
|
530
547
|
|
|
531
548
|
Processes a MSSQL `SELECT` query row by row.
|
|
@@ -542,22 +559,25 @@ Processes a MSSQL `SELECT` query row by row.
|
|
|
542
559
|
- **port**
|
|
543
560
|
- **database**
|
|
544
561
|
- **driver**\
|
|
545
|
-
|
|
562
|
+
Optional [mssql][mssql-url] TDS driver; defaults to the pure JavaScript [Tedious][tedious-url] driver.
|
|
546
563
|
|
|
547
564
|
#### Usage
|
|
548
565
|
|
|
549
566
|
Here is an example of how to configure `MssqlProcessor` with a native TDS driver instead of the default pure JavasScript Tedious driver.
|
|
550
567
|
|
|
551
568
|
```javascript
|
|
552
|
-
const nativeDriver: ITdsDriver = await import(
|
|
569
|
+
const nativeDriver: ITdsDriver = await import("mssql/msnodesqlV8");
|
|
553
570
|
const connection: IMssqlDbConnection = {
|
|
554
|
-
user:
|
|
555
|
-
password:
|
|
556
|
-
server:
|
|
557
|
-
database:
|
|
558
|
-
driver: nativeDriver
|
|
571
|
+
user: "user",
|
|
572
|
+
password: "password",
|
|
573
|
+
server: "server",
|
|
574
|
+
database: "database",
|
|
575
|
+
driver: nativeDriver,
|
|
559
576
|
};
|
|
560
|
-
const source = new MssqlProcessor({
|
|
577
|
+
const source = new MssqlProcessor({
|
|
578
|
+
connection,
|
|
579
|
+
query: "select * from orders",
|
|
580
|
+
});
|
|
561
581
|
```
|
|
562
582
|
|
|
563
583
|
In previous versions of `bellboy`, `connection.driver` was a `string` parameter.
|
|
@@ -592,6 +612,7 @@ Every [job](#job) can have as many destinations (outputs) as needed. For example
|
|
|
592
612
|
- [StdoutDestination](#stdout-destination) logs data to **console**.
|
|
593
613
|
- [HttpDestination](#http-destination) executes **HTTP** request calls.
|
|
594
614
|
- [PostgresDestination](#postgres-destination) inserts/upserts data to **PostgreSQL** database.
|
|
615
|
+
- [MySqlDestination](#mysql-destination) inserts/upserts data to **MySQL** database.
|
|
595
616
|
- [MssqlDestination](#mssql-destination) inserts data to **MSSQL** database.
|
|
596
617
|
|
|
597
618
|
### Options <div id='destination-options'/>
|
|
@@ -658,6 +679,23 @@ Inserts data to PostgreSQL.
|
|
|
658
679
|
- **database**
|
|
659
680
|
- **schema**
|
|
660
681
|
|
|
682
|
+
### MySqlDestination <div id='mysql-destination'/>
|
|
683
|
+
|
|
684
|
+
[Usage examples](tests/mysql-destination.spec.ts)
|
|
685
|
+
|
|
686
|
+
Inserts data to MySQL.
|
|
687
|
+
|
|
688
|
+
#### Options
|
|
689
|
+
|
|
690
|
+
- [General destination options](#destination-options)
|
|
691
|
+
- **table** `string` `required`\
|
|
692
|
+
Table name.
|
|
693
|
+
- **connection** `object` `required`
|
|
694
|
+
- **user**
|
|
695
|
+
- **password**
|
|
696
|
+
- **host**
|
|
697
|
+
- **database**
|
|
698
|
+
|
|
661
699
|
### MssqlDestination <div id='mssql-destination'/>
|
|
662
700
|
|
|
663
701
|
[Usage examples](tests/mssql-destination.spec.ts)
|
|
@@ -675,22 +713,26 @@ Inserts data to MSSQL.
|
|
|
675
713
|
- **server**
|
|
676
714
|
- **database**
|
|
677
715
|
- **driver** \
|
|
678
|
-
|
|
716
|
+
Optional [mssql][mssql-url] TDS driver; defaults to the pure JavaScript [Tedious][tedious-url] driver.
|
|
679
717
|
|
|
680
718
|
#### Usage
|
|
681
719
|
|
|
682
720
|
Here is an example of how to configure `MssqlDestination` with a native TDS driver instead of the default pure JavasScript Tedious driver.
|
|
683
721
|
|
|
684
722
|
```javascript
|
|
685
|
-
const nativeDriver: ITdsDriver = await import(
|
|
723
|
+
const nativeDriver: ITdsDriver = await import("mssql/msnodesqlV8");
|
|
686
724
|
const connection: IMssqlDbConnection = {
|
|
687
|
-
user:
|
|
688
|
-
password:
|
|
689
|
-
server:
|
|
690
|
-
database:
|
|
691
|
-
driver: nativeDriver
|
|
725
|
+
user: "user",
|
|
726
|
+
password: "password",
|
|
727
|
+
server: "server",
|
|
728
|
+
database: "database",
|
|
729
|
+
driver: nativeDriver,
|
|
692
730
|
};
|
|
693
|
-
const sink = new MssqlDestination({
|
|
731
|
+
const sink = new MssqlDestination({
|
|
732
|
+
connection,
|
|
733
|
+
table: "orders",
|
|
734
|
+
batchSize: 1000,
|
|
735
|
+
});
|
|
694
736
|
```
|
|
695
737
|
|
|
696
738
|
[More usage examples](tests/mssql-destination.spec.ts)
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { IMySqlDestinationConfig, IMySqlDbConnection } from '../types';
|
|
2
|
+
import { DatabaseDestination } from './base/database-destination';
|
|
3
|
+
export declare class MySqlDestination extends DatabaseDestination {
|
|
4
|
+
protected connection: IMySqlDbConnection;
|
|
5
|
+
constructor(config: IMySqlDestinationConfig);
|
|
6
|
+
loadBatch(data: any[]): Promise<void>;
|
|
7
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __extends = (this && this.__extends) || (function () {
|
|
3
|
+
var extendStatics = function (d, b) {
|
|
4
|
+
extendStatics = Object.setPrototypeOf ||
|
|
5
|
+
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
|
|
6
|
+
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
|
|
7
|
+
return extendStatics(d, b);
|
|
8
|
+
};
|
|
9
|
+
return function (d, b) {
|
|
10
|
+
extendStatics(d, b);
|
|
11
|
+
function __() { this.constructor = d; }
|
|
12
|
+
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
|
|
13
|
+
};
|
|
14
|
+
})();
|
|
15
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
16
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
17
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
18
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
19
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
20
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
21
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
22
|
+
});
|
|
23
|
+
};
|
|
24
|
+
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
25
|
+
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
26
|
+
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
27
|
+
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
28
|
+
function step(op) {
|
|
29
|
+
if (f) throw new TypeError("Generator is already executing.");
|
|
30
|
+
while (_) try {
|
|
31
|
+
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
32
|
+
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
33
|
+
switch (op[0]) {
|
|
34
|
+
case 0: case 1: t = op; break;
|
|
35
|
+
case 4: _.label++; return { value: op[1], done: false };
|
|
36
|
+
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
37
|
+
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
38
|
+
default:
|
|
39
|
+
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
40
|
+
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
41
|
+
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
42
|
+
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
43
|
+
if (t[2]) _.ops.pop();
|
|
44
|
+
_.trys.pop(); continue;
|
|
45
|
+
}
|
|
46
|
+
op = body.call(thisArg, _);
|
|
47
|
+
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
48
|
+
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
49
|
+
}
|
|
50
|
+
};
|
|
51
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
52
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
53
|
+
};
|
|
54
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
55
|
+
exports.MySqlDestination = void 0;
|
|
56
|
+
var database_destination_1 = require("./base/database-destination");
|
|
57
|
+
var promise_1 = __importDefault(require("mysql2/promise"));
|
|
58
|
+
var MySqlDestination = /** @class */ (function (_super) {
|
|
59
|
+
__extends(MySqlDestination, _super);
|
|
60
|
+
function MySqlDestination(config) {
|
|
61
|
+
var _this = _super.call(this, config) || this;
|
|
62
|
+
_this.connection = config.connection;
|
|
63
|
+
return _this;
|
|
64
|
+
}
|
|
65
|
+
MySqlDestination.prototype.loadBatch = function (data) {
|
|
66
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
67
|
+
var pool, dbConnection, columns, columnNames, insertQuery, insertData, err_1;
|
|
68
|
+
return __generator(this, function (_a) {
|
|
69
|
+
switch (_a.label) {
|
|
70
|
+
case 0:
|
|
71
|
+
pool = promise_1.default.createPool(this.connection);
|
|
72
|
+
return [4 /*yield*/, pool.getConnection()];
|
|
73
|
+
case 1:
|
|
74
|
+
dbConnection = _a.sent();
|
|
75
|
+
return [4 /*yield*/, dbConnection.query("DESCRIBE " + this.table)];
|
|
76
|
+
case 2:
|
|
77
|
+
columns = (_a.sent())[0];
|
|
78
|
+
columnNames = columns.map(function (col) { return "`" + col.Field + "`"; });
|
|
79
|
+
insertQuery = "INSERT INTO " + this.table + " (" + columnNames.join(', ') + ") VALUES ?";
|
|
80
|
+
return [4 /*yield*/, dbConnection.beginTransaction()];
|
|
81
|
+
case 3:
|
|
82
|
+
_a.sent();
|
|
83
|
+
_a.label = 4;
|
|
84
|
+
case 4:
|
|
85
|
+
_a.trys.push([4, 8, 10, 12]);
|
|
86
|
+
insertData = data.map(function (row) { return columnNames.map(function (column) { return row[column.replace(/`/g, '')]; }); });
|
|
87
|
+
return [4 /*yield*/, dbConnection.beginTransaction()];
|
|
88
|
+
case 5:
|
|
89
|
+
_a.sent();
|
|
90
|
+
return [4 /*yield*/, dbConnection.query(insertQuery, [insertData])];
|
|
91
|
+
case 6:
|
|
92
|
+
_a.sent();
|
|
93
|
+
return [4 /*yield*/, dbConnection.commit()];
|
|
94
|
+
case 7:
|
|
95
|
+
_a.sent();
|
|
96
|
+
return [3 /*break*/, 12];
|
|
97
|
+
case 8:
|
|
98
|
+
err_1 = _a.sent();
|
|
99
|
+
return [4 /*yield*/, dbConnection.rollback()];
|
|
100
|
+
case 9:
|
|
101
|
+
_a.sent();
|
|
102
|
+
throw err_1;
|
|
103
|
+
case 10: return [4 /*yield*/, pool.end()];
|
|
104
|
+
case 11:
|
|
105
|
+
_a.sent();
|
|
106
|
+
return [7 /*endfinally*/];
|
|
107
|
+
case 12: return [2 /*return*/];
|
|
108
|
+
}
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
};
|
|
112
|
+
return MySqlDestination;
|
|
113
|
+
}(database_destination_1.DatabaseDestination));
|
|
114
|
+
exports.MySqlDestination = MySqlDestination;
|
package/lib/index.d.ts
CHANGED
|
@@ -4,6 +4,7 @@ export { DatabaseProcessor } from './processors/base/database-processor';
|
|
|
4
4
|
export { DirectoryProcessor } from './processors/base/directory-processor';
|
|
5
5
|
export { ExcelProcessor } from './processors/excel-processor';
|
|
6
6
|
export { PostgresProcessor } from './processors/postgres-processor';
|
|
7
|
+
export { MySqlProcessor } from './processors/mysql-processor';
|
|
7
8
|
export { MssqlProcessor } from './processors/mssql-processor';
|
|
8
9
|
export { HttpProcessor } from './processors/http-processor';
|
|
9
10
|
export { DynamicProcessor } from './processors/dynamic-processor';
|
|
@@ -17,6 +18,7 @@ export { HttpDestination } from './destinations/http-destination';
|
|
|
17
18
|
export { MssqlDestination } from './destinations/mssql-destination';
|
|
18
19
|
export { PostgresDestination } from './destinations/postgres-destination';
|
|
19
20
|
export { StdoutDestination } from './destinations/stdout-destination';
|
|
21
|
+
export { MySqlDestination } from './destinations/mysql-destination';
|
|
20
22
|
export { getDb } from './utils';
|
|
21
23
|
export { Reporter } from './reporters/base/reporter';
|
|
22
24
|
export { IBellboyEvent } from './types';
|
package/lib/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Reporter = exports.getDb = exports.StdoutDestination = exports.PostgresDestination = exports.MssqlDestination = exports.HttpDestination = exports.DatabaseDestination = exports.Destination = exports.DelimitedProcessor = exports.TailProcessor = exports.MqttProcessor = exports.JsonProcessor = exports.DynamicProcessor = exports.HttpProcessor = exports.MssqlProcessor = exports.PostgresProcessor = exports.ExcelProcessor = exports.DirectoryProcessor = exports.DatabaseProcessor = exports.Processor = exports.Job = void 0;
|
|
3
|
+
exports.Reporter = exports.getDb = exports.MySqlDestination = exports.StdoutDestination = exports.PostgresDestination = exports.MssqlDestination = exports.HttpDestination = exports.DatabaseDestination = exports.Destination = exports.DelimitedProcessor = exports.TailProcessor = exports.MqttProcessor = exports.JsonProcessor = exports.DynamicProcessor = exports.HttpProcessor = exports.MssqlProcessor = exports.MySqlProcessor = exports.PostgresProcessor = exports.ExcelProcessor = exports.DirectoryProcessor = exports.DatabaseProcessor = exports.Processor = exports.Job = void 0;
|
|
4
4
|
var job_1 = require("./job");
|
|
5
5
|
Object.defineProperty(exports, "Job", { enumerable: true, get: function () { return job_1.Job; } });
|
|
6
6
|
var processor_1 = require("./processors/base/processor");
|
|
@@ -13,6 +13,8 @@ var excel_processor_1 = require("./processors/excel-processor");
|
|
|
13
13
|
Object.defineProperty(exports, "ExcelProcessor", { enumerable: true, get: function () { return excel_processor_1.ExcelProcessor; } });
|
|
14
14
|
var postgres_processor_1 = require("./processors/postgres-processor");
|
|
15
15
|
Object.defineProperty(exports, "PostgresProcessor", { enumerable: true, get: function () { return postgres_processor_1.PostgresProcessor; } });
|
|
16
|
+
var mysql_processor_1 = require("./processors/mysql-processor");
|
|
17
|
+
Object.defineProperty(exports, "MySqlProcessor", { enumerable: true, get: function () { return mysql_processor_1.MySqlProcessor; } });
|
|
16
18
|
var mssql_processor_1 = require("./processors/mssql-processor");
|
|
17
19
|
Object.defineProperty(exports, "MssqlProcessor", { enumerable: true, get: function () { return mssql_processor_1.MssqlProcessor; } });
|
|
18
20
|
var http_processor_1 = require("./processors/http-processor");
|
|
@@ -39,6 +41,8 @@ var postgres_destination_1 = require("./destinations/postgres-destination");
|
|
|
39
41
|
Object.defineProperty(exports, "PostgresDestination", { enumerable: true, get: function () { return postgres_destination_1.PostgresDestination; } });
|
|
40
42
|
var stdout_destination_1 = require("./destinations/stdout-destination");
|
|
41
43
|
Object.defineProperty(exports, "StdoutDestination", { enumerable: true, get: function () { return stdout_destination_1.StdoutDestination; } });
|
|
44
|
+
var mysql_destination_1 = require("./destinations/mysql-destination");
|
|
45
|
+
Object.defineProperty(exports, "MySqlDestination", { enumerable: true, get: function () { return mysql_destination_1.MySqlDestination; } });
|
|
42
46
|
var utils_1 = require("./utils");
|
|
43
47
|
Object.defineProperty(exports, "getDb", { enumerable: true, get: function () { return utils_1.getDb; } });
|
|
44
48
|
var reporter_1 = require("./reporters/base/reporter");
|
package/lib/job.js
CHANGED
|
@@ -90,12 +90,14 @@ var Job = /** @class */ (function () {
|
|
|
90
90
|
case 4:
|
|
91
91
|
this.settingReporterEvents = false;
|
|
92
92
|
this.stop = function (message) {
|
|
93
|
-
_this.stopped
|
|
94
|
-
|
|
95
|
-
readStream.
|
|
96
|
-
|
|
93
|
+
if (!_this.stopped) {
|
|
94
|
+
_this.stopped = true;
|
|
95
|
+
if (readStream && (readStream instanceof stream_1.Readable)) {
|
|
96
|
+
readStream.emit('end');
|
|
97
|
+
readStream.destroy();
|
|
98
|
+
}
|
|
99
|
+
errorMessage = message;
|
|
97
100
|
}
|
|
98
|
-
errorMessage = message;
|
|
99
101
|
};
|
|
100
102
|
return [4 /*yield*/, this.emit('startProcessing', this.processor, this.destinations)];
|
|
101
103
|
case 5:
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { IMySqlDbConnection, IMySqlProcessorConfig, processStream } from '../types';
|
|
2
|
+
import { DatabaseProcessor } from './base/database-processor';
|
|
3
|
+
export declare class MySqlProcessor extends DatabaseProcessor {
|
|
4
|
+
protected connection: IMySqlDbConnection;
|
|
5
|
+
constructor(config: IMySqlProcessorConfig);
|
|
6
|
+
process(processStream: processStream): Promise<void>;
|
|
7
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __extends = (this && this.__extends) || (function () {
|
|
3
|
+
var extendStatics = function (d, b) {
|
|
4
|
+
extendStatics = Object.setPrototypeOf ||
|
|
5
|
+
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
|
|
6
|
+
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
|
|
7
|
+
return extendStatics(d, b);
|
|
8
|
+
};
|
|
9
|
+
return function (d, b) {
|
|
10
|
+
extendStatics(d, b);
|
|
11
|
+
function __() { this.constructor = d; }
|
|
12
|
+
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
|
|
13
|
+
};
|
|
14
|
+
})();
|
|
15
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
16
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
17
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
18
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
19
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
20
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
21
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
22
|
+
});
|
|
23
|
+
};
|
|
24
|
+
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
25
|
+
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
26
|
+
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
27
|
+
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
28
|
+
function step(op) {
|
|
29
|
+
if (f) throw new TypeError("Generator is already executing.");
|
|
30
|
+
while (_) try {
|
|
31
|
+
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
32
|
+
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
33
|
+
switch (op[0]) {
|
|
34
|
+
case 0: case 1: t = op; break;
|
|
35
|
+
case 4: _.label++; return { value: op[1], done: false };
|
|
36
|
+
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
37
|
+
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
38
|
+
default:
|
|
39
|
+
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
40
|
+
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
41
|
+
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
42
|
+
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
43
|
+
if (t[2]) _.ops.pop();
|
|
44
|
+
_.trys.pop(); continue;
|
|
45
|
+
}
|
|
46
|
+
op = body.call(thisArg, _);
|
|
47
|
+
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
48
|
+
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
49
|
+
}
|
|
50
|
+
};
|
|
51
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
52
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
53
|
+
};
|
|
54
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
55
|
+
exports.MySqlProcessor = void 0;
|
|
56
|
+
var stream_1 = require("stream");
|
|
57
|
+
var database_processor_1 = require("./base/database-processor");
|
|
58
|
+
var promise_1 = __importDefault(require("mysql2/promise"));
|
|
59
|
+
var MySqlProcessor = /** @class */ (function (_super) {
|
|
60
|
+
__extends(MySqlProcessor, _super);
|
|
61
|
+
function MySqlProcessor(config) {
|
|
62
|
+
var _this = _super.call(this, config) || this;
|
|
63
|
+
_this.connection = config.connection;
|
|
64
|
+
return _this;
|
|
65
|
+
}
|
|
66
|
+
MySqlProcessor.prototype.process = function (processStream) {
|
|
67
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
68
|
+
var dbConnection, query, stream;
|
|
69
|
+
return __generator(this, function (_a) {
|
|
70
|
+
switch (_a.label) {
|
|
71
|
+
case 0:
|
|
72
|
+
dbConnection = promise_1.default.createPool(this.connection);
|
|
73
|
+
query = dbConnection.pool.query(this.query);
|
|
74
|
+
stream = new stream_1.Readable({ objectMode: true });
|
|
75
|
+
stream._read = function () {
|
|
76
|
+
//dbConnection && dbConnection.resume();
|
|
77
|
+
};
|
|
78
|
+
query.on('result', function (row) {
|
|
79
|
+
if (!stream.push(row)) {
|
|
80
|
+
dbConnection.pause();
|
|
81
|
+
}
|
|
82
|
+
stream.emit('result', row);
|
|
83
|
+
});
|
|
84
|
+
query.on('error', function (err) {
|
|
85
|
+
stream.emit('error', err);
|
|
86
|
+
});
|
|
87
|
+
query.on('end', function () {
|
|
88
|
+
stream.push(null);
|
|
89
|
+
setImmediate(function () {
|
|
90
|
+
stream.emit('close');
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
query.on('fields', function (fields) {
|
|
94
|
+
stream.emit('fields', fields);
|
|
95
|
+
});
|
|
96
|
+
return [4 /*yield*/, processStream(stream)];
|
|
97
|
+
case 1:
|
|
98
|
+
_a.sent();
|
|
99
|
+
return [4 /*yield*/, dbConnection.end()];
|
|
100
|
+
case 2:
|
|
101
|
+
_a.sent();
|
|
102
|
+
return [2 /*return*/];
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
});
|
|
106
|
+
};
|
|
107
|
+
return MySqlProcessor;
|
|
108
|
+
}(database_processor_1.DatabaseProcessor));
|
|
109
|
+
exports.MySqlProcessor = MySqlProcessor;
|
package/lib/types.d.ts
CHANGED
|
@@ -28,7 +28,13 @@ export interface IMssqlDbConnection {
|
|
|
28
28
|
driver?: ITdsDriver;
|
|
29
29
|
options?: any;
|
|
30
30
|
}
|
|
31
|
-
export
|
|
31
|
+
export interface IMySqlDbConnection {
|
|
32
|
+
user?: string;
|
|
33
|
+
password?: string;
|
|
34
|
+
host?: string;
|
|
35
|
+
database?: string;
|
|
36
|
+
}
|
|
37
|
+
export declare type IDbConnection = IPostgresDbConnection | IMssqlDbConnection | IMySqlDbConnection;
|
|
32
38
|
export interface IReporter {
|
|
33
39
|
report(job: IJob): Promise<void> | void;
|
|
34
40
|
}
|
|
@@ -87,6 +93,9 @@ export interface IPostgresDestinationConfig extends IDatabaseDestinationConfig {
|
|
|
87
93
|
export interface IMssqlDestinationConfig extends IDatabaseDestinationConfig {
|
|
88
94
|
connection: IMssqlDbConnection;
|
|
89
95
|
}
|
|
96
|
+
export interface IMySqlDestinationConfig extends IDatabaseDestinationConfig {
|
|
97
|
+
connection: IMssqlDbConnection;
|
|
98
|
+
}
|
|
90
99
|
export interface IHttpDestinationConfig extends IDestinationConfig {
|
|
91
100
|
request: AxiosRequestConfig;
|
|
92
101
|
authorizationRequest?: AuthorizationRequest;
|
|
@@ -132,6 +141,9 @@ export interface IMssqlProcessorConfig extends IDatabaseProcessorConfig {
|
|
|
132
141
|
export interface IPostgresProcessorConfig extends IDatabaseProcessorConfig {
|
|
133
142
|
connection: IPostgresDbConnection;
|
|
134
143
|
}
|
|
144
|
+
export interface IMySqlProcessorConfig extends IDatabaseProcessorConfig {
|
|
145
|
+
connection: IMySqlDbConnection;
|
|
146
|
+
}
|
|
135
147
|
export interface ITailProcessorConfig extends IDirectoryProcessorConfig {
|
|
136
148
|
fromBeginning?: boolean;
|
|
137
149
|
}
|
|
@@ -148,7 +160,7 @@ export interface IBellboyEvent {
|
|
|
148
160
|
timestamp: number;
|
|
149
161
|
jobStopped: boolean;
|
|
150
162
|
}
|
|
151
|
-
export declare type DbTypes = 'postgres' | 'mssql';
|
|
163
|
+
export declare type DbTypes = 'postgres' | 'mssql' | 'mysql';
|
|
152
164
|
export declare type event = (...args: any) => Promise<any>;
|
|
153
165
|
export declare type extendedEvent = (bellboyEvent: IBellboyEvent) => Promise<any>;
|
|
154
166
|
export declare type sheetGetter = (sheets: IWorksheet[]) => Promise<string[]>;
|
package/lib/utils.js
CHANGED
|
@@ -90,6 +90,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
90
90
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
91
91
|
exports.applyHttpAuthorization = exports.getDelimitedGenerator = exports.removeCircularReferencesFromChunk = exports.getValueFromJSONChunk = exports.closeDbConnection = exports.getDb = void 0;
|
|
92
92
|
var axios_1 = __importDefault(require("axios"));
|
|
93
|
+
var promise_1 = __importDefault(require("mysql2/promise"));
|
|
93
94
|
var stream_1 = require("stream");
|
|
94
95
|
var cachedDbConnections = new Map();
|
|
95
96
|
function getCachedDbConnection(databaseConfig) {
|
|
@@ -114,23 +115,40 @@ function unsetCachedDbConnection(databaseConfig) {
|
|
|
114
115
|
}
|
|
115
116
|
function getDb(databaseConfig, dbType) {
|
|
116
117
|
return __awaiter(this, void 0, void 0, function () {
|
|
117
|
-
var dbConnection;
|
|
118
|
-
return __generator(this, function (
|
|
119
|
-
switch (
|
|
118
|
+
var dbConnection, _a;
|
|
119
|
+
return __generator(this, function (_b) {
|
|
120
|
+
switch (_b.label) {
|
|
120
121
|
case 0:
|
|
121
122
|
dbConnection = getCachedDbConnection(databaseConfig);
|
|
122
123
|
if (dbConnection) {
|
|
123
124
|
return [2 /*return*/, dbConnection.db];
|
|
124
125
|
}
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
126
|
+
_a = dbType;
|
|
127
|
+
switch (_a) {
|
|
128
|
+
case 'mssql': return [3 /*break*/, 1];
|
|
129
|
+
case 'mysql': return [3 /*break*/, 3];
|
|
130
|
+
case 'postgres': return [3 /*break*/, 4];
|
|
131
|
+
}
|
|
132
|
+
return [3 /*break*/, 4];
|
|
133
|
+
case 1: return [4 /*yield*/, getMssqlDb(databaseConfig)];
|
|
134
|
+
case 2: return [2 /*return*/, _b.sent()];
|
|
135
|
+
case 3: return [2 /*return*/, getMySqlDb(databaseConfig)];
|
|
136
|
+
case 4: return [2 /*return*/, getPostgresDb(databaseConfig)];
|
|
129
137
|
}
|
|
130
138
|
});
|
|
131
139
|
});
|
|
132
140
|
}
|
|
133
141
|
exports.getDb = getDb;
|
|
142
|
+
function getMySqlDb(databaseConfig) {
|
|
143
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
144
|
+
var dbConnection;
|
|
145
|
+
return __generator(this, function (_a) {
|
|
146
|
+
dbConnection = promise_1.default.createPool(databaseConfig);
|
|
147
|
+
setCachedDbConnection(databaseConfig, dbConnection, dbConnection.end.bind(dbConnection));
|
|
148
|
+
return [2 /*return*/, dbConnection];
|
|
149
|
+
});
|
|
150
|
+
});
|
|
151
|
+
}
|
|
134
152
|
function getMssqlDb(databaseConfig) {
|
|
135
153
|
var _a;
|
|
136
154
|
return __awaiter(this, void 0, void 0, function () {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bellboy",
|
|
3
|
-
"version": "8.
|
|
3
|
+
"version": "8.1.1",
|
|
4
4
|
"description": "Highly performant JavaScript data stream ETL engine.",
|
|
5
5
|
"main": "lib/index",
|
|
6
6
|
"types": "lib/index",
|
|
@@ -55,6 +55,7 @@
|
|
|
55
55
|
"axios": "^0.21.1",
|
|
56
56
|
"csv": "^6.0.5",
|
|
57
57
|
"mssql": "8.1.0",
|
|
58
|
+
"mysql2": "^3.6.5",
|
|
58
59
|
"pg": "8.0.3",
|
|
59
60
|
"pg-promise": "10.5.3",
|
|
60
61
|
"pg-query-stream": "3.0.7",
|