bellboy 8.5.2 → 8.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -3
- package/lib/destinations/mysql-destination.d.ts +3 -1
- package/lib/destinations/mysql-destination.js +30 -5
- package/lib/job.d.ts +1 -1
- package/lib/job.js +2 -2
- package/lib/processors/http-processor.js +23 -3
- package/lib/types.d.ts +3 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -207,9 +207,12 @@ job.on("loadingBatch", async (destinationIndex: number, data: any[]) => {
|
|
|
207
207
|
```
|
|
208
208
|
|
|
209
209
|
```ts
|
|
210
|
-
job.on(
|
|
211
|
-
|
|
212
|
-
|
|
210
|
+
job.on(
|
|
211
|
+
"loadedBatch",
|
|
212
|
+
async (destinationIndex: number, data: any[], result: any) => {
|
|
213
|
+
// Batch has been loaded into destination.
|
|
214
|
+
}
|
|
215
|
+
);
|
|
213
216
|
```
|
|
214
217
|
|
|
215
218
|
```ts
|
|
@@ -714,6 +717,10 @@ Inserts data to MySQL.
|
|
|
714
717
|
- **password**
|
|
715
718
|
- **host**
|
|
716
719
|
- **database**
|
|
720
|
+
- **useSourceColumns** `boolean` \
|
|
721
|
+
If `true`, only the columns in the source data will be used for data load. Default is `false`, using all destination table columns.
|
|
722
|
+
- **postLoadQuery** `string` \
|
|
723
|
+
A query which will be executed after the data load and before connection is closed. Result will be available in the `result` object of the `loadedBatch` event.
|
|
717
724
|
|
|
718
725
|
### MssqlDestination <div id='mssql-destination'/>
|
|
719
726
|
|
|
@@ -2,6 +2,8 @@ import { IMySqlDestinationConfig, IMySqlDbConnection } from '../types';
|
|
|
2
2
|
import { DatabaseDestination } from './base/database-destination';
|
|
3
3
|
export declare class MySqlDestination extends DatabaseDestination {
|
|
4
4
|
protected connection: IMySqlDbConnection;
|
|
5
|
+
protected useSourceColumns: boolean;
|
|
6
|
+
protected postLoadQuery?: string;
|
|
5
7
|
constructor(config: IMySqlDestinationConfig);
|
|
6
|
-
loadBatch(data: any[]): Promise<
|
|
8
|
+
loadBatch(data: any[]): Promise<any>;
|
|
7
9
|
}
|
|
@@ -6,20 +6,33 @@ const database_destination_1 = require("./base/database-destination");
|
|
|
6
6
|
const promise_1 = tslib_1.__importDefault(require("mysql2-stream-fix/promise"));
|
|
7
7
|
class MySqlDestination extends database_destination_1.DatabaseDestination {
|
|
8
8
|
connection;
|
|
9
|
+
useSourceColumns;
|
|
10
|
+
postLoadQuery;
|
|
9
11
|
constructor(config) {
|
|
10
12
|
super(config);
|
|
11
13
|
this.connection = config.connection;
|
|
14
|
+
this.useSourceColumns = config.useSourceColumns || false;
|
|
15
|
+
this.postLoadQuery = config.postLoadQuery;
|
|
12
16
|
}
|
|
13
17
|
async loadBatch(data) {
|
|
14
18
|
const pool = promise_1.default.createPool(this.connection);
|
|
15
19
|
const dbConnection = await pool.getConnection();
|
|
16
|
-
|
|
17
|
-
|
|
20
|
+
let columnNames;
|
|
21
|
+
if (this.useSourceColumns) {
|
|
22
|
+
const uniqueColumns = new Set();
|
|
23
|
+
data.forEach(row => {
|
|
24
|
+
Object.keys(row).forEach(key => uniqueColumns.add(key));
|
|
25
|
+
});
|
|
26
|
+
columnNames = Array.from(uniqueColumns).map(col => `\`${col}\``);
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
const [columns] = await dbConnection.query(`DESCRIBE ${this.table}`);
|
|
30
|
+
columnNames = columns.map((col) => `\`${col.Field}\``);
|
|
31
|
+
}
|
|
18
32
|
const insertQuery = `INSERT INTO ${this.table} (${columnNames.join(', ')}) VALUES ?`;
|
|
19
33
|
await dbConnection.beginTransaction();
|
|
20
34
|
try {
|
|
21
|
-
const insertData = data.map(row => columnNames.map(column => row[column.replace(/`/g, '')]));
|
|
22
|
-
await dbConnection.beginTransaction();
|
|
35
|
+
const insertData = data.map(row => columnNames.map(column => row[column.replace(/`/g, '')]));
|
|
23
36
|
await dbConnection.query(insertQuery, [insertData]);
|
|
24
37
|
await dbConnection.commit();
|
|
25
38
|
}
|
|
@@ -28,7 +41,19 @@ class MySqlDestination extends database_destination_1.DatabaseDestination {
|
|
|
28
41
|
throw err;
|
|
29
42
|
}
|
|
30
43
|
finally {
|
|
31
|
-
|
|
44
|
+
let result;
|
|
45
|
+
try {
|
|
46
|
+
if (this.postLoadQuery) {
|
|
47
|
+
result = await dbConnection.query(this.postLoadQuery);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
catch (postLoadError) {
|
|
51
|
+
throw postLoadError;
|
|
52
|
+
}
|
|
53
|
+
finally {
|
|
54
|
+
await pool.end();
|
|
55
|
+
}
|
|
56
|
+
return result;
|
|
32
57
|
}
|
|
33
58
|
}
|
|
34
59
|
}
|
package/lib/job.d.ts
CHANGED
|
@@ -21,7 +21,7 @@ export declare class Job implements IJob {
|
|
|
21
21
|
stop(message?: string): void;
|
|
22
22
|
run(): Promise<void>;
|
|
23
23
|
on(eventName: 'endProcessing', event?: (() => Promise<any>), extendedEvent?: extendedEvent): any;
|
|
24
|
-
on(eventName: 'loadedBatch', event?: ((destinationIndex: number, data: any[]) => Promise<any>), extendedEvent?: extendedEvent): any;
|
|
24
|
+
on(eventName: 'loadedBatch', event?: ((destinationIndex: number, data: any[], result?: any) => Promise<any>), extendedEvent?: extendedEvent): any;
|
|
25
25
|
on(eventName: 'processingError', event?: ((error: any) => Promise<any>), extendedEvent?: extendedEvent): any;
|
|
26
26
|
on(eventName: 'loadingBatchError', event?: ((destinationIndex: number, data: any[], error: any) => Promise<any>), extendedEvent?: extendedEvent): any;
|
|
27
27
|
on(eventName: 'loadingBatch', event?: ((destinationIndex: number, data: any[]) => Promise<any>), extendedEvent?: extendedEvent): any;
|
package/lib/job.js
CHANGED
|
@@ -153,8 +153,8 @@ class Job {
|
|
|
153
153
|
await this.emit('loadingBatch', destinationIndex, data);
|
|
154
154
|
if (!this.stopped) {
|
|
155
155
|
try {
|
|
156
|
-
await destination.loadBatch(data);
|
|
157
|
-
await this.emit('loadedBatch', destinationIndex, data);
|
|
156
|
+
const result = await destination.loadBatch(data);
|
|
157
|
+
result ? await this.emit('loadedBatch', destinationIndex, data, result) : await this.emit('loadedBatch', destinationIndex, data);
|
|
158
158
|
}
|
|
159
159
|
catch (err) {
|
|
160
160
|
await this.emit('loadingBatchError', destinationIndex, data, err);
|
|
@@ -55,8 +55,28 @@ class HttpProcessor extends processor_1.Processor {
|
|
|
55
55
|
this.authorizationRequest = config.authorizationRequest;
|
|
56
56
|
}
|
|
57
57
|
async processHttpData(processStream, options) {
|
|
58
|
+
const errorHandler = async (err) => {
|
|
59
|
+
const stream = err.response?.data;
|
|
60
|
+
if (stream) {
|
|
61
|
+
let streamString = '';
|
|
62
|
+
await new Promise((resolve, reject) => {
|
|
63
|
+
stream.setEncoding('utf8');
|
|
64
|
+
stream.on('data', (utf8Chunk) => {
|
|
65
|
+
streamString += utf8Chunk;
|
|
66
|
+
});
|
|
67
|
+
stream.on('end', () => {
|
|
68
|
+
err.message = `${err.message}. Response: ${streamString}`;
|
|
69
|
+
reject(err);
|
|
70
|
+
});
|
|
71
|
+
stream.on('error', () => {
|
|
72
|
+
reject(err);
|
|
73
|
+
});
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
throw err;
|
|
77
|
+
};
|
|
58
78
|
if (this.dataFormat === 'delimited') {
|
|
59
|
-
const requestStream = await (0, axios_1.default)({ ...options, responseType: 'stream' });
|
|
79
|
+
const requestStream = await (0, axios_1.default)({ ...options, responseType: 'stream' }).catch(errorHandler);
|
|
60
80
|
const decodedStream = this.encoding ? requestStream.data.pipe(iconv_lite_1.default.decodeStream(this.encoding)) : requestStream.data;
|
|
61
81
|
const parser = (0, csv_parse_1.parse)({
|
|
62
82
|
quote: this.qualifier,
|
|
@@ -74,7 +94,7 @@ class HttpProcessor extends processor_1.Processor {
|
|
|
74
94
|
await processStream(generator());
|
|
75
95
|
}
|
|
76
96
|
else if (this.dataFormat === 'json') {
|
|
77
|
-
const requestStream = await (0, axios_1.default)({ ...options, responseType: 'stream' });
|
|
97
|
+
const requestStream = await (0, axios_1.default)({ ...options, responseType: 'stream', }).catch(errorHandler);
|
|
78
98
|
const jsonStream = requestStream.data
|
|
79
99
|
.pipe((0, Parser_1.parser)())
|
|
80
100
|
.pipe((0, Pick_1.pick)({ filter: this.jsonPath || '' }))
|
|
@@ -83,7 +103,7 @@ class HttpProcessor extends processor_1.Processor {
|
|
|
83
103
|
await processStream(jsonStream);
|
|
84
104
|
}
|
|
85
105
|
else if (this.dataFormat === 'xml') {
|
|
86
|
-
const requestStream = await (0, axios_1.default)({ ...options, responseType: 'stream' });
|
|
106
|
+
const requestStream = await (0, axios_1.default)({ ...options, responseType: 'stream' }).catch(errorHandler);
|
|
87
107
|
const xmlStream = requestStream.data
|
|
88
108
|
.pipe(saxStream(this.saxOptions))
|
|
89
109
|
.pipe((0, utils_1.removeCircularReferencesFromChunk)());
|
package/lib/types.d.ts
CHANGED
|
@@ -79,7 +79,7 @@ export interface IProcessor {
|
|
|
79
79
|
rowLimit: number;
|
|
80
80
|
}
|
|
81
81
|
export interface IDestination {
|
|
82
|
-
loadBatch: (data: any[]) => Promise<
|
|
82
|
+
loadBatch: (data: any[]) => Promise<any>;
|
|
83
83
|
batchSize: number;
|
|
84
84
|
recordGenerator: ((row: any) => AsyncIterableIterator<{}>) | undefined;
|
|
85
85
|
batchTransformer: ((rows: any[]) => Promise<any[]>) | undefined;
|
|
@@ -103,6 +103,8 @@ export interface IMssqlDestinationConfig extends IDatabaseDestinationConfig {
|
|
|
103
103
|
}
|
|
104
104
|
export interface IMySqlDestinationConfig extends IDatabaseDestinationConfig {
|
|
105
105
|
connection: IMssqlDbConnection;
|
|
106
|
+
useSourceColumns?: boolean;
|
|
107
|
+
postLoadQuery?: string;
|
|
106
108
|
}
|
|
107
109
|
export interface IHttpDestinationConfig extends IDestinationConfig {
|
|
108
110
|
request: AxiosRequestConfig;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bellboy",
|
|
3
|
-
"version": "8.
|
|
3
|
+
"version": "8.7.0",
|
|
4
4
|
"description": "Highly performant JavaScript data stream ETL engine.",
|
|
5
5
|
"main": "lib/index",
|
|
6
6
|
"types": "lib/index",
|
|
@@ -43,7 +43,7 @@
|
|
|
43
43
|
"aedes": "0.42.1",
|
|
44
44
|
"body-parser": "1.20.3",
|
|
45
45
|
"codecov": "3.8.3",
|
|
46
|
-
"express": "4.21.
|
|
46
|
+
"express": "4.21.1",
|
|
47
47
|
"jest": "29.7.0",
|
|
48
48
|
"node-xlsx": "0.15.0",
|
|
49
49
|
"ts-jest": "29.2.5",
|