bellboy 8.5.1 → 8.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -714,6 +714,8 @@ Inserts data to MySQL.
714
714
  - **password**
715
715
  - **host**
716
716
  - **database**
717
+ - **useSourceColumns** `boolean` \
718
+ If `true`, only the columns in the source data will be used for data load. Default is `false`, using all destination table columns.
717
719
 
718
720
  ### MssqlDestination <div id='mssql-destination'/>
719
721
 
@@ -2,6 +2,7 @@ import { IMySqlDestinationConfig, IMySqlDbConnection } from '../types';
2
2
  import { DatabaseDestination } from './base/database-destination';
3
3
  export declare class MySqlDestination extends DatabaseDestination {
4
4
  protected connection: IMySqlDbConnection;
5
+ protected useSourceColumns: boolean;
5
6
  constructor(config: IMySqlDestinationConfig);
6
7
  loadBatch(data: any[]): Promise<void>;
7
8
  }
@@ -6,20 +6,31 @@ const database_destination_1 = require("./base/database-destination");
6
6
  const promise_1 = tslib_1.__importDefault(require("mysql2-stream-fix/promise"));
7
7
  class MySqlDestination extends database_destination_1.DatabaseDestination {
8
8
  connection;
9
+ useSourceColumns;
9
10
  constructor(config) {
10
11
  super(config);
11
12
  this.connection = config.connection;
13
+ this.useSourceColumns = config.useSourceColumns || false;
12
14
  }
13
15
  async loadBatch(data) {
14
16
  const pool = promise_1.default.createPool(this.connection);
15
17
  const dbConnection = await pool.getConnection();
16
- const [columns] = await dbConnection.query(`DESCRIBE ${this.table}`);
17
- const columnNames = columns.map((col) => `\`${col.Field}\``);
18
+ let columnNames;
19
+ if (this.useSourceColumns) {
20
+ const uniqueColumns = new Set();
21
+ data.forEach(row => {
22
+ Object.keys(row).forEach(key => uniqueColumns.add(key));
23
+ });
24
+ columnNames = Array.from(uniqueColumns).map(col => `\`${col}\``);
25
+ }
26
+ else {
27
+ const [columns] = await dbConnection.query(`DESCRIBE ${this.table}`);
28
+ columnNames = columns.map((col) => `\`${col.Field}\``);
29
+ }
18
30
  const insertQuery = `INSERT INTO ${this.table} (${columnNames.join(', ')}) VALUES ?`;
19
31
  await dbConnection.beginTransaction();
20
32
  try {
21
- const insertData = data.map(row => columnNames.map(column => row[column.replace(/`/g, '')])); // Remove backticks when accessing row data
22
- await dbConnection.beginTransaction();
33
+ const insertData = data.map(row => columnNames.map(column => row[column.replace(/`/g, '')] || null));
23
34
  await dbConnection.query(insertQuery, [insertData]);
24
35
  await dbConnection.commit();
25
36
  }
@@ -55,8 +55,28 @@ class HttpProcessor extends processor_1.Processor {
55
55
  this.authorizationRequest = config.authorizationRequest;
56
56
  }
57
57
  async processHttpData(processStream, options) {
58
+ const errorHandler = async (err) => {
59
+ const stream = err.response?.data;
60
+ if (stream) {
61
+ let streamString = '';
62
+ await new Promise((resolve, reject) => {
63
+ stream.setEncoding('utf8');
64
+ stream.on('data', (utf8Chunk) => {
65
+ streamString += utf8Chunk;
66
+ });
67
+ stream.on('end', () => {
68
+ err.message = `${err.message}. Response: ${streamString}`;
69
+ reject(err);
70
+ });
71
+ stream.on('error', () => {
72
+ reject(err);
73
+ });
74
+ });
75
+ }
76
+ throw err;
77
+ };
58
78
  if (this.dataFormat === 'delimited') {
59
- const requestStream = await (0, axios_1.default)({ ...options, responseType: 'stream' });
79
+ const requestStream = await (0, axios_1.default)({ ...options, responseType: 'stream' }).catch(errorHandler);
60
80
  const decodedStream = this.encoding ? requestStream.data.pipe(iconv_lite_1.default.decodeStream(this.encoding)) : requestStream.data;
61
81
  const parser = (0, csv_parse_1.parse)({
62
82
  quote: this.qualifier,
@@ -74,7 +94,7 @@ class HttpProcessor extends processor_1.Processor {
74
94
  await processStream(generator());
75
95
  }
76
96
  else if (this.dataFormat === 'json') {
77
- const requestStream = await (0, axios_1.default)({ ...options, responseType: 'stream' });
97
+ const requestStream = await (0, axios_1.default)({ ...options, responseType: 'stream', }).catch(errorHandler);
78
98
  const jsonStream = requestStream.data
79
99
  .pipe((0, Parser_1.parser)())
80
100
  .pipe((0, Pick_1.pick)({ filter: this.jsonPath || '' }))
@@ -83,7 +103,7 @@ class HttpProcessor extends processor_1.Processor {
83
103
  await processStream(jsonStream);
84
104
  }
85
105
  else if (this.dataFormat === 'xml') {
86
- const requestStream = await (0, axios_1.default)({ ...options, responseType: 'stream' });
106
+ const requestStream = await (0, axios_1.default)({ ...options, responseType: 'stream' }).catch(errorHandler);
87
107
  const xmlStream = requestStream.data
88
108
  .pipe(saxStream(this.saxOptions))
89
109
  .pipe((0, utils_1.removeCircularReferencesFromChunk)());
package/lib/types.d.ts CHANGED
@@ -103,6 +103,7 @@ export interface IMssqlDestinationConfig extends IDatabaseDestinationConfig {
103
103
  }
104
104
  export interface IMySqlDestinationConfig extends IDatabaseDestinationConfig {
105
105
  connection: IMssqlDbConnection;
106
+ useSourceColumns?: boolean;
106
107
  }
107
108
  export interface IHttpDestinationConfig extends IDestinationConfig {
108
109
  request: AxiosRequestConfig;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bellboy",
3
- "version": "8.5.1",
3
+ "version": "8.6.0",
4
4
  "description": "Highly performant JavaScript data stream ETL engine.",
5
5
  "main": "lib/index",
6
6
  "types": "lib/index",
@@ -41,9 +41,9 @@
41
41
  "@types/shortid": "0.0.29",
42
42
  "@types/stream-json": "1.5.1",
43
43
  "aedes": "0.42.1",
44
- "body-parser": "1.19.0",
44
+ "body-parser": "1.20.3",
45
45
  "codecov": "3.8.3",
46
- "express": "4.16.4",
46
+ "express": "4.21.0",
47
47
  "jest": "29.7.0",
48
48
  "node-xlsx": "0.15.0",
49
49
  "ts-jest": "29.2.5",
@@ -52,7 +52,7 @@
52
52
  },
53
53
  "dependencies": {
54
54
  "async-mqtt": "2.2.1",
55
- "axios": "0.21.1",
55
+ "axios": "1.7.7",
56
56
  "claviz-node-firebird": "2.0.1",
57
57
  "csv": "6.0.5",
58
58
  "iconv-lite": "0.6.3",
@@ -60,7 +60,7 @@
60
60
  "mysql2-stream-fix": "3.9.1",
61
61
  "node-firebird": "1.1.8",
62
62
  "pg": "8.0.3",
63
- "pg-promise": "10.5.3",
63
+ "pg-promise": "10.15.4",
64
64
  "pg-query-stream": "3.0.7",
65
65
  "sax-stream": "1.3.0",
66
66
  "shortid": "2.2.15",