bellboy 8.7.8 → 8.7.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,30 @@
1
1
  "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
2
25
  Object.defineProperty(exports, "__esModule", { value: true });
3
26
  exports.MssqlProcessor = void 0;
4
27
  const stream_1 = require("stream");
5
- const utils_1 = require("../utils");
6
28
  const database_processor_1 = require("./base/database-processor");
7
29
  class MssqlProcessor extends database_processor_1.DatabaseProcessor {
8
30
  connection;
@@ -33,7 +55,10 @@ class MssqlProcessor extends database_processor_1.DatabaseProcessor {
33
55
  },
34
56
  });
35
57
  const data = [];
36
- const db = await (0, utils_1.getDb)(this.connection, 'mssql');
58
+ // Each run owns its pool: a pool shared between jobs would be closed by the first one to finish.
59
+ const sql = this.connection.driver ?? await Promise.resolve().then(() => __importStar(require('mssql')));
60
+ const pool = new sql.ConnectionPool({ ...this.connection });
61
+ const db = await pool.connect();
37
62
  const dbRequest = db.request();
38
63
  dbRequest.on('error', (err) => {
39
64
  if (err.code !== 'ECANCEL') {
@@ -80,8 +105,12 @@ class MssqlProcessor extends database_processor_1.DatabaseProcessor {
80
105
  });
81
106
  };
82
107
  dbRequest.pause();
83
- await processStream(readStream);
84
- await (0, utils_1.closeDbConnection)(this.connection);
108
+ try {
109
+ await processStream(readStream);
110
+ }
111
+ finally {
112
+ await pool.close();
113
+ }
85
114
  }
86
115
  }
87
116
  exports.MssqlProcessor = MssqlProcessor;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bellboy",
3
- "version": "8.7.8",
3
+ "version": "8.7.9",
4
4
  "description": "Highly performant JavaScript data stream ETL engine.",
5
5
  "main": "lib/index",
6
6
  "types": "lib/index",
@@ -69,4 +69,4 @@
69
69
  "tail": "2.0.2",
70
70
  "xlstream": "2.5.4"
71
71
  }
72
- }
72
+ }
@@ -1,15 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.DatabaseDestination = void 0;
4
- const destination_1 = require("./destination");
5
- class DatabaseDestination extends destination_1.Destination {
6
- table;
7
- constructor(config) {
8
- super(config);
9
- this.table = config.table;
10
- }
11
- async loadBatch(data) {
12
- await super.loadBatch(data);
13
- }
14
- }
15
- exports.DatabaseDestination = DatabaseDestination;
@@ -1,23 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.Destination = void 0;
4
- class Destination {
5
- batchSize;
6
- recordGenerator;
7
- batchTransformer;
8
- disableLoad;
9
- constructor(config) {
10
- this.batchSize = 0;
11
- this.disableLoad = false;
12
- if (config) {
13
- this.batchSize = config.batchSize || 0;
14
- this.recordGenerator = config.recordGenerator;
15
- this.batchTransformer = config.batchTransformer;
16
- this.disableLoad = !!config.disableLoad;
17
- }
18
- }
19
- async loadBatch(data) {
20
- throw new Error('Method not implemented.');
21
- }
22
- }
23
- exports.Destination = Destination;
@@ -1,26 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.HttpDestination = void 0;
4
- const tslib_1 = require("tslib");
5
- const axios_1 = tslib_1.__importDefault(require("axios"));
6
- const utils_1 = require("../utils");
7
- const destination_1 = require("./base/destination");
8
- class HttpDestination extends destination_1.Destination {
9
- request;
10
- authorizationRequest;
11
- constructor(config) {
12
- super(config);
13
- this.request = config.request;
14
- this.authorizationRequest = config.authorizationRequest;
15
- }
16
- async loadBatch(data) {
17
- await (0, utils_1.applyHttpAuthorization)(this.request, this.authorizationRequest);
18
- for (let i = 0; i < data.length; i++) {
19
- await (0, axios_1.default)({
20
- ...this.request,
21
- data: data[i],
22
- });
23
- }
24
- }
25
- }
26
- exports.HttpDestination = HttpDestination;
@@ -1,64 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- Object.defineProperty(exports, "__esModule", { value: true });
26
- exports.MssqlDestination = void 0;
27
- const database_destination_1 = require("./base/database-destination");
28
- class MssqlDestination extends database_destination_1.DatabaseDestination {
29
- connection;
30
- constructor(config) {
31
- super(config);
32
- this.connection = config.connection;
33
- }
34
- async loadBatch(data) {
35
- const sql = this.connection.driver ?? await Promise.resolve().then(() => __importStar(require('mssql')));
36
- const pool = new sql.ConnectionPool({ ...this.connection });
37
- const db = await pool.connect();
38
- const query = await db.request().query(`SELECT TOP(0) * FROM ${this.table}`);
39
- let table = query.recordset.toTable(this.table);
40
- const columns = table.columns.map((x) => x.name);
41
- for (let x of data) {
42
- let args = [];
43
- for (let ii = 0; ii < columns.length; ii++) {
44
- args.push(x[columns[ii]]);
45
- }
46
- table.rows.add(...args);
47
- }
48
- const transaction = new sql.Transaction(db);
49
- try {
50
- await transaction.begin();
51
- await transaction.request().bulk(table);
52
- await transaction.commit();
53
- }
54
- catch (err) {
55
- if (!transaction._aborted) {
56
- await transaction.rollback();
57
- }
58
- await pool.close();
59
- throw err;
60
- }
61
- await pool.close();
62
- }
63
- }
64
- exports.MssqlDestination = MssqlDestination;
@@ -1,58 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.MySqlDestination = void 0;
4
- const tslib_1 = require("tslib");
5
- const database_destination_1 = require("./base/database-destination");
6
- const promise_1 = tslib_1.__importDefault(require("mysql2-stream-fix/promise"));
7
- class MySqlDestination extends database_destination_1.DatabaseDestination {
8
- connection;
9
- useSourceColumns;
10
- postLoadQuery;
11
- constructor(config) {
12
- super(config);
13
- this.connection = config.connection;
14
- this.useSourceColumns = config.useSourceColumns || false;
15
- this.postLoadQuery = config.postLoadQuery;
16
- }
17
- async loadBatch(data) {
18
- const pool = promise_1.default.createPool(this.connection);
19
- const dbConnection = await pool.getConnection();
20
- let columnNames;
21
- if (this.useSourceColumns) {
22
- const uniqueColumns = new Set();
23
- data.forEach(row => {
24
- Object.keys(row).forEach(key => uniqueColumns.add(key));
25
- });
26
- columnNames = Array.from(uniqueColumns).map(col => `\`${col}\``);
27
- }
28
- else {
29
- const [columns] = await dbConnection.query(`DESCRIBE ${this.table}`);
30
- columnNames = columns.map((col) => `\`${col.Field}\``);
31
- }
32
- const insertQuery = `INSERT INTO ${this.table} (${columnNames.join(', ')}) VALUES ?`;
33
- await dbConnection.beginTransaction();
34
- let result;
35
- try {
36
- const insertData = data.map(row => columnNames.map(column => row[column.replace(/`/g, '')]));
37
- await dbConnection.query(insertQuery, [insertData]);
38
- await dbConnection.commit();
39
- if (this.postLoadQuery) {
40
- result = await dbConnection.query(this.postLoadQuery);
41
- }
42
- }
43
- catch (err) {
44
- await dbConnection.rollback();
45
- throw err;
46
- }
47
- finally {
48
- try {
49
- await pool.end();
50
- }
51
- catch (poolCloseError) {
52
- throw poolCloseError;
53
- }
54
- }
55
- return result;
56
- }
57
- }
58
- exports.MySqlDestination = MySqlDestination;
@@ -1,56 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.PostgresDestination = void 0;
4
- const utils_1 = require("../utils");
5
- const database_destination_1 = require("./base/database-destination");
6
- const pgp = require('pg-promise')();
7
- class PostgresDestination extends database_destination_1.DatabaseDestination {
8
- connection;
9
- upsertConstraints;
10
- constructor(config) {
11
- super(config);
12
- this.connection = config.connection;
13
- this.upsertConstraints = config.upsertConstraints;
14
- }
15
- async loadBatch(data) {
16
- const dataToUpload = [];
17
- const columnDict = {};
18
- const upsertConstraints = this.upsertConstraints;
19
- let columnIndex = 0;
20
- for (let i = 0; i < data.length; i++) {
21
- const transformed = {};
22
- for (const key of Object.keys(data[i])) {
23
- if (!columnDict[key]) {
24
- columnDict[key] = `column_${columnIndex}`;
25
- columnIndex += 1;
26
- }
27
- transformed[columnDict[key]] = data[i][key];
28
- }
29
- dataToUpload.push(transformed);
30
- }
31
- const columnKeys = Object.values(columnDict);
32
- for (const item of dataToUpload) {
33
- const itemDataKeys = Object.keys(item);
34
- for (const key of columnKeys) {
35
- if (!itemDataKeys.includes(key)) {
36
- item[key] = undefined;
37
- }
38
- }
39
- }
40
- const columns = Object.keys(columnDict).map(x => ({ name: x, prop: columnDict[x] }));
41
- const cs = new pgp.helpers.ColumnSet(columns, { table: this.table });
42
- const db = await (0, utils_1.getDb)(this.connection, 'postgres');
43
- await db.tx(async (t) => {
44
- let query = pgp.helpers.insert(dataToUpload, cs);
45
- if (upsertConstraints && upsertConstraints.length) {
46
- const columnsString = upsertConstraints.map(x => `"${x}"`).join(',');
47
- query += ` on conflict(${columnsString}) do update set ` +
48
- cs.assignColumns({ from: 'EXCLUDED', skip: upsertConstraints });
49
- }
50
- return t.batch([
51
- t.none(query),
52
- ]);
53
- });
54
- }
55
- }
56
- exports.PostgresDestination = PostgresDestination;
@@ -1,18 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.StdoutDestination = void 0;
4
- const destination_1 = require("./base/destination");
5
- class StdoutDestination extends destination_1.Destination {
6
- asTable;
7
- constructor(config) {
8
- super(config);
9
- this.asTable = false;
10
- if (config) {
11
- this.asTable = config.asTable || false;
12
- }
13
- }
14
- async loadBatch(data) {
15
- this.asTable ? console.table(data) : console.log(data);
16
- }
17
- }
18
- exports.StdoutDestination = StdoutDestination;
package/lib/index.js DELETED
@@ -1,51 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.Reporter = exports.getDb = exports.MySqlDestination = exports.StdoutDestination = exports.PostgresDestination = exports.MssqlDestination = exports.HttpDestination = exports.DatabaseDestination = exports.Destination = exports.FirebirdProcessor = exports.DelimitedProcessor = exports.TailProcessor = exports.MqttProcessor = exports.JsonProcessor = exports.DynamicProcessor = exports.HttpProcessor = exports.MssqlProcessor = exports.MySqlProcessor = exports.PostgresProcessor = exports.ExcelProcessor = exports.DirectoryProcessor = exports.DatabaseProcessor = exports.Processor = exports.Job = void 0;
4
- var job_1 = require("./job");
5
- Object.defineProperty(exports, "Job", { enumerable: true, get: function () { return job_1.Job; } });
6
- var processor_1 = require("./processors/base/processor");
7
- Object.defineProperty(exports, "Processor", { enumerable: true, get: function () { return processor_1.Processor; } });
8
- var database_processor_1 = require("./processors/base/database-processor");
9
- Object.defineProperty(exports, "DatabaseProcessor", { enumerable: true, get: function () { return database_processor_1.DatabaseProcessor; } });
10
- var directory_processor_1 = require("./processors/base/directory-processor");
11
- Object.defineProperty(exports, "DirectoryProcessor", { enumerable: true, get: function () { return directory_processor_1.DirectoryProcessor; } });
12
- var excel_processor_1 = require("./processors/excel-processor");
13
- Object.defineProperty(exports, "ExcelProcessor", { enumerable: true, get: function () { return excel_processor_1.ExcelProcessor; } });
14
- var postgres_processor_1 = require("./processors/postgres-processor");
15
- Object.defineProperty(exports, "PostgresProcessor", { enumerable: true, get: function () { return postgres_processor_1.PostgresProcessor; } });
16
- var mysql_processor_1 = require("./processors/mysql-processor");
17
- Object.defineProperty(exports, "MySqlProcessor", { enumerable: true, get: function () { return mysql_processor_1.MySqlProcessor; } });
18
- var mssql_processor_1 = require("./processors/mssql-processor");
19
- Object.defineProperty(exports, "MssqlProcessor", { enumerable: true, get: function () { return mssql_processor_1.MssqlProcessor; } });
20
- var http_processor_1 = require("./processors/http-processor");
21
- Object.defineProperty(exports, "HttpProcessor", { enumerable: true, get: function () { return http_processor_1.HttpProcessor; } });
22
- var dynamic_processor_1 = require("./processors/dynamic-processor");
23
- Object.defineProperty(exports, "DynamicProcessor", { enumerable: true, get: function () { return dynamic_processor_1.DynamicProcessor; } });
24
- var json_processor_1 = require("./processors/json-processor");
25
- Object.defineProperty(exports, "JsonProcessor", { enumerable: true, get: function () { return json_processor_1.JsonProcessor; } });
26
- var mqtt_processor_1 = require("./processors/mqtt-processor");
27
- Object.defineProperty(exports, "MqttProcessor", { enumerable: true, get: function () { return mqtt_processor_1.MqttProcessor; } });
28
- var tail_processor_1 = require("./processors/tail-processor");
29
- Object.defineProperty(exports, "TailProcessor", { enumerable: true, get: function () { return tail_processor_1.TailProcessor; } });
30
- var delimited_processor_1 = require("./processors/delimited-processor");
31
- Object.defineProperty(exports, "DelimitedProcessor", { enumerable: true, get: function () { return delimited_processor_1.DelimitedProcessor; } });
32
- var firebird_processor_1 = require("./processors/firebird-processor");
33
- Object.defineProperty(exports, "FirebirdProcessor", { enumerable: true, get: function () { return firebird_processor_1.FirebirdProcessor; } });
34
- var destination_1 = require("./destinations/base/destination");
35
- Object.defineProperty(exports, "Destination", { enumerable: true, get: function () { return destination_1.Destination; } });
36
- var database_destination_1 = require("./destinations/base/database-destination");
37
- Object.defineProperty(exports, "DatabaseDestination", { enumerable: true, get: function () { return database_destination_1.DatabaseDestination; } });
38
- var http_destination_1 = require("./destinations/http-destination");
39
- Object.defineProperty(exports, "HttpDestination", { enumerable: true, get: function () { return http_destination_1.HttpDestination; } });
40
- var mssql_destination_1 = require("./destinations/mssql-destination");
41
- Object.defineProperty(exports, "MssqlDestination", { enumerable: true, get: function () { return mssql_destination_1.MssqlDestination; } });
42
- var postgres_destination_1 = require("./destinations/postgres-destination");
43
- Object.defineProperty(exports, "PostgresDestination", { enumerable: true, get: function () { return postgres_destination_1.PostgresDestination; } });
44
- var stdout_destination_1 = require("./destinations/stdout-destination");
45
- Object.defineProperty(exports, "StdoutDestination", { enumerable: true, get: function () { return stdout_destination_1.StdoutDestination; } });
46
- var mysql_destination_1 = require("./destinations/mysql-destination");
47
- Object.defineProperty(exports, "MySqlDestination", { enumerable: true, get: function () { return mysql_destination_1.MySqlDestination; } });
48
- var utils_1 = require("./utils");
49
- Object.defineProperty(exports, "getDb", { enumerable: true, get: function () { return utils_1.getDb; } });
50
- var reporter_1 = require("./reporters/base/reporter");
51
- Object.defineProperty(exports, "Reporter", { enumerable: true, get: function () { return reporter_1.Reporter; } });
package/lib/job.js DELETED
@@ -1,195 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.Job = void 0;
4
- const node_perf_hooks_1 = require("node:perf_hooks");
5
- const shortid_1 = require("shortid");
6
- const stream_1 = require("stream");
7
- class Job {
8
- processor;
9
- destinations;
10
- events = {};
11
- reporters;
12
- anyEvent = [];
13
- stopped = false;
14
- jobId = (0, shortid_1.generate)();
15
- jobName;
16
- settingReporterEvents = false;
17
- constructor(processor, destinations, config = {}) {
18
- this.processor = processor;
19
- this.destinations = destinations;
20
- this.jobName = config?.jobName;
21
- this.reporters = config.reporters || [];
22
- }
23
- stop(message) { }
24
- async run() {
25
- this.settingReporterEvents = true;
26
- for (let i = 0; i < this.reporters.length; i++) {
27
- await this.reporters[i].report(this);
28
- }
29
- this.settingReporterEvents = false;
30
- let readStream;
31
- let errorMessage;
32
- this.stop = (message) => {
33
- if (!this.stopped) {
34
- this.stopped = true;
35
- if (readStream && (readStream instanceof stream_1.Readable)) {
36
- readStream.emit('end');
37
- readStream.destroy();
38
- }
39
- errorMessage = message;
40
- }
41
- };
42
- await this.emit('startProcessing', this.processor, this.destinations);
43
- if (!this.stopped) {
44
- try {
45
- let e;
46
- await this.processor.process(async (_readStream, ...args) => {
47
- readStream = _readStream;
48
- await this.emit('startProcessingStream', ...args);
49
- try {
50
- if (!this.stopped) {
51
- const results = [];
52
- for (let j = 0; j < this.destinations.length; j++) {
53
- results[j] = [];
54
- }
55
- let processedRowCount = 0;
56
- for await (const row of readStream) {
57
- await this.emit('startProcessingRow', row);
58
- if (!this.stopped) {
59
- for (let i = 0; i < this.destinations.length; i++) {
60
- const recordGeneratorFn = this.destinations[i].recordGenerator;
61
- if (!recordGeneratorFn) {
62
- results[i].push(row);
63
- }
64
- else {
65
- try {
66
- for await (const record of recordGeneratorFn(row)) {
67
- results[i].push(record);
68
- await this.emit('rowGenerated', i, record);
69
- }
70
- }
71
- catch (err) {
72
- await this.emit('rowGenerationError', i, row, err);
73
- }
74
- }
75
- }
76
- }
77
- await this.emit('endProcessingRow');
78
- for (let i = 0; i < this.destinations.length; i++) {
79
- while (this.destinations[i].batchSize && results[i].length >= this.destinations[i].batchSize) {
80
- const destination = this.destinations[i];
81
- const toSend = results[i].splice(0, destination.batchSize);
82
- await this.loadBatch(i, destination, toSend);
83
- }
84
- }
85
- processedRowCount++;
86
- if (this.processor.rowLimit === processedRowCount) {
87
- await this.flushRows(results);
88
- this.stop();
89
- break;
90
- }
91
- }
92
- if (!this.stopped) {
93
- await this.flushRows(results);
94
- }
95
- }
96
- }
97
- catch (err) {
98
- e = err;
99
- }
100
- await this.emit('endProcessingStream', ...args);
101
- });
102
- if (e) {
103
- throw e;
104
- }
105
- }
106
- catch (err) {
107
- await this.emit('processingError', err);
108
- }
109
- }
110
- await this.emit('endProcessing');
111
- if (errorMessage) {
112
- throw new Error(errorMessage);
113
- }
114
- }
115
- on(eventName, event, extendedEvent) {
116
- const eventEntry = { event, extendedEvent };
117
- if (!this.events[eventName]) {
118
- this.events[eventName] = [];
119
- }
120
- this.settingReporterEvents
121
- ? this.events[eventName].unshift(eventEntry)
122
- : this.events[eventName].push(eventEntry);
123
- }
124
- onAny(event, extendedEvent) {
125
- const eventEntry = { event, extendedEvent };
126
- this.settingReporterEvents
127
- ? this.anyEvent.unshift(eventEntry)
128
- : this.anyEvent.push(eventEntry);
129
- }
130
- async flushRows(rows) {
131
- for (let i = 0; i < this.destinations.length; i++) {
132
- if (rows[i].length) {
133
- await this.loadBatch(i, this.destinations[i], rows[i]);
134
- }
135
- }
136
- }
137
- async loadBatch(destinationIndex, destination, data) {
138
- if (data.length) {
139
- if (destination.batchTransformer) {
140
- await this.emit('transformingBatch', destinationIndex, data);
141
- if (!this.stopped) {
142
- try {
143
- data = await destination.batchTransformer(data);
144
- await this.emit('transformedBatch', destinationIndex, data);
145
- }
146
- catch (err) {
147
- await this.emit('transformingBatchError', destinationIndex, data, err);
148
- }
149
- }
150
- await this.emit('endTransformingBatch', destinationIndex);
151
- }
152
- if (!destination.disableLoad && !this.stopped) {
153
- await this.emit('loadingBatch', destinationIndex, data);
154
- if (!this.stopped) {
155
- try {
156
- const result = await destination.loadBatch(data);
157
- result ? await this.emit('loadedBatch', destinationIndex, data, result) : await this.emit('loadedBatch', destinationIndex, data);
158
- }
159
- catch (err) {
160
- await this.emit('loadingBatchError', destinationIndex, data, err);
161
- }
162
- }
163
- await this.emit('endLoadingBatch', destinationIndex);
164
- }
165
- }
166
- }
167
- async emit(eventName, ...args) {
168
- try {
169
- const fn = this.events[eventName];
170
- const bellboyEvent = {
171
- eventName,
172
- eventArguments: args,
173
- timestamp: node_perf_hooks_1.performance.now() + node_perf_hooks_1.performance.timeOrigin,
174
- jobStopped: this.stopped,
175
- eventId: (0, shortid_1.generate)(),
176
- jobId: this.jobId,
177
- jobName: this.jobName,
178
- };
179
- if (fn) {
180
- for (let i = 0; i < fn.length; i++) {
181
- await fn[i].event?.apply(this, args);
182
- await fn[i].extendedEvent?.apply(this, [bellboyEvent]);
183
- }
184
- }
185
- for (let i = 0; i < this.anyEvent.length; i++) {
186
- await this.anyEvent[i].event?.apply(this, [eventName, ...args]);
187
- await this.anyEvent[i].extendedEvent?.apply(this, [bellboyEvent]);
188
- }
189
- }
190
- catch (err) {
191
- console.error(`Warning. Exception was thrown inside ${eventName} event.`, err);
192
- }
193
- }
194
- }
195
- exports.Job = Job;
@@ -1,15 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.DatabaseProcessor = void 0;
4
- const processor_1 = require("./processor");
5
- class DatabaseProcessor extends processor_1.Processor {
6
- query;
7
- constructor(config) {
8
- super(config);
9
- if (!config.query) {
10
- throw new Error(`No query specified.`);
11
- }
12
- this.query = config.query;
13
- }
14
- }
15
- exports.DatabaseProcessor = DatabaseProcessor;
@@ -1,30 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.DirectoryProcessor = void 0;
4
- const tslib_1 = require("tslib");
5
- const fs_1 = tslib_1.__importDefault(require("fs"));
6
- const processor_1 = require("./processor");
7
- class DirectoryProcessor extends processor_1.Processor {
8
- path;
9
- files = [];
10
- constructor(config) {
11
- super(config);
12
- if (!config.path && (!config.files || config.files.length === 0)) {
13
- throw new Error(`No path or files specified.`);
14
- }
15
- this.path = config.path || '';
16
- let filePattern = `(.*?)`;
17
- if (config.filePattern) {
18
- filePattern = config.filePattern;
19
- }
20
- if (!config.files || config.files.length === 0) {
21
- const dir = fs_1.default.readdirSync(this.path);
22
- const regex = new RegExp(filePattern, 'g');
23
- this.files = dir.sort().filter((x) => x.match(regex));
24
- }
25
- else {
26
- this.files = config.files;
27
- }
28
- }
29
- }
30
- exports.DirectoryProcessor = DirectoryProcessor;
@@ -1,16 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.Processor = void 0;
4
- class Processor {
5
- rowLimit;
6
- constructor(config) {
7
- this.rowLimit = 0;
8
- if (config) {
9
- this.rowLimit = config.rowLimit || 0;
10
- }
11
- }
12
- async process(processStream) {
13
- throw new Error('Method not implemented.');
14
- }
15
- }
16
- exports.Processor = Processor;
@@ -1,52 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.DelimitedProcessor = void 0;
4
- const tslib_1 = require("tslib");
5
- const csv_parse_1 = require("csv-parse");
6
- const fs_1 = tslib_1.__importDefault(require("fs"));
7
- const path_1 = tslib_1.__importDefault(require("path"));
8
- const iconv_lite_1 = tslib_1.__importDefault(require("iconv-lite"));
9
- const utils_1 = require("../utils");
10
- const directory_processor_1 = require("./base/directory-processor");
11
- class DelimitedProcessor extends directory_processor_1.DirectoryProcessor {
12
- rowSeparator;
13
- hasHeader;
14
- delimiter;
15
- qualifier;
16
- encoding;
17
- constructor(config) {
18
- super(config);
19
- if (!config.rowSeparator) {
20
- throw new Error('No rowSeparator specified.');
21
- }
22
- this.rowSeparator = config.rowSeparator;
23
- this.hasHeader = !!config.hasHeader;
24
- this.delimiter = config.delimiter;
25
- this.qualifier = config.qualifier;
26
- this.encoding = config.encoding;
27
- }
28
- async process(processStream) {
29
- for (const file of this.files) {
30
- const filePath = path_1.default.join(this.path, file);
31
- const fileReadStream = fs_1.default.createReadStream(filePath);
32
- const streamToParse = this.encoding ? fileReadStream.pipe(iconv_lite_1.default.decodeStream(this.encoding)) : fileReadStream;
33
- const parser = (0, csv_parse_1.parse)({
34
- quote: this.qualifier,
35
- delimiter: this.delimiter,
36
- record_delimiter: this.rowSeparator,
37
- raw: true,
38
- relax_quotes: true,
39
- relax_column_count: true,
40
- });
41
- const readStream = streamToParse.pipe(parser);
42
- const generator = (0, utils_1.getDelimitedGenerator)({
43
- hasHeader: this.hasHeader,
44
- readStream,
45
- });
46
- await processStream(generator(), file, filePath);
47
- fileReadStream.destroy();
48
- }
49
- ;
50
- }
51
- }
52
- exports.DelimitedProcessor = DelimitedProcessor;
@@ -1,19 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.DynamicProcessor = void 0;
4
- const stream_1 = require("stream");
5
- const processor_1 = require("./base/processor");
6
- class DynamicProcessor extends processor_1.Processor {
7
- generator;
8
- constructor(config) {
9
- super(config);
10
- if (!config.generator) {
11
- throw Error(`No generator function specified.`);
12
- }
13
- this.generator = config.generator;
14
- }
15
- async process(processStream) {
16
- await processStream(stream_1.Readable.from(this.generator()));
17
- }
18
- }
19
- exports.DynamicProcessor = DynamicProcessor;
@@ -1,52 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.ExcelProcessor = void 0;
4
- const tslib_1 = require("tslib");
5
- const path_1 = tslib_1.__importDefault(require("path"));
6
- const xlstream_1 = require("xlstream");
7
- const directory_processor_1 = require("./base/directory-processor");
8
- class ExcelProcessor extends directory_processor_1.DirectoryProcessor {
9
- hasHeader;
10
- ignoreEmpty;
11
- encoding;
12
- fillMergedCells;
13
- sheets;
14
- constructor(config) {
15
- super(config);
16
- this.encoding = config.encoding;
17
- this.hasHeader = config.hasHeader;
18
- this.ignoreEmpty = config.ignoreEmpty === false ? false : true;
19
- this.fillMergedCells = !!config.fillMergedCells;
20
- if (config.sheets) {
21
- this.sheets = config.sheets;
22
- }
23
- else {
24
- this.sheets = [0];
25
- }
26
- }
27
- async process(processStream) {
28
- for (let file of this.files) {
29
- const filePath = path_1.default.join(this.path, file);
30
- let sheets = [];
31
- if (this.sheets instanceof Function) {
32
- const allSheets = await (0, xlstream_1.getWorksheets)({ filePath });
33
- sheets = await this.sheets(allSheets);
34
- }
35
- else {
36
- sheets = this.sheets;
37
- }
38
- for (let i = 0; i < sheets.length; i++) {
39
- const readStream = await (0, xlstream_1.getXlsxStream)({
40
- filePath,
41
- sheet: sheets[i],
42
- withHeader: this.hasHeader,
43
- ignoreEmpty: this.ignoreEmpty,
44
- fillMergedCells: this.fillMergedCells,
45
- encoding: this.encoding,
46
- });
47
- await processStream(readStream, file, filePath, sheets[i]);
48
- }
49
- }
50
- }
51
- }
52
- exports.ExcelProcessor = ExcelProcessor;
@@ -1,55 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.FirebirdProcessor = void 0;
4
- const stream_1 = require("stream");
5
- const database_processor_1 = require("./base/database-processor");
6
- class FirebirdProcessor extends database_processor_1.DatabaseProcessor {
7
- connection;
8
- constructor(config) {
9
- super(config);
10
- this.connection = config.connection;
11
- }
12
- async process(processStream) {
13
- const Firebird = this.connection.useClavizNodeFirebird
14
- ? require('claviz-node-firebird')
15
- : require('node-firebird');
16
- const db = await new Promise((resolve, reject) => {
17
- Firebird.attach(this.connection, (err, db) => {
18
- if (err) {
19
- reject(err);
20
- }
21
- else {
22
- resolve(db);
23
- }
24
- });
25
- });
26
- let resume = null;
27
- const stream = new stream_1.Readable({
28
- objectMode: true,
29
- read() {
30
- if (resume) {
31
- const r = resume;
32
- resume = null;
33
- r();
34
- }
35
- }
36
- });
37
- db.sequentially(this.query, [], (row) => {
38
- const ok = stream.push(row);
39
- if (ok) {
40
- return;
41
- }
42
- return new Promise((resolve) => {
43
- resume = resolve;
44
- });
45
- }, (err) => {
46
- if (err) {
47
- stream.emit('error', err);
48
- }
49
- stream.push(null);
50
- db.detach();
51
- });
52
- await processStream(stream);
53
- }
54
- }
55
- exports.FirebirdProcessor = FirebirdProcessor;
@@ -1,30 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.MqttProcessor = void 0;
4
- const stream_1 = require("stream");
5
- const processor_1 = require("./base/processor");
6
- let mqtt = require('async-mqtt');
7
- class MqttProcessor extends processor_1.Processor {
8
- url;
9
- topics;
10
- constructor(config) {
11
- super(config);
12
- this.url = config.url;
13
- this.topics = config.topics;
14
- }
15
- async process(processStream) {
16
- const readStream = new stream_1.Readable({
17
- objectMode: true,
18
- read() { },
19
- });
20
- const client = mqtt.connect(this.url);
21
- await client.subscribe(this.topics);
22
- client.on('message', (topic, message) => {
23
- readStream.push({ topic, message: message.toString() });
24
- });
25
- await processStream(readStream);
26
- await client.unsubscribe(this.topics);
27
- await client.end(true);
28
- }
29
- }
30
- exports.MqttProcessor = MqttProcessor;
@@ -1,20 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.MySqlProcessor = void 0;
4
- const tslib_1 = require("tslib");
5
- const database_processor_1 = require("./base/database-processor");
6
- const promise_1 = tslib_1.__importDefault(require("mysql2-stream-fix/promise"));
7
- class MySqlProcessor extends database_processor_1.DatabaseProcessor {
8
- connection;
9
- constructor(config) {
10
- super(config);
11
- this.connection = config.connection;
12
- }
13
- async process(processStream) {
14
- const dbConnection = promise_1.default.createPool(this.connection);
15
- const query = dbConnection.pool.query(this.query).stream();
16
- await processStream(query);
17
- await dbConnection.end();
18
- }
19
- }
20
- exports.MySqlProcessor = MySqlProcessor;
@@ -1,35 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.PostgresProcessor = void 0;
4
- const utils_1 = require("../utils");
5
- const database_processor_1 = require("./base/database-processor");
6
- const QueryStream = require('pg-query-stream');
7
- class PostgresProcessor extends database_processor_1.DatabaseProcessor {
8
- connection;
9
- constructor(config) {
10
- super(config);
11
- this.connection = config.connection;
12
- }
13
- async process(processStream) {
14
- const query = new QueryStream(this.query);
15
- const db = await (0, utils_1.getDb)(this.connection, 'postgres');
16
- let dbConnection;
17
- let stream;
18
- try {
19
- dbConnection = await db.connect();
20
- stream = dbConnection.client.query(query);
21
- await processStream(stream);
22
- }
23
- catch (err) {
24
- try {
25
- stream?.destroy();
26
- }
27
- catch (_) { }
28
- throw err;
29
- }
30
- finally {
31
- dbConnection?.done();
32
- }
33
- }
34
- }
35
- exports.PostgresProcessor = PostgresProcessor;
@@ -1,39 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.TailProcessor = void 0;
4
- const tslib_1 = require("tslib");
5
- const path_1 = tslib_1.__importDefault(require("path"));
6
- const stream_1 = require("stream");
7
- const directory_processor_1 = require("./base/directory-processor");
8
- const Tail = require('tail').Tail;
9
- class TailProcessor extends directory_processor_1.DirectoryProcessor {
10
- fromBeginning;
11
- constructor(config) {
12
- super(config);
13
- this.fromBeginning = !!config.fromBeginning;
14
- }
15
- async process(processStream) {
16
- const readStream = new stream_1.Readable({
17
- objectMode: true,
18
- read() { },
19
- });
20
- const tails = [];
21
- for (const file of this.files) {
22
- const filePath = path_1.default.join(this.path, file);
23
- const tail = new Tail(filePath, {
24
- fromBeginning: this.fromBeginning,
25
- });
26
- tail.on('line', (data) => {
27
- readStream.push({ file, data });
28
- });
29
- tail.on('error', (exception) => {
30
- readStream.emit('error', exception);
31
- });
32
- tails.push(tail);
33
- }
34
- ;
35
- await processStream(readStream);
36
- tails.forEach(tail => tail.unwatch());
37
- }
38
- }
39
- exports.TailProcessor = TailProcessor;
@@ -1,10 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.Reporter = void 0;
4
- class Reporter {
5
- constructor() { }
6
- report(job) {
7
- throw new Error('Method not implemented.');
8
- }
9
- }
10
- exports.Reporter = Reporter;
package/lib/types.js DELETED
@@ -1,2 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
package/lib/utils.js DELETED
@@ -1,178 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.getDb = getDb;
4
- exports.closeDbConnection = closeDbConnection;
5
- exports.getValueFromJSONChunk = getValueFromJSONChunk;
6
- exports.removeCircularReferencesFromChunk = removeCircularReferencesFromChunk;
7
- exports.getDelimitedGenerator = getDelimitedGenerator;
8
- exports.applyHttpAuthorization = applyHttpAuthorization;
9
- const tslib_1 = require("tslib");
10
- const axios_1 = tslib_1.__importDefault(require("axios"));
11
- const promise_1 = tslib_1.__importDefault(require("mysql2-stream-fix/promise"));
12
- const stream_1 = require("stream");
13
- const cachedDbConnections = new Map();
14
- function getCachedDbConnection(databaseConfig) {
15
- const dbKey = getDbKey(databaseConfig);
16
- return cachedDbConnections.get(dbKey);
17
- }
18
- function getDbKey(databaseConfig) {
19
- if ('driver' in databaseConfig) {
20
- const configWithoutDriver = { ...databaseConfig };
21
- delete configWithoutDriver.driver;
22
- return JSON.stringify(configWithoutDriver);
23
- }
24
- return JSON.stringify(databaseConfig);
25
- }
26
- function setCachedDbConnection(databaseConfig, db, close) {
27
- const dbKey = getDbKey(databaseConfig);
28
- cachedDbConnections.set(dbKey, { db, close });
29
- }
30
- function unsetCachedDbConnection(databaseConfig) {
31
- const dbKey = getDbKey(databaseConfig);
32
- cachedDbConnections.delete(dbKey);
33
- }
34
- async function getDb(databaseConfig, dbType) {
35
- const dbConnection = getCachedDbConnection(databaseConfig);
36
- if (dbConnection) {
37
- return dbConnection.db;
38
- }
39
- switch (dbType) {
40
- case 'mssql':
41
- return await getMssqlDb(databaseConfig);
42
- case 'firebird':
43
- return await getFirebirdDb(databaseConfig);
44
- case 'mysql':
45
- return getMySqlDb(databaseConfig);
46
- case 'postgres':
47
- default:
48
- return getPostgresDb(databaseConfig);
49
- }
50
- }
51
- async function getMySqlDb(databaseConfig) {
52
- const dbConnection = promise_1.default.createPool(databaseConfig);
53
- setCachedDbConnection(databaseConfig, dbConnection, dbConnection.end.bind(dbConnection));
54
- return dbConnection;
55
- }
56
- async function getMssqlDb(databaseConfig) {
57
- const driver = databaseConfig.driver ?? await Promise.resolve().then(() => tslib_1.__importStar(require('mssql')));
58
- const pool = getMssqlDbPool(driver, databaseConfig);
59
- const db = await pool.connect();
60
- setCachedDbConnection(databaseConfig, db, pool.close.bind(pool));
61
- return db;
62
- }
63
- function getMssqlDbPool(driver, databaseConfig) {
64
- const poolConfig = { ...databaseConfig };
65
- delete poolConfig.driver;
66
- return new driver.ConnectionPool({ ...databaseConfig });
67
- }
68
- function getPostgresDb(databaseConfig) {
69
- const pgp = require('pg-promise')({ schema: databaseConfig.schema || 'public' });
70
- const db = pgp(databaseConfig);
71
- setCachedDbConnection(databaseConfig, db, pgp.end);
72
- return db;
73
- }
74
- async function getFirebirdDb(databaseConfig) {
75
- const Firebird = databaseConfig.useClavizNodeFirebird ? require('claviz-node-firebird') : require('node-firebird');
76
- const db = await new Promise((resolve, reject) => {
77
- Firebird.attach(databaseConfig, (err, db) => {
78
- if (err)
79
- reject(err);
80
- else
81
- resolve(db);
82
- });
83
- });
84
- setCachedDbConnection(databaseConfig, db, db.detach.bind(db));
85
- return db;
86
- }
87
- async function closeDbConnection(databaseConfig) {
88
- const dbConnection = getCachedDbConnection(databaseConfig);
89
- if (dbConnection) {
90
- await dbConnection.close();
91
- unsetCachedDbConnection(databaseConfig);
92
- }
93
- }
94
- function getValueFromJSONChunk() {
95
- return new stream_1.Transform({
96
- objectMode: true,
97
- transform(chunk, encoding, callback) {
98
- if (chunk.value) {
99
- this.push(chunk.value);
100
- }
101
- callback();
102
- }
103
- });
104
- }
105
- function removeCircularReferencesFromChunk() {
106
- return new stream_1.Transform({
107
- objectMode: true,
108
- transform(chunk, encoding, callback) {
109
- function getCircularReplacer() {
110
- const seen = new WeakSet();
111
- return (key, value) => {
112
- if (typeof value === "object" && value !== null) {
113
- if (seen.has(value)) {
114
- return;
115
- }
116
- seen.add(value);
117
- }
118
- return value;
119
- };
120
- }
121
- this.push(JSON.parse(JSON.stringify(chunk, getCircularReplacer())));
122
- callback();
123
- }
124
- });
125
- }
126
- function getDelimitedGenerator({ readStream, hasHeader, }) {
127
- let header = [];
128
- const makeUniqueHeader = (names) => {
129
- const counts = new Map();
130
- const seen = new Map();
131
- for (const n of names) {
132
- const base = n.trim();
133
- counts.set(base, (counts.get(base) ?? 0) + 1);
134
- }
135
- return names.map((n) => {
136
- const base = n.trim();
137
- const total = counts.get(base) ?? 0;
138
- if (total <= 1) {
139
- return base;
140
- }
141
- const idx = (seen.get(base) ?? 0) + 1;
142
- seen.set(base, idx);
143
- return `${base} (${idx})`;
144
- });
145
- };
146
- const processRow = ({ raw, record }) => {
147
- if (hasHeader && header.length === 0) {
148
- header = makeUniqueHeader(record);
149
- return undefined;
150
- }
151
- let obj;
152
- if (header.length) {
153
- obj = Object.fromEntries(header.map((key, i) => [key, record[i]]));
154
- }
155
- return { header, arr: record, obj, row: raw };
156
- };
157
- const generator = async function* () {
158
- for await (const row of readStream) {
159
- const result = processRow(row);
160
- if (result) {
161
- yield result;
162
- }
163
- }
164
- };
165
- return generator;
166
- }
167
- async function applyHttpAuthorization(connection, authorizationRequest) {
168
- if (authorizationRequest) {
169
- const authorizationResponse = await (0, axios_1.default)(authorizationRequest.connection);
170
- let sourceField = authorizationRequest.sourceField.split('.');
171
- const authorizationToken = sourceField.reduce((acc, key) => acc[key], authorizationResponse.data);
172
- const applyTo = authorizationRequest.applyTo === 'header' ? 'headers' : 'params';
173
- connection[applyTo] = {
174
- ...connection[applyTo],
175
- [authorizationRequest.destinationField]: authorizationRequest.prefix ? `${authorizationRequest.prefix}${authorizationToken}` : authorizationToken,
176
- };
177
- }
178
- }