bellboy 8.4.1 → 8.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/lib/destinations/base/database-destination.js +10 -70
- package/lib/destinations/base/destination.js +10 -47
- package/lib/destinations/http-destination.js +20 -98
- package/lib/destinations/mssql-destination.js +41 -137
- package/lib/destinations/mysql-destination.js +30 -109
- package/lib/destinations/postgres-destination.js +49 -119
- package/lib/destinations/stdout-destination.js +11 -67
- package/lib/job.js +169 -424
- package/lib/processors/base/database-processor.js +8 -23
- package/lib/processors/base/directory-processor.js +16 -33
- package/lib/processors/base/processor.js +7 -47
- package/lib/processors/delimited-processor.js +43 -111
- package/lib/processors/dynamic-processor.js +12 -72
- package/lib/processors/excel-processor.js +43 -122
- package/lib/processors/firebird-processor.js +31 -96
- package/lib/processors/http-processor.js +82 -175
- package/lib/processors/json-processor.js +27 -99
- package/lib/processors/mqtt-processor.js +24 -91
- package/lib/processors/mssql-processor.js +79 -179
- package/lib/processors/mysql-processor.js +15 -81
- package/lib/processors/postgres-processor.js +17 -83
- package/lib/processors/tail-processor.js +33 -102
- package/lib/reporters/base/reporter.js +5 -7
- package/lib/types.d.ts +6 -7
- package/lib/utils.d.ts +0 -1
- package/lib/utils.js +91 -281
- package/package.json +31 -31
|
@@ -1,126 +1,56 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __extends = (this && this.__extends) || (function () {
|
|
3
|
-
var extendStatics = function (d, b) {
|
|
4
|
-
extendStatics = Object.setPrototypeOf ||
|
|
5
|
-
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
|
|
6
|
-
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
|
|
7
|
-
return extendStatics(d, b);
|
|
8
|
-
};
|
|
9
|
-
return function (d, b) {
|
|
10
|
-
extendStatics(d, b);
|
|
11
|
-
function __() { this.constructor = d; }
|
|
12
|
-
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
|
|
13
|
-
};
|
|
14
|
-
})();
|
|
15
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
16
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
17
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
18
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
19
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
20
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
21
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
22
|
-
});
|
|
23
|
-
};
|
|
24
|
-
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
25
|
-
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
26
|
-
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
27
|
-
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
28
|
-
function step(op) {
|
|
29
|
-
if (f) throw new TypeError("Generator is already executing.");
|
|
30
|
-
while (_) try {
|
|
31
|
-
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
32
|
-
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
33
|
-
switch (op[0]) {
|
|
34
|
-
case 0: case 1: t = op; break;
|
|
35
|
-
case 4: _.label++; return { value: op[1], done: false };
|
|
36
|
-
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
37
|
-
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
38
|
-
default:
|
|
39
|
-
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
40
|
-
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
41
|
-
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
42
|
-
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
43
|
-
if (t[2]) _.ops.pop();
|
|
44
|
-
_.trys.pop(); continue;
|
|
45
|
-
}
|
|
46
|
-
op = body.call(thisArg, _);
|
|
47
|
-
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
48
|
-
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
49
|
-
}
|
|
50
|
-
};
|
|
51
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
52
3
|
exports.PostgresDestination = void 0;
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
4
|
+
const utils_1 = require("../utils");
|
|
5
|
+
const database_destination_1 = require("./base/database-destination");
|
|
6
|
+
const pgp = require('pg-promise')();
|
|
7
|
+
class PostgresDestination extends database_destination_1.DatabaseDestination {
|
|
8
|
+
connection;
|
|
9
|
+
upsertConstraints;
|
|
10
|
+
constructor(config) {
|
|
11
|
+
super(config);
|
|
12
|
+
this.connection = config.connection;
|
|
13
|
+
this.upsertConstraints = config.upsertConstraints;
|
|
63
14
|
}
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
}
|
|
87
|
-
columnKeys = Object.values(columnDict);
|
|
88
|
-
for (_b = 0, dataToUpload_1 = dataToUpload; _b < dataToUpload_1.length; _b++) {
|
|
89
|
-
item = dataToUpload_1[_b];
|
|
90
|
-
itemDataKeys = Object.keys(item);
|
|
91
|
-
for (_c = 0, columnKeys_1 = columnKeys; _c < columnKeys_1.length; _c++) {
|
|
92
|
-
key = columnKeys_1[_c];
|
|
93
|
-
if (!itemDataKeys.includes(key)) {
|
|
94
|
-
item[key] = undefined;
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
columns = Object.keys(columnDict).map(function (x) { return ({ name: x, prop: columnDict[x] }); });
|
|
99
|
-
cs = new pgp.helpers.ColumnSet(columns, { table: this.table });
|
|
100
|
-
return [4 /*yield*/, utils_1.getDb(this.connection, 'postgres')];
|
|
101
|
-
case 1:
|
|
102
|
-
db = _d.sent();
|
|
103
|
-
return [4 /*yield*/, db.tx(function (t) { return __awaiter(_this, void 0, void 0, function () {
|
|
104
|
-
var query, columnsString;
|
|
105
|
-
return __generator(this, function (_a) {
|
|
106
|
-
query = pgp.helpers.insert(dataToUpload, cs);
|
|
107
|
-
if (upsertConstraints && upsertConstraints.length) {
|
|
108
|
-
columnsString = upsertConstraints.map(function (x) { return "\"" + x + "\""; }).join(',');
|
|
109
|
-
query += " on conflict(" + columnsString + ") do update set " +
|
|
110
|
-
cs.assignColumns({ from: 'EXCLUDED', skip: upsertConstraints });
|
|
111
|
-
}
|
|
112
|
-
return [2 /*return*/, t.batch([
|
|
113
|
-
t.none(query),
|
|
114
|
-
])];
|
|
115
|
-
});
|
|
116
|
-
}); })];
|
|
117
|
-
case 2:
|
|
118
|
-
_d.sent();
|
|
119
|
-
return [2 /*return*/];
|
|
15
|
+
async loadBatch(data) {
|
|
16
|
+
const dataToUpload = [];
|
|
17
|
+
const columnDict = {};
|
|
18
|
+
const upsertConstraints = this.upsertConstraints;
|
|
19
|
+
let columnIndex = 0;
|
|
20
|
+
for (let i = 0; i < data.length; i++) {
|
|
21
|
+
const transformed = {};
|
|
22
|
+
for (const key of Object.keys(data[i])) {
|
|
23
|
+
if (!columnDict[key]) {
|
|
24
|
+
columnDict[key] = `column_${columnIndex}`;
|
|
25
|
+
columnIndex += 1;
|
|
26
|
+
}
|
|
27
|
+
transformed[columnDict[key]] = data[i][key];
|
|
28
|
+
}
|
|
29
|
+
dataToUpload.push(transformed);
|
|
30
|
+
}
|
|
31
|
+
const columnKeys = Object.values(columnDict);
|
|
32
|
+
for (const item of dataToUpload) {
|
|
33
|
+
const itemDataKeys = Object.keys(item);
|
|
34
|
+
for (const key of columnKeys) {
|
|
35
|
+
if (!itemDataKeys.includes(key)) {
|
|
36
|
+
item[key] = undefined;
|
|
120
37
|
}
|
|
121
|
-
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
const columns = Object.keys(columnDict).map(x => ({ name: x, prop: columnDict[x] }));
|
|
41
|
+
const cs = new pgp.helpers.ColumnSet(columns, { table: this.table });
|
|
42
|
+
const db = await (0, utils_1.getDb)(this.connection, 'postgres');
|
|
43
|
+
await db.tx(async (t) => {
|
|
44
|
+
let query = pgp.helpers.insert(dataToUpload, cs);
|
|
45
|
+
if (upsertConstraints && upsertConstraints.length) {
|
|
46
|
+
const columnsString = upsertConstraints.map(x => `"${x}"`).join(',');
|
|
47
|
+
query += ` on conflict(${columnsString}) do update set ` +
|
|
48
|
+
cs.assignColumns({ from: 'EXCLUDED', skip: upsertConstraints });
|
|
49
|
+
}
|
|
50
|
+
return t.batch([
|
|
51
|
+
t.none(query),
|
|
52
|
+
]);
|
|
122
53
|
});
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
}(database_destination_1.DatabaseDestination));
|
|
54
|
+
}
|
|
55
|
+
}
|
|
126
56
|
exports.PostgresDestination = PostgresDestination;
|
|
@@ -1,74 +1,18 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __extends = (this && this.__extends) || (function () {
|
|
3
|
-
var extendStatics = function (d, b) {
|
|
4
|
-
extendStatics = Object.setPrototypeOf ||
|
|
5
|
-
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
|
|
6
|
-
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
|
|
7
|
-
return extendStatics(d, b);
|
|
8
|
-
};
|
|
9
|
-
return function (d, b) {
|
|
10
|
-
extendStatics(d, b);
|
|
11
|
-
function __() { this.constructor = d; }
|
|
12
|
-
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
|
|
13
|
-
};
|
|
14
|
-
})();
|
|
15
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
16
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
17
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
18
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
19
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
20
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
21
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
22
|
-
});
|
|
23
|
-
};
|
|
24
|
-
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
25
|
-
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
26
|
-
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
27
|
-
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
28
|
-
function step(op) {
|
|
29
|
-
if (f) throw new TypeError("Generator is already executing.");
|
|
30
|
-
while (_) try {
|
|
31
|
-
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
32
|
-
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
33
|
-
switch (op[0]) {
|
|
34
|
-
case 0: case 1: t = op; break;
|
|
35
|
-
case 4: _.label++; return { value: op[1], done: false };
|
|
36
|
-
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
37
|
-
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
38
|
-
default:
|
|
39
|
-
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
40
|
-
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
41
|
-
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
42
|
-
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
43
|
-
if (t[2]) _.ops.pop();
|
|
44
|
-
_.trys.pop(); continue;
|
|
45
|
-
}
|
|
46
|
-
op = body.call(thisArg, _);
|
|
47
|
-
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
48
|
-
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
49
|
-
}
|
|
50
|
-
};
|
|
51
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
52
3
|
exports.StdoutDestination = void 0;
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
4
|
+
const destination_1 = require("./base/destination");
|
|
5
|
+
class StdoutDestination extends destination_1.Destination {
|
|
6
|
+
asTable;
|
|
7
|
+
constructor(config) {
|
|
8
|
+
super(config);
|
|
9
|
+
this.asTable = false;
|
|
59
10
|
if (config) {
|
|
60
|
-
|
|
11
|
+
this.asTable = config.asTable || false;
|
|
61
12
|
}
|
|
62
|
-
return _this;
|
|
63
13
|
}
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
return [2 /*return*/];
|
|
69
|
-
});
|
|
70
|
-
});
|
|
71
|
-
};
|
|
72
|
-
return StdoutDestination;
|
|
73
|
-
}(destination_1.Destination));
|
|
14
|
+
async loadBatch(data) {
|
|
15
|
+
this.asTable ? console.table(data) : console.log(data);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
74
18
|
exports.StdoutDestination = StdoutDestination;
|