bellboy 8.4.1 → 8.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/lib/destinations/base/database-destination.js +10 -70
- package/lib/destinations/base/destination.js +10 -47
- package/lib/destinations/http-destination.js +20 -98
- package/lib/destinations/mssql-destination.js +41 -137
- package/lib/destinations/mysql-destination.js +30 -109
- package/lib/destinations/postgres-destination.js +49 -119
- package/lib/destinations/stdout-destination.js +11 -67
- package/lib/job.js +169 -424
- package/lib/processors/base/database-processor.js +8 -23
- package/lib/processors/base/directory-processor.js +16 -33
- package/lib/processors/base/processor.js +7 -47
- package/lib/processors/delimited-processor.js +43 -111
- package/lib/processors/dynamic-processor.js +12 -72
- package/lib/processors/excel-processor.js +43 -122
- package/lib/processors/firebird-processor.js +31 -96
- package/lib/processors/http-processor.js +82 -175
- package/lib/processors/json-processor.js +27 -99
- package/lib/processors/mqtt-processor.js +24 -91
- package/lib/processors/mssql-processor.js +79 -179
- package/lib/processors/mysql-processor.js +15 -81
- package/lib/processors/postgres-processor.js +17 -83
- package/lib/processors/tail-processor.js +33 -102
- package/lib/reporters/base/reporter.js +5 -7
- package/lib/types.d.ts +6 -7
- package/lib/utils.d.ts +0 -1
- package/lib/utils.js +91 -281
- package/package.json +30 -30
|
@@ -1,103 +1,38 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __extends = (this && this.__extends) || (function () {
|
|
3
|
-
var extendStatics = function (d, b) {
|
|
4
|
-
extendStatics = Object.setPrototypeOf ||
|
|
5
|
-
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
|
|
6
|
-
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
|
|
7
|
-
return extendStatics(d, b);
|
|
8
|
-
};
|
|
9
|
-
return function (d, b) {
|
|
10
|
-
extendStatics(d, b);
|
|
11
|
-
function __() { this.constructor = d; }
|
|
12
|
-
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
|
|
13
|
-
};
|
|
14
|
-
})();
|
|
15
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
16
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
17
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
18
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
19
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
20
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
21
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
22
|
-
});
|
|
23
|
-
};
|
|
24
|
-
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
25
|
-
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
26
|
-
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
27
|
-
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
28
|
-
function step(op) {
|
|
29
|
-
if (f) throw new TypeError("Generator is already executing.");
|
|
30
|
-
while (_) try {
|
|
31
|
-
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
32
|
-
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
33
|
-
switch (op[0]) {
|
|
34
|
-
case 0: case 1: t = op; break;
|
|
35
|
-
case 4: _.label++; return { value: op[1], done: false };
|
|
36
|
-
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
37
|
-
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
38
|
-
default:
|
|
39
|
-
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
40
|
-
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
41
|
-
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
42
|
-
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
43
|
-
if (t[2]) _.ops.pop();
|
|
44
|
-
_.trys.pop(); continue;
|
|
45
|
-
}
|
|
46
|
-
op = body.call(thisArg, _);
|
|
47
|
-
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
48
|
-
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
49
|
-
}
|
|
50
|
-
};
|
|
51
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
52
3
|
exports.FirebirdProcessor = void 0;
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
return _this;
|
|
4
|
+
const stream_1 = require("stream");
|
|
5
|
+
const database_processor_1 = require("./base/database-processor");
|
|
6
|
+
class FirebirdProcessor extends database_processor_1.DatabaseProcessor {
|
|
7
|
+
connection;
|
|
8
|
+
constructor(config) {
|
|
9
|
+
super(config);
|
|
10
|
+
this.connection = config.connection;
|
|
61
11
|
}
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
return [4 /*yield*/, new Promise(function (resolve, reject) {
|
|
71
|
-
Firebird.attach(_this.connection, function (err, db) {
|
|
72
|
-
if (err)
|
|
73
|
-
reject(err);
|
|
74
|
-
else
|
|
75
|
-
resolve(db);
|
|
76
|
-
});
|
|
77
|
-
})];
|
|
78
|
-
case 1:
|
|
79
|
-
db = _a.sent();
|
|
80
|
-
stream = new stream_1.Readable({
|
|
81
|
-
objectMode: true,
|
|
82
|
-
read: function () { }
|
|
83
|
-
});
|
|
84
|
-
db.sequentially(this.query, [], function (row) {
|
|
85
|
-
if (!stream.push(row)) { }
|
|
86
|
-
}, function (err) {
|
|
87
|
-
if (err) {
|
|
88
|
-
stream.emit('error', err);
|
|
89
|
-
}
|
|
90
|
-
stream.push(null);
|
|
91
|
-
db.detach();
|
|
92
|
-
});
|
|
93
|
-
return [4 /*yield*/, processStream(stream)];
|
|
94
|
-
case 2:
|
|
95
|
-
_a.sent();
|
|
96
|
-
return [2 /*return*/];
|
|
97
|
-
}
|
|
12
|
+
async process(processStream) {
|
|
13
|
+
const Firebird = this.connection.useClavizNodeFirebird ? require('claviz-node-firebird') : require('node-firebird');
|
|
14
|
+
const db = await new Promise((resolve, reject) => {
|
|
15
|
+
Firebird.attach(this.connection, (err, db) => {
|
|
16
|
+
if (err)
|
|
17
|
+
reject(err);
|
|
18
|
+
else
|
|
19
|
+
resolve(db);
|
|
98
20
|
});
|
|
99
21
|
});
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
22
|
+
const stream = new stream_1.Readable({
|
|
23
|
+
objectMode: true,
|
|
24
|
+
read() { }
|
|
25
|
+
});
|
|
26
|
+
db.sequentially(this.query, [], (row) => {
|
|
27
|
+
if (!stream.push(row)) { }
|
|
28
|
+
}, (err) => {
|
|
29
|
+
if (err) {
|
|
30
|
+
stream.emit('error', err);
|
|
31
|
+
}
|
|
32
|
+
stream.push(null);
|
|
33
|
+
db.detach();
|
|
34
|
+
});
|
|
35
|
+
await processStream(stream);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
103
38
|
exports.FirebirdProcessor = FirebirdProcessor;
|
|
@@ -1,197 +1,104 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __extends = (this && this.__extends) || (function () {
|
|
3
|
-
var extendStatics = function (d, b) {
|
|
4
|
-
extendStatics = Object.setPrototypeOf ||
|
|
5
|
-
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
|
|
6
|
-
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
|
|
7
|
-
return extendStatics(d, b);
|
|
8
|
-
};
|
|
9
|
-
return function (d, b) {
|
|
10
|
-
extendStatics(d, b);
|
|
11
|
-
function __() { this.constructor = d; }
|
|
12
|
-
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
|
|
13
|
-
};
|
|
14
|
-
})();
|
|
15
|
-
var __assign = (this && this.__assign) || function () {
|
|
16
|
-
__assign = Object.assign || function(t) {
|
|
17
|
-
for (var s, i = 1, n = arguments.length; i < n; i++) {
|
|
18
|
-
s = arguments[i];
|
|
19
|
-
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
|
|
20
|
-
t[p] = s[p];
|
|
21
|
-
}
|
|
22
|
-
return t;
|
|
23
|
-
};
|
|
24
|
-
return __assign.apply(this, arguments);
|
|
25
|
-
};
|
|
26
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
27
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
28
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
29
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
30
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
31
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
32
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
33
|
-
});
|
|
34
|
-
};
|
|
35
|
-
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
36
|
-
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
37
|
-
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
38
|
-
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
39
|
-
function step(op) {
|
|
40
|
-
if (f) throw new TypeError("Generator is already executing.");
|
|
41
|
-
while (_) try {
|
|
42
|
-
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
43
|
-
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
44
|
-
switch (op[0]) {
|
|
45
|
-
case 0: case 1: t = op; break;
|
|
46
|
-
case 4: _.label++; return { value: op[1], done: false };
|
|
47
|
-
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
48
|
-
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
49
|
-
default:
|
|
50
|
-
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
51
|
-
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
52
|
-
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
53
|
-
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
54
|
-
if (t[2]) _.ops.pop();
|
|
55
|
-
_.trys.pop(); continue;
|
|
56
|
-
}
|
|
57
|
-
op = body.call(thisArg, _);
|
|
58
|
-
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
59
|
-
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
60
|
-
}
|
|
61
|
-
};
|
|
62
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
63
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
64
|
-
};
|
|
65
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
66
3
|
exports.HttpProcessor = void 0;
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
5
|
+
const axios_1 = tslib_1.__importDefault(require("axios"));
|
|
6
|
+
const csv_parse_1 = require("csv-parse");
|
|
7
|
+
const Pick_1 = require("stream-json/filters/Pick");
|
|
8
|
+
const Parser_1 = require("stream-json/Parser");
|
|
9
|
+
const StreamValues_1 = require("stream-json/streamers/StreamValues");
|
|
10
|
+
const iconv_lite_1 = tslib_1.__importDefault(require("iconv-lite"));
|
|
11
|
+
const utils_1 = require("../utils");
|
|
12
|
+
const processor_1 = require("./base/processor");
|
|
13
|
+
const saxStream = require('sax-stream');
|
|
14
|
+
class HttpProcessor extends processor_1.Processor {
|
|
15
|
+
connection;
|
|
16
|
+
nextRequest;
|
|
17
|
+
jsonPath;
|
|
18
|
+
rowSeparator;
|
|
19
|
+
dataFormat;
|
|
20
|
+
hasHeader = false;
|
|
21
|
+
delimiter;
|
|
22
|
+
qualifier;
|
|
23
|
+
authorizationRequest;
|
|
24
|
+
saxOptions;
|
|
25
|
+
encoding;
|
|
26
|
+
constructor(config) {
|
|
27
|
+
super(config);
|
|
81
28
|
if (!config.connection) {
|
|
82
|
-
throw new Error(
|
|
29
|
+
throw new Error(`No connection specified.`);
|
|
83
30
|
}
|
|
84
|
-
|
|
85
|
-
|
|
31
|
+
this.connection = config.connection;
|
|
32
|
+
this.dataFormat = config.dataFormat;
|
|
86
33
|
if (config.dataFormat === 'delimited') {
|
|
87
34
|
if (!config.rowSeparator) {
|
|
88
35
|
throw new Error('No rowSeparator specified.');
|
|
89
36
|
}
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
37
|
+
this.rowSeparator = config.rowSeparator;
|
|
38
|
+
this.hasHeader = !!config.hasHeader;
|
|
39
|
+
this.delimiter = config.delimiter;
|
|
40
|
+
this.qualifier = config.qualifier;
|
|
41
|
+
this.encoding = config.encoding;
|
|
95
42
|
}
|
|
96
43
|
else if (config.dataFormat === 'json' && config.jsonPath) {
|
|
97
44
|
if (config.jsonPath instanceof RegExp) {
|
|
98
|
-
|
|
45
|
+
this.jsonPath = config.jsonPath;
|
|
99
46
|
}
|
|
100
47
|
else {
|
|
101
|
-
|
|
48
|
+
this.jsonPath = new RegExp(config.jsonPath);
|
|
102
49
|
}
|
|
103
50
|
}
|
|
104
51
|
else if (config.dataFormat === 'xml' && config.saxOptions) {
|
|
105
|
-
|
|
52
|
+
this.saxOptions = config.saxOptions;
|
|
106
53
|
}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
return _this;
|
|
54
|
+
this.nextRequest = config.nextRequest;
|
|
55
|
+
this.authorizationRequest = config.authorizationRequest;
|
|
110
56
|
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
parser_1 = csv_parse_1.parse({
|
|
123
|
-
quote: this.qualifier,
|
|
124
|
-
delimiter: this.delimiter,
|
|
125
|
-
record_delimiter: this.rowSeparator,
|
|
126
|
-
raw: true,
|
|
127
|
-
relax_quotes: true,
|
|
128
|
-
relax_column_count: true,
|
|
129
|
-
});
|
|
130
|
-
delimitedStream = decodedStream.pipe(parser_1);
|
|
131
|
-
generator = utils_1.getDelimitedGenerator({
|
|
132
|
-
readStream: delimitedStream,
|
|
133
|
-
hasHeader: this.hasHeader,
|
|
134
|
-
});
|
|
135
|
-
return [4 /*yield*/, processStream(generator())];
|
|
136
|
-
case 2:
|
|
137
|
-
_a.sent();
|
|
138
|
-
return [3 /*break*/, 9];
|
|
139
|
-
case 3:
|
|
140
|
-
if (!(this.dataFormat === 'json')) return [3 /*break*/, 6];
|
|
141
|
-
return [4 /*yield*/, axios_1.default(__assign(__assign({}, options), { responseType: 'stream' }))];
|
|
142
|
-
case 4:
|
|
143
|
-
requestStream = _a.sent();
|
|
144
|
-
jsonStream = requestStream.data
|
|
145
|
-
.pipe(Parser_1.parser())
|
|
146
|
-
.pipe(Pick_1.pick({ filter: this.jsonPath || '' }))
|
|
147
|
-
.pipe(StreamValues_1.streamValues())
|
|
148
|
-
.pipe(utils_1.getValueFromJSONChunk());
|
|
149
|
-
return [4 /*yield*/, processStream(jsonStream)];
|
|
150
|
-
case 5:
|
|
151
|
-
_a.sent();
|
|
152
|
-
return [3 /*break*/, 9];
|
|
153
|
-
case 6:
|
|
154
|
-
if (!(this.dataFormat === 'xml')) return [3 /*break*/, 9];
|
|
155
|
-
return [4 /*yield*/, axios_1.default(__assign(__assign({}, options), { responseType: 'stream' }))];
|
|
156
|
-
case 7:
|
|
157
|
-
requestStream = _a.sent();
|
|
158
|
-
xmlStream = requestStream.data
|
|
159
|
-
.pipe(saxStream(this.saxOptions))
|
|
160
|
-
.pipe(utils_1.removeCircularReferencesFromChunk());
|
|
161
|
-
return [4 /*yield*/, processStream(xmlStream)];
|
|
162
|
-
case 8:
|
|
163
|
-
_a.sent();
|
|
164
|
-
_a.label = 9;
|
|
165
|
-
case 9:
|
|
166
|
-
if (!this.nextRequest) return [3 /*break*/, 12];
|
|
167
|
-
return [4 /*yield*/, this.nextRequest()];
|
|
168
|
-
case 10:
|
|
169
|
-
nextOptions = _a.sent();
|
|
170
|
-
if (!nextOptions) return [3 /*break*/, 12];
|
|
171
|
-
return [4 /*yield*/, this.processHttpData(processStream, nextOptions)];
|
|
172
|
-
case 11:
|
|
173
|
-
_a.sent();
|
|
174
|
-
_a.label = 12;
|
|
175
|
-
case 12: return [2 /*return*/];
|
|
176
|
-
}
|
|
57
|
+
async processHttpData(processStream, options) {
|
|
58
|
+
if (this.dataFormat === 'delimited') {
|
|
59
|
+
const requestStream = await (0, axios_1.default)({ ...options, responseType: 'stream' });
|
|
60
|
+
const decodedStream = this.encoding ? requestStream.data.pipe(iconv_lite_1.default.decodeStream(this.encoding)) : requestStream.data;
|
|
61
|
+
const parser = (0, csv_parse_1.parse)({
|
|
62
|
+
quote: this.qualifier,
|
|
63
|
+
delimiter: this.delimiter,
|
|
64
|
+
record_delimiter: this.rowSeparator,
|
|
65
|
+
raw: true,
|
|
66
|
+
relax_quotes: true,
|
|
67
|
+
relax_column_count: true,
|
|
177
68
|
});
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
return __generator(this, function (_a) {
|
|
183
|
-
switch (_a.label) {
|
|
184
|
-
case 0: return [4 /*yield*/, utils_1.applyHttpAuthorization(this.connection, this.authorizationRequest)];
|
|
185
|
-
case 1:
|
|
186
|
-
_a.sent();
|
|
187
|
-
return [4 /*yield*/, this.processHttpData(processStream, this.connection)];
|
|
188
|
-
case 2:
|
|
189
|
-
_a.sent();
|
|
190
|
-
return [2 /*return*/];
|
|
191
|
-
}
|
|
69
|
+
const delimitedStream = decodedStream.pipe(parser);
|
|
70
|
+
const generator = (0, utils_1.getDelimitedGenerator)({
|
|
71
|
+
readStream: delimitedStream,
|
|
72
|
+
hasHeader: this.hasHeader,
|
|
192
73
|
});
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
74
|
+
await processStream(generator());
|
|
75
|
+
}
|
|
76
|
+
else if (this.dataFormat === 'json') {
|
|
77
|
+
const requestStream = await (0, axios_1.default)({ ...options, responseType: 'stream' });
|
|
78
|
+
const jsonStream = requestStream.data
|
|
79
|
+
.pipe((0, Parser_1.parser)())
|
|
80
|
+
.pipe((0, Pick_1.pick)({ filter: this.jsonPath || '' }))
|
|
81
|
+
.pipe((0, StreamValues_1.streamValues)())
|
|
82
|
+
.pipe((0, utils_1.getValueFromJSONChunk)());
|
|
83
|
+
await processStream(jsonStream);
|
|
84
|
+
}
|
|
85
|
+
else if (this.dataFormat === 'xml') {
|
|
86
|
+
const requestStream = await (0, axios_1.default)({ ...options, responseType: 'stream' });
|
|
87
|
+
const xmlStream = requestStream.data
|
|
88
|
+
.pipe(saxStream(this.saxOptions))
|
|
89
|
+
.pipe((0, utils_1.removeCircularReferencesFromChunk)());
|
|
90
|
+
await processStream(xmlStream);
|
|
91
|
+
}
|
|
92
|
+
if (this.nextRequest) {
|
|
93
|
+
const nextOptions = await this.nextRequest();
|
|
94
|
+
if (nextOptions) {
|
|
95
|
+
await this.processHttpData(processStream, nextOptions);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
async process(processStream) {
|
|
100
|
+
await (0, utils_1.applyHttpAuthorization)(this.connection, this.authorizationRequest);
|
|
101
|
+
await this.processHttpData(processStream, this.connection);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
197
104
|
exports.HttpProcessor = HttpProcessor;
|
|
@@ -1,110 +1,38 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __extends = (this && this.__extends) || (function () {
|
|
3
|
-
var extendStatics = function (d, b) {
|
|
4
|
-
extendStatics = Object.setPrototypeOf ||
|
|
5
|
-
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
|
|
6
|
-
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
|
|
7
|
-
return extendStatics(d, b);
|
|
8
|
-
};
|
|
9
|
-
return function (d, b) {
|
|
10
|
-
extendStatics(d, b);
|
|
11
|
-
function __() { this.constructor = d; }
|
|
12
|
-
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
|
|
13
|
-
};
|
|
14
|
-
})();
|
|
15
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
16
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
17
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
18
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
19
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
20
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
21
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
22
|
-
});
|
|
23
|
-
};
|
|
24
|
-
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
25
|
-
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
26
|
-
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
27
|
-
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
28
|
-
function step(op) {
|
|
29
|
-
if (f) throw new TypeError("Generator is already executing.");
|
|
30
|
-
while (_) try {
|
|
31
|
-
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
32
|
-
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
33
|
-
switch (op[0]) {
|
|
34
|
-
case 0: case 1: t = op; break;
|
|
35
|
-
case 4: _.label++; return { value: op[1], done: false };
|
|
36
|
-
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
37
|
-
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
38
|
-
default:
|
|
39
|
-
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
40
|
-
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
41
|
-
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
42
|
-
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
43
|
-
if (t[2]) _.ops.pop();
|
|
44
|
-
_.trys.pop(); continue;
|
|
45
|
-
}
|
|
46
|
-
op = body.call(thisArg, _);
|
|
47
|
-
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
48
|
-
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
49
|
-
}
|
|
50
|
-
};
|
|
51
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
52
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
53
|
-
};
|
|
54
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
55
3
|
exports.JsonProcessor = void 0;
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
5
|
+
const fs_1 = tslib_1.__importDefault(require("fs"));
|
|
6
|
+
const path_1 = tslib_1.__importDefault(require("path"));
|
|
7
|
+
const Pick_1 = require("stream-json/filters/Pick");
|
|
8
|
+
const Parser_1 = require("stream-json/Parser");
|
|
9
|
+
const StreamValues_1 = require("stream-json/streamers/StreamValues");
|
|
10
|
+
const utils_1 = require("../utils");
|
|
11
|
+
const directory_processor_1 = require("./base/directory-processor");
|
|
12
|
+
class JsonProcessor extends directory_processor_1.DirectoryProcessor {
|
|
13
|
+
jsonPath;
|
|
14
|
+
constructor(config) {
|
|
15
|
+
super(config);
|
|
67
16
|
if (config.jsonPath) {
|
|
68
17
|
if (config.jsonPath instanceof RegExp) {
|
|
69
|
-
|
|
18
|
+
this.jsonPath = config.jsonPath;
|
|
70
19
|
}
|
|
71
20
|
else {
|
|
72
|
-
|
|
21
|
+
this.jsonPath = new RegExp(config.jsonPath);
|
|
73
22
|
}
|
|
74
23
|
}
|
|
75
|
-
return _this;
|
|
76
24
|
}
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
.pipe(Parser_1.parser())
|
|
91
|
-
.pipe(Pick_1.pick({ filter: this.jsonPath || '' }))
|
|
92
|
-
.pipe(StreamValues_1.streamValues())
|
|
93
|
-
.pipe(utils_1.getValueFromJSONChunk());
|
|
94
|
-
return [4 /*yield*/, processStream(stream, file, filePath)];
|
|
95
|
-
case 2:
|
|
96
|
-
_b.sent();
|
|
97
|
-
_b.label = 3;
|
|
98
|
-
case 3:
|
|
99
|
-
_i++;
|
|
100
|
-
return [3 /*break*/, 1];
|
|
101
|
-
case 4:
|
|
102
|
-
;
|
|
103
|
-
return [2 /*return*/];
|
|
104
|
-
}
|
|
105
|
-
});
|
|
106
|
-
});
|
|
107
|
-
};
|
|
108
|
-
return JsonProcessor;
|
|
109
|
-
}(directory_processor_1.DirectoryProcessor));
|
|
25
|
+
async process(processStream) {
|
|
26
|
+
for (const file of this.files) {
|
|
27
|
+
const filePath = path_1.default.join(this.path, file);
|
|
28
|
+
const stream = fs_1.default.createReadStream(filePath)
|
|
29
|
+
.pipe((0, Parser_1.parser)())
|
|
30
|
+
.pipe((0, Pick_1.pick)({ filter: this.jsonPath || '' }))
|
|
31
|
+
.pipe((0, StreamValues_1.streamValues)())
|
|
32
|
+
.pipe((0, utils_1.getValueFromJSONChunk)());
|
|
33
|
+
await processStream(stream, file, filePath);
|
|
34
|
+
}
|
|
35
|
+
;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
110
38
|
exports.JsonProcessor = JsonProcessor;
|