@golemio/core 1.18.1-dev.1592261843 → 1.18.1-dev.1622172793
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/helpers/DateTime.js +4 -5
- package/dist/helpers/DateTime.js.map +1 -1
- package/dist/helpers/HealthCheck.js +5 -14
- package/dist/helpers/HealthCheck.js.map +1 -1
- package/dist/helpers/configuration/SimpleConfig.js +4 -1
- package/dist/helpers/configuration/SimpleConfig.js.map +1 -1
- package/dist/helpers/data-access/amqp/AlternateExchangeCreator.js +4 -13
- package/dist/helpers/data-access/amqp/AlternateExchangeCreator.js.map +1 -1
- package/dist/helpers/data-access/amqp/AmqpConnector.js +28 -40
- package/dist/helpers/data-access/amqp/AmqpConnector.js.map +1 -1
- package/dist/helpers/data-access/feature-flags/helpers/AbstractFeatureFlagServiceWrapper.js +2 -13
- package/dist/helpers/data-access/feature-flags/helpers/AbstractFeatureFlagServiceWrapper.js.map +1 -1
- package/dist/helpers/data-access/feature-flags/providers/NoopFeatureFlagService.js +9 -24
- package/dist/helpers/data-access/feature-flags/providers/NoopFeatureFlagService.js.map +1 -1
- package/dist/helpers/data-access/postgres/DatabaseConnector.js +23 -25
- package/dist/helpers/data-access/postgres/DatabaseConnector.js.map +1 -1
- package/dist/helpers/data-access/postgres/repositories/AbstractCachedRepository.js +3 -12
- package/dist/helpers/data-access/postgres/repositories/AbstractCachedRepository.js.map +1 -1
- package/dist/helpers/data-access/pubsub/subscribers/RedisSubscriber.js +15 -30
- package/dist/helpers/data-access/pubsub/subscribers/RedisSubscriber.js.map +1 -1
- package/dist/helpers/data-access/redis/IoRedisConnector.js +8 -17
- package/dist/helpers/data-access/redis/IoRedisConnector.js.map +1 -1
- package/dist/helpers/data-access/storage/AzureBlobStorageService.js +64 -81
- package/dist/helpers/data-access/storage/AzureBlobStorageService.js.map +1 -1
- package/dist/helpers/data-access/storage/NoopStorageService.js +9 -26
- package/dist/helpers/data-access/storage/NoopStorageService.js.map +1 -1
- package/dist/helpers/data-access/table-storage/TableStorageServiceFactory.js +1 -2
- package/dist/helpers/data-access/table-storage/TableStorageServiceFactory.js.map +1 -1
- package/dist/helpers/data-access/table-storage/providers/AzureTableStorageService.js +39 -68
- package/dist/helpers/data-access/table-storage/providers/AzureTableStorageService.js.map +1 -1
- package/dist/helpers/data-access/table-storage/providers/NoopTableStorageService.js +4 -17
- package/dist/helpers/data-access/table-storage/providers/NoopTableStorageService.js.map +1 -1
- package/dist/helpers/logger/LoggerProvider.js +33 -14
- package/dist/helpers/logger/LoggerProvider.js.map +1 -1
- package/dist/helpers/logger/RequestLoggerProvider.js +1 -2
- package/dist/helpers/logger/RequestLoggerProvider.js.map +1 -1
- package/dist/helpers/routing/AbstractGeoJsonRouter.js +16 -34
- package/dist/helpers/routing/AbstractGeoJsonRouter.js.map +1 -1
- package/dist/input-gateway/config/InputGatewayConfiguration.js +1 -2
- package/dist/input-gateway/config/InputGatewayConfiguration.js.map +1 -1
- package/dist/input-gateway/connectors/AMQPConnector.js +52 -68
- package/dist/input-gateway/connectors/AMQPConnector.js.map +1 -1
- package/dist/input-gateway/controllers/BaseController.js +4 -13
- package/dist/input-gateway/controllers/BaseController.js.map +1 -1
- package/dist/integration-engine/connectors/AMQPConnector.js +28 -40
- package/dist/integration-engine/connectors/AMQPConnector.js.map +1 -1
- package/dist/integration-engine/data-access/RawDataStorage.js +7 -16
- package/dist/integration-engine/data-access/RawDataStorage.js.map +1 -1
- package/dist/integration-engine/data-access/pubsub/RedisPubSubChannel.js +3 -14
- package/dist/integration-engine/data-access/pubsub/RedisPubSubChannel.js.map +1 -1
- package/dist/integration-engine/data-access/pubsub/subscribers/RedisSubscriber.js +12 -25
- package/dist/integration-engine/data-access/pubsub/subscribers/RedisSubscriber.js.map +1 -1
- package/dist/integration-engine/datasources/DataSource.js +41 -60
- package/dist/integration-engine/datasources/DataSource.js.map +1 -1
- package/dist/integration-engine/datasources/DataSourceStream.js +12 -21
- package/dist/integration-engine/datasources/DataSourceStream.js.map +1 -1
- package/dist/integration-engine/datasources/DataSourceStreamManager.js +4 -13
- package/dist/integration-engine/datasources/DataSourceStreamManager.js.map +1 -1
- package/dist/integration-engine/datasources/DataSourceStreamed.js +37 -51
- package/dist/integration-engine/datasources/DataSourceStreamed.js.map +1 -1
- package/dist/integration-engine/datasources/RetryDataSource.js +25 -40
- package/dist/integration-engine/datasources/RetryDataSource.js.map +1 -1
- package/dist/integration-engine/datasources/datatype-strategy/CSVDataTypeStrategy.js +22 -33
- package/dist/integration-engine/datasources/datatype-strategy/CSVDataTypeStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/datatype-strategy/JSONDataTypeStrategy.js +14 -25
- package/dist/integration-engine/datasources/datatype-strategy/JSONDataTypeStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/datatype-strategy/XMLDataTypeStrategy.js +11 -22
- package/dist/integration-engine/datasources/datatype-strategy/XMLDataTypeStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/datatype-strategy/ZipFileDataTypeStrategy.js +33 -45
- package/dist/integration-engine/datasources/datatype-strategy/ZipFileDataTypeStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/FTPProtocolStrategy.js +17 -26
- package/dist/integration-engine/datasources/protocol-strategy/FTPProtocolStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/GoogleCloudStorageProtocolStrategy.js +10 -19
- package/dist/integration-engine/datasources/protocol-strategy/GoogleCloudStorageProtocolStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/HTTPFetchProtocolStrategy.js +42 -51
- package/dist/integration-engine/datasources/protocol-strategy/HTTPFetchProtocolStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategy.js +49 -62
- package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.js +67 -81
- package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/PaginatedHTTPProtocolStrategy.js +29 -38
- package/dist/integration-engine/datasources/protocol-strategy/PaginatedHTTPProtocolStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/PostgresProtocolStrategy.js +37 -42
- package/dist/integration-engine/datasources/protocol-strategy/PostgresProtocolStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/PostgresProtocolStrategyStreamed.js +29 -32
- package/dist/integration-engine/datasources/protocol-strategy/PostgresProtocolStrategyStreamed.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/ProtocolStrategy.js +73 -88
- package/dist/integration-engine/datasources/protocol-strategy/ProtocolStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/SFTPProtocolStrategy.js +17 -26
- package/dist/integration-engine/datasources/protocol-strategy/SFTPProtocolStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/helpers/FTPFileProcessor.js +37 -49
- package/dist/integration-engine/datasources/protocol-strategy/helpers/FTPFileProcessor.js.map +1 -1
- package/dist/integration-engine/helpers/FileCompressor.js +59 -76
- package/dist/integration-engine/helpers/FileCompressor.js.map +1 -1
- package/dist/integration-engine/helpers/GeocodeApi.js +20 -30
- package/dist/integration-engine/helpers/GeocodeApi.js.map +1 -1
- package/dist/integration-engine/helpers/IntegrationErrorHandler.js +3 -3
- package/dist/integration-engine/helpers/IntegrationErrorHandler.js.map +1 -1
- package/dist/integration-engine/helpers/PaginationHelper.js +1 -2
- package/dist/integration-engine/helpers/PaginationHelper.js.map +1 -1
- package/dist/integration-engine/models/PostgresModel.js +70 -75
- package/dist/integration-engine/models/PostgresModel.js.map +1 -1
- package/dist/integration-engine/models/RedisModel.js +28 -39
- package/dist/integration-engine/models/RedisModel.js.map +1 -1
- package/dist/integration-engine/queueprocessors/QueueProcessor.js +23 -28
- package/dist/integration-engine/queueprocessors/QueueProcessor.js.map +1 -1
- package/dist/integration-engine/transformations/BaseTransformation.js +8 -17
- package/dist/integration-engine/transformations/BaseTransformation.js.map +1 -1
- package/dist/integration-engine/workers/AbstractTask.js +7 -18
- package/dist/integration-engine/workers/AbstractTask.js.map +1 -1
- package/dist/integration-engine/workers/AbstractTaskJsonSchema.js +25 -38
- package/dist/integration-engine/workers/AbstractTaskJsonSchema.js.map +1 -1
- package/dist/integration-engine/workers/AbstractTaskWithoutValidation.js +2 -13
- package/dist/integration-engine/workers/AbstractTaskWithoutValidation.js.map +1 -1
- package/dist/integration-engine/workers/BaseWorker.js +4 -13
- package/dist/integration-engine/workers/BaseWorker.js.map +1 -1
- package/dist/integration-engine/workers/helpers/MessageDataParser.js +1 -2
- package/dist/integration-engine/workers/helpers/MessageDataParser.js.map +1 -1
- package/dist/integration-engine/workers/helpers/MessageDataValidator.js +18 -29
- package/dist/integration-engine/workers/helpers/MessageDataValidator.js.map +1 -1
- package/dist/integration-engine/workers/helpers/WebhookDecorators.js +1 -10
- package/dist/integration-engine/workers/helpers/WebhookDecorators.js.map +1 -1
- package/dist/integration-engine/workers/webhooks/CloudflareCachePurgeWebhook.js +13 -24
- package/dist/integration-engine/workers/webhooks/CloudflareCachePurgeWebhook.js.map +1 -1
- package/dist/monitoring/metrics/metrics-builder.js +21 -17
- package/dist/monitoring/metrics/metrics-builder.js.map +1 -1
- package/dist/monitoring/metrics/metrics-service.js +4 -14
- package/dist/monitoring/metrics/metrics-service.js.map +1 -1
- package/dist/monitoring/opentelemetry/instrumentation/configs/amqplib.js +1 -2
- package/dist/monitoring/opentelemetry/instrumentation/configs/amqplib.js.map +1 -1
- package/dist/monitoring/opentelemetry/instrumentation/configs/aws-sdk.js +1 -1
- package/dist/monitoring/opentelemetry/instrumentation/configs/aws-sdk.js.map +1 -1
- package/dist/monitoring/opentelemetry/trace-provider.js +8 -2
- package/dist/monitoring/opentelemetry/trace-provider.js.map +1 -1
- package/dist/monitoring/sentry/sentry-provider.js +1 -1
- package/dist/monitoring/sentry/sentry-provider.js.map +1 -1
- package/dist/monitoring/sentry/sentry-transactions.js +9 -19
- package/dist/monitoring/sentry/sentry-transactions.js.map +1 -1
- package/dist/output-gateway/Geo.js +3 -24
- package/dist/output-gateway/Geo.js.map +1 -1
- package/dist/output-gateway/config/OutputGatewayConfiguration.js +1 -2
- package/dist/output-gateway/config/OutputGatewayConfiguration.js.map +1 -1
- package/dist/output-gateway/database/index.js +3 -12
- package/dist/output-gateway/database/index.js.map +1 -1
- package/dist/output-gateway/ioc/Di.js +1 -10
- package/dist/output-gateway/ioc/Di.js.map +1 -1
- package/dist/output-gateway/models/SequelizeModel.js +4 -2
- package/dist/output-gateway/models/SequelizeModel.js.map +1 -1
- package/dist/output-gateway/routes/GeoJsonRouter.js +14 -32
- package/dist/output-gateway/routes/GeoJsonRouter.js.map +1 -1
- package/package.json +1 -1
- package/tsconfig.base.json +2 -2
|
@@ -1,13 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
3
|
exports.DataSourceStreamed = void 0;
|
|
13
4
|
const logger_1 = require("../../helpers/logger");
|
|
@@ -19,13 +10,13 @@ class DataSourceStreamed extends datasources_1.DataSource {
|
|
|
19
10
|
constructor() {
|
|
20
11
|
super(...arguments);
|
|
21
12
|
this.dataBuffer = [];
|
|
22
|
-
this.proceed = () =>
|
|
13
|
+
this.proceed = async () => {
|
|
23
14
|
return this.dataStream.proceed();
|
|
24
|
-
}
|
|
15
|
+
};
|
|
25
16
|
/**
|
|
26
17
|
* @param {boolean} useDataBuffer data is buffered and sent to output stream in `config.DATA_BATCH_SIZE` batches
|
|
27
18
|
*/
|
|
28
|
-
this.getOutputStream = (useDataBuffer = false) =>
|
|
19
|
+
this.getOutputStream = async (useDataBuffer = false) => {
|
|
29
20
|
let inputStreamEndedAttempts = 0;
|
|
30
21
|
this.dataStream = new datasources_1.DataSourceStream({
|
|
31
22
|
objectMode: true,
|
|
@@ -33,7 +24,7 @@ class DataSourceStreamed extends datasources_1.DataSource {
|
|
|
33
24
|
return;
|
|
34
25
|
},
|
|
35
26
|
});
|
|
36
|
-
const inputStream =
|
|
27
|
+
const inputStream = await this.protocolStrategy.getData();
|
|
37
28
|
inputStream.on("error", (error) => {
|
|
38
29
|
if (error instanceof errors_1.AbstractGolemioError) {
|
|
39
30
|
this.dataStream.emit("error", error);
|
|
@@ -42,20 +33,20 @@ class DataSourceStreamed extends datasources_1.DataSource {
|
|
|
42
33
|
this.dataStream.emit("error", new errors_1.GeneralError("DataSourceStreamed general error", this.constructor.name, error));
|
|
43
34
|
}
|
|
44
35
|
});
|
|
45
|
-
inputStream.onDataListeners.push((data) =>
|
|
36
|
+
inputStream.onDataListeners.push(async (data) => {
|
|
46
37
|
inputStream.pause();
|
|
47
38
|
if (useDataBuffer) {
|
|
48
39
|
this.dataBuffer.push(data);
|
|
49
|
-
|
|
40
|
+
await this.processData();
|
|
50
41
|
}
|
|
51
42
|
else {
|
|
52
|
-
|
|
43
|
+
await this.processData(false, data);
|
|
53
44
|
}
|
|
54
45
|
inputStream.resume();
|
|
55
|
-
})
|
|
56
|
-
inputStream.on("end", () =>
|
|
46
|
+
});
|
|
47
|
+
inputStream.on("end", async () => {
|
|
57
48
|
if (useDataBuffer) {
|
|
58
|
-
|
|
49
|
+
await this.processData(true);
|
|
59
50
|
}
|
|
60
51
|
if (!inputStream.isPaused()) {
|
|
61
52
|
this.dataStream.push(null);
|
|
@@ -75,17 +66,16 @@ class DataSourceStreamed extends datasources_1.DataSource {
|
|
|
75
66
|
}
|
|
76
67
|
}, config_1.config.stream.wait_for_end_interval);
|
|
77
68
|
}
|
|
78
|
-
})
|
|
69
|
+
});
|
|
79
70
|
inputStream.proceed().catch((error) => this.dataStream.emit("error", error));
|
|
80
71
|
return this.dataStream;
|
|
81
|
-
}
|
|
82
|
-
this.processData = (force = false, data = null) =>
|
|
83
|
-
var _a;
|
|
72
|
+
};
|
|
73
|
+
this.processData = async (force = false, data = null) => {
|
|
84
74
|
if (this.dataBuffer.length >= config_1.config.DATA_BATCH_SIZE || force || data) {
|
|
85
75
|
try {
|
|
86
76
|
let content;
|
|
87
|
-
if (
|
|
88
|
-
content =
|
|
77
|
+
if (this.dataTypeStrategy?.parseData) {
|
|
78
|
+
content = await this.dataTypeStrategy.parseData(data || this.dataBuffer);
|
|
89
79
|
}
|
|
90
80
|
else {
|
|
91
81
|
content = data || this.dataBuffer;
|
|
@@ -106,37 +96,33 @@ class DataSourceStreamed extends datasources_1.DataSource {
|
|
|
106
96
|
this.dataStream.emit("error", new errors_1.GeneralError("Retrieving of the source data failed.", this.name, err));
|
|
107
97
|
}
|
|
108
98
|
}
|
|
109
|
-
}
|
|
99
|
+
};
|
|
110
100
|
}
|
|
111
|
-
getAll(useDataBuffer = false) {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
return data;
|
|
115
|
-
});
|
|
101
|
+
async getAll(useDataBuffer = false) {
|
|
102
|
+
const { data } = await this.getAllWithMetadata(useDataBuffer);
|
|
103
|
+
return data;
|
|
116
104
|
}
|
|
117
|
-
getAllWithMetadata(useDataBuffer = false) {
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
this.dataStream.
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
this.dataStream.resume();
|
|
129
|
-
}
|
|
130
|
-
catch (err) {
|
|
131
|
-
this.dataStream.destroy(new errors_1.GeneralError("Error while validating source data.", this.name, err));
|
|
132
|
-
}
|
|
105
|
+
async getAllWithMetadata(useDataBuffer = false) {
|
|
106
|
+
this.dataStream = await this.getOutputStream(useDataBuffer);
|
|
107
|
+
this.dataStream.on("end", () => {
|
|
108
|
+
this.dataStream.destroy();
|
|
109
|
+
});
|
|
110
|
+
this.dataStream.onDataListeners.push(async (data) => {
|
|
111
|
+
if (this.validator) {
|
|
112
|
+
try {
|
|
113
|
+
this.dataStream.pause();
|
|
114
|
+
await this.validator.Validate(data);
|
|
115
|
+
this.dataStream.resume();
|
|
133
116
|
}
|
|
134
|
-
|
|
135
|
-
|
|
117
|
+
catch (err) {
|
|
118
|
+
this.dataStream.destroy(new errors_1.GeneralError("Error while validating source data.", this.name, err));
|
|
136
119
|
}
|
|
137
|
-
}
|
|
138
|
-
|
|
120
|
+
}
|
|
121
|
+
else {
|
|
122
|
+
helpers_1.log.warn("DataSource validator is not set.");
|
|
123
|
+
}
|
|
139
124
|
});
|
|
125
|
+
return { data: this.dataStream, metadata: undefined };
|
|
140
126
|
}
|
|
141
127
|
}
|
|
142
128
|
exports.DataSourceStreamed = DataSourceStreamed;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DataSourceStreamed.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/DataSourceStreamed.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"DataSourceStreamed.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/DataSourceStreamed.ts"],"names":[],"mappings":";;;AAAA,iDAAkD;AAClD,sCAAoC;AACpC,oCAA4E;AAC5E,wCAAkC;AAClC,4CAAqE;AAErE,MAAa,kBAAmB,SAAQ,wBAAU;IAAlD;;QACY,eAAU,GAAU,EAAE,CAAC;QAExB,YAAO,GAAG,KAAK,IAAmB,EAAE;YACvC,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QACrC,CAAC,CAAC;QA+BF;;WAEG;QACO,oBAAe,GAAG,KAAK,EAAE,gBAAyB,KAAK,EAA6B,EAAE;YAC5F,IAAI,wBAAwB,GAAG,CAAC,CAAC;YACjC,IAAI,CAAC,UAAU,GAAG,IAAI,8BAAgB,CAAC;gBACnC,UAAU,EAAE,IAAI;gBAChB,IAAI,EAAE,GAAG,EAAE;oBACP,OAAO;gBACX,CAAC;aACJ,CAAC,CAAC;YAEH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC;YAE1D,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBACnC,IAAI,KAAK,YAAY,6BAAoB,EAAE;oBACvC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;iBACxC;qBAAM;oBACH,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,kCAAkC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;iBACrH;YACL,CAAC,CAAC,CAAC;YAEH,WAAW,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,IAAS,EAAiB,EAAE;gBAChE,WAAW,CAAC,KAAK,EAAE,CAAC;gBAEpB,IAAI,aAAa,EAAE;oBACf,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC3B,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;iBAC5B;qBAAM;oBACH,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;iBACvC;gBAED,WAAW,CAAC,MAAM,EAAE,CAAC;YACzB,CAAC,CAAC,CAAC;YAEH,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,IAAmB,EAAE;gBAC5C,IAAI,aAAa,EAAE;oBACf,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;iBAChC;gBAED,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE;oBACzB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBAC9B;qBAAM;oBACH,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE;wBAC7B,wBAAwB,EAAE,CAAC;wBAC3B,kCAAkC;wBAClC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE;4BACzB,aAAa,CAAC,OAAO,CAAC,CAAC;4BACvB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;yBAC9B;6BAAM,IAAI,wBAAwB,GAAG,eAAM,CAAC,MAAM,CAAC,qBAAqB,EAAE;4BACvE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,kCAAkC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;4BAC/F,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;4BAC3B,aAAa,CAAC,OAAO,CAAC,CAAC;yBAC1B;oBACL,CAAC,EAAE,eAAM,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;iBAC3C;YACL,CAAC,CAAC,CAAC;YAEH,WAAW,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAEpF,OAAO,IAAI,CAAC,UAAU,CAAC;QAC3B,CAAC,CAAC;QAEM,gBAAW,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE,IAAI,GAAG,IAAI,EAAiB,EAAE;YACtE,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,eAAM,CAAC,eAAe,IAAI,KAAK,IAAI,IAAI,EAAE;gBACnE,IAAI;oBACA,IAAI,OAAY,CAAC;oBAEjB,IAAI,IAAI,CAAC,gBAAgB,EAAE,SAAS,EAAE;wBAClC,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC;qBAC5E;yBAAM;wBACH,OAAO,GAAG,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC;qBACrC;oBAED,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;wBACvB,aAAG,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,IAAI,oCAAoC,CAAC,CAAC;wBAC/D,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,wBAAe,CAAC,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC,CAAC;qBACrG;yBAAM;wBACH,MAAM,eAAe,GAAG,OAAO,YAAY,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;wBACtE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,wBAAe,CAAC,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;wBAE/F,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;wBAC9B,mBAAmB;wBACnB,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;qBAC9B;iBACJ;gBAAC,OAAO,GAAG,EAAE;oBACV,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,uCAAuC,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;iBAC5G;aACJ;QACL,CAAC,CAAC;IACN,CAAC;IAvHmB,KAAK,CAAC,MAAM,CAAC,aAAa,GAAG,KAAK;QAC9C,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAC9D,OAAO,IAAI,CAAC;IAChB,CAAC;IAEe,KAAK,CAAC,kBAAkB,CAAC,aAAa,GAAG,KAAK;QAC1D,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC;QAE5D,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YAC3B,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QAC9B,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,IAAS,EAAE,EAAE;YACrD,IAAI,IAAI,CAAC,SAAS,EAAE;gBAChB,IAAI;oBACA,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;oBACxB,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;oBACpC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;iBAC5B;gBAAC,OAAO,GAAG,EAAE;oBACV,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,qBAAY,CAAC,qCAAqC,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;iBACpG;aACJ;iBAAM;gBACH,aAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;aAChD;QACL,CAAC,CAAC,CAAC;QAEH,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;IAC1D,CAAC;CA4FJ;AA9HD,gDA8HC"}
|
|
@@ -1,13 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
3
|
exports.RetryDataSource = void 0;
|
|
13
4
|
const promises_1 = require("timers/promises");
|
|
@@ -17,41 +8,35 @@ class RetryDataSource {
|
|
|
17
8
|
this.datasource = new DataSource_1.DataSource(name, protocolStrategy, dataTypeStrategy, validator);
|
|
18
9
|
this.retryCount = retryCount;
|
|
19
10
|
}
|
|
20
|
-
getAll() {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
11
|
+
async getAll() {
|
|
12
|
+
try {
|
|
13
|
+
return await this.datasource.getAll();
|
|
14
|
+
}
|
|
15
|
+
catch (err) {
|
|
16
|
+
if (this.retryCount > 0) {
|
|
17
|
+
this.retryCount--;
|
|
18
|
+
await this.sleep();
|
|
19
|
+
return await this.getAll();
|
|
24
20
|
}
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
this.retryCount--;
|
|
28
|
-
yield this.sleep();
|
|
29
|
-
return yield this.getAll();
|
|
30
|
-
}
|
|
31
|
-
throw err;
|
|
32
|
-
}
|
|
33
|
-
});
|
|
21
|
+
throw err;
|
|
22
|
+
}
|
|
34
23
|
}
|
|
35
|
-
getLastModified() {
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
return yield this.getLastModified();
|
|
45
|
-
}
|
|
46
|
-
throw err;
|
|
24
|
+
async getLastModified() {
|
|
25
|
+
try {
|
|
26
|
+
return await this.datasource.getLastModified();
|
|
27
|
+
}
|
|
28
|
+
catch (err) {
|
|
29
|
+
if (this.retryCount > 0) {
|
|
30
|
+
this.retryCount--;
|
|
31
|
+
await this.sleep();
|
|
32
|
+
return await this.getLastModified();
|
|
47
33
|
}
|
|
48
|
-
|
|
34
|
+
throw err;
|
|
35
|
+
}
|
|
49
36
|
}
|
|
50
|
-
sleep() {
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
yield (0, promises_1.setTimeout)(Math.random() * 55000 + 5000);
|
|
54
|
-
});
|
|
37
|
+
async sleep() {
|
|
38
|
+
// sleep random time between 5 seconds to 60 seconds
|
|
39
|
+
await (0, promises_1.setTimeout)(Math.random() * 55000 + 5000);
|
|
55
40
|
}
|
|
56
41
|
}
|
|
57
42
|
exports.RetryDataSource = RetryDataSource;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RetryDataSource.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/RetryDataSource.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"RetryDataSource.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/RetryDataSource.ts"],"names":[],"mappings":";;;AACA,8CAAsD;AACtD,6CAA0C;AAI1C,MAAa,eAAe;IAIxB,YACI,IAAY,EACZ,gBAAmC,EACnC,mBAA6C,IAAI,EACjD,YAA+B,IAAI,EACnC,aAAqB,CAAC;QAEtB,IAAI,CAAC,UAAU,GAAG,IAAI,uBAAU,CAAI,IAAI,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,SAAS,CAAC,CAAC;QACzF,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IACjC,CAAC;IAEM,KAAK,CAAC,MAAM;QACf,IAAI;YACA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;SACzC;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE;gBACrB,IAAI,CAAC,UAAU,EAAE,CAAC;gBAClB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACnB,OAAO,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;aAC9B;YACD,MAAM,GAAG,CAAC;SACb;IACL,CAAC;IAEM,KAAK,CAAC,eAAe;QACxB,IAAI;YACA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,eAAe,EAAE,CAAC;SAClD;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE;gBACrB,IAAI,CAAC,UAAU,EAAE,CAAC;gBAClB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACnB,OAAO,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;aACvC;YACD,MAAM,GAAG,CAAC;SACb;IACL,CAAC;IAEO,KAAK,CAAC,KAAK;QACf,oDAAoD;QACpD,MAAM,IAAA,qBAAK,EAAC,IAAI,CAAC,MAAM,EAAE,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC;IAC9C,CAAC;CACJ;AA7CD,0CA6CC"}
|
|
@@ -1,13 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
3
|
exports.CSVDataTypeStrategy = void 0;
|
|
13
4
|
const errors_1 = require("@golemio/errors");
|
|
@@ -24,30 +15,28 @@ class CSVDataTypeStrategy {
|
|
|
24
15
|
setFilter(filterFunction) {
|
|
25
16
|
this.filter = filterFunction;
|
|
26
17
|
}
|
|
27
|
-
parseData(data) {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
return resolve(resultsArray);
|
|
50
|
-
});
|
|
18
|
+
async parseData(data) {
|
|
19
|
+
const readable = new stream_1.Readable();
|
|
20
|
+
readable._read = () => {
|
|
21
|
+
// _read is required but you can noop it
|
|
22
|
+
};
|
|
23
|
+
readable.push(data);
|
|
24
|
+
readable.push(null);
|
|
25
|
+
return new Promise((resolve, reject) => {
|
|
26
|
+
let resultsArray = [];
|
|
27
|
+
readable
|
|
28
|
+
.pipe((0, fast_csv_1.parse)(this.dataTypeSettings.fastcsvParams))
|
|
29
|
+
.on("error", (error) => {
|
|
30
|
+
reject(new errors_1.GeneralError("Error while parsing source data.", this.constructor.name, error));
|
|
31
|
+
})
|
|
32
|
+
.on("data", (row) => {
|
|
33
|
+
resultsArray.push(this.dataTypeSettings.subscribe(row));
|
|
34
|
+
})
|
|
35
|
+
.on("end", (rowCount) => {
|
|
36
|
+
if (this.filter) {
|
|
37
|
+
resultsArray = resultsArray.filter(this.filter);
|
|
38
|
+
}
|
|
39
|
+
return resolve(resultsArray);
|
|
51
40
|
});
|
|
52
41
|
});
|
|
53
42
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CSVDataTypeStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/CSVDataTypeStrategy.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"CSVDataTypeStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/CSVDataTypeStrategy.ts"],"names":[],"mappings":";;;AAAA,4CAA+C;AAC/C,uCAAiD;AACjD,mCAAkC;AAGlC,MAAa,mBAAmB;IAI5B,YAAY,QAAsB;QAC9B,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAC;QACjC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;IAC5B,CAAC;IAEM,mBAAmB,CAAC,QAAsB;QAC7C,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAC;IACrC,CAAC;IAEM,SAAS,CAAC,cAAkC;QAC/C,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC;IACjC,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,IAAS;QAC5B,MAAM,QAAQ,GAAG,IAAI,iBAAQ,EAAE,CAAC;QAChC,QAAQ,CAAC,KAAK,GAAG,GAAG,EAAE;YAClB,wCAAwC;QAC5C,CAAC,CAAC;QACF,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,YAAY,GAAU,EAAE,CAAC;YAC7B,QAAQ;iBACH,IAAI,CAAC,IAAA,gBAAY,EAAC,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;iBACvD,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBACnB,MAAM,CAAC,IAAI,qBAAY,CAAC,kCAAkC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;YAC/F,CAAC,CAAC;iBACD,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE;gBAChB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;YAC5D,CAAC,CAAC;iBACD,EAAE,CAAC,KAAK,EAAE,CAAC,QAAgB,EAAE,EAAE;gBAC5B,IAAI,IAAI,CAAC,MAAM,EAAE;oBACb,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;iBACnD;gBACD,OAAO,OAAO,CAAC,YAAY,CAAC,CAAC;YACjC,CAAC,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;IACP,CAAC;CACJ;AA3CD,kDA2CC"}
|
|
@@ -1,13 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
3
|
exports.JSONDataTypeStrategy = void 0;
|
|
13
4
|
const utils_1 = require("../../../helpers/utils");
|
|
@@ -23,24 +14,22 @@ class JSONDataTypeStrategy {
|
|
|
23
14
|
setFilter(filterFunction) {
|
|
24
15
|
this.filter = filterFunction;
|
|
25
16
|
}
|
|
26
|
-
parseData(data) {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
data = JSON.parse(data);
|
|
31
|
-
}
|
|
32
|
-
if (this.resultsPath) {
|
|
33
|
-
data = (0, utils_1.getSubProperty)(this.resultsPath, data);
|
|
34
|
-
}
|
|
35
|
-
if (this.filter) {
|
|
36
|
-
data = data.filter(this.filter);
|
|
37
|
-
}
|
|
38
|
-
return data;
|
|
17
|
+
async parseData(data) {
|
|
18
|
+
try {
|
|
19
|
+
if (typeof data === "string") {
|
|
20
|
+
data = JSON.parse(data);
|
|
39
21
|
}
|
|
40
|
-
|
|
41
|
-
|
|
22
|
+
if (this.resultsPath) {
|
|
23
|
+
data = (0, utils_1.getSubProperty)(this.resultsPath, data);
|
|
42
24
|
}
|
|
43
|
-
|
|
25
|
+
if (this.filter) {
|
|
26
|
+
data = data.filter(this.filter);
|
|
27
|
+
}
|
|
28
|
+
return data;
|
|
29
|
+
}
|
|
30
|
+
catch (err) {
|
|
31
|
+
throw new errors_1.GeneralError("Error while parsing source data.", this.constructor.name, err);
|
|
32
|
+
}
|
|
44
33
|
}
|
|
45
34
|
}
|
|
46
35
|
exports.JSONDataTypeStrategy = JSONDataTypeStrategy;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"JSONDataTypeStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/JSONDataTypeStrategy.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"JSONDataTypeStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/JSONDataTypeStrategy.ts"],"names":[],"mappings":";;;AAAA,kDAAgD;AAEhD,4CAA+C;AAE/C,MAAa,oBAAoB;IAI7B,YAAY,QAAuB;QAC/B,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC;QACxC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;IAC5B,CAAC;IAEM,mBAAmB,CAAC,QAAuB;QAC9C,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC;IAC5C,CAAC;IAEM,SAAS,CAAC,cAAkC;QAC/C,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC;IACjC,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,IAAS;QAC5B,IAAI;YACA,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;gBAC1B,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;aAC3B;YAED,IAAI,IAAI,CAAC,WAAW,EAAE;gBAClB,IAAI,GAAG,IAAA,sBAAc,EAAM,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;aACtD;YAED,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aACnC;YACD,OAAO,IAAI,CAAC;SACf;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,IAAI,qBAAY,CAAC,kCAAkC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;SAC1F;IACL,CAAC;CACJ;AAnCD,oDAmCC"}
|
|
@@ -1,13 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
2
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
4
|
};
|
|
@@ -27,20 +18,18 @@ class XMLDataTypeStrategy {
|
|
|
27
18
|
setFilter(filterFunction) {
|
|
28
19
|
this.filter = filterFunction;
|
|
29
20
|
}
|
|
30
|
-
parseData(data) {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
parsed = parsed.filter(this.filter);
|
|
37
|
-
}
|
|
38
|
-
return parsed;
|
|
39
|
-
}
|
|
40
|
-
catch (err) {
|
|
41
|
-
throw new errors_1.GeneralError("Error while parsing source data.", this.constructor.name, err);
|
|
21
|
+
async parseData(data) {
|
|
22
|
+
try {
|
|
23
|
+
let parsed = await xml2js_1.default.parseStringPromise(data, this.dataTypeSettings.xml2jsParams);
|
|
24
|
+
parsed = (0, utils_1.getSubProperty)(this.dataTypeSettings.resultsPath, parsed);
|
|
25
|
+
if (this.filter) {
|
|
26
|
+
parsed = parsed.filter(this.filter);
|
|
42
27
|
}
|
|
43
|
-
|
|
28
|
+
return parsed;
|
|
29
|
+
}
|
|
30
|
+
catch (err) {
|
|
31
|
+
throw new errors_1.GeneralError("Error while parsing source data.", this.constructor.name, err);
|
|
32
|
+
}
|
|
44
33
|
}
|
|
45
34
|
}
|
|
46
35
|
exports.XMLDataTypeStrategy = XMLDataTypeStrategy;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"XMLDataTypeStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/XMLDataTypeStrategy.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"XMLDataTypeStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/XMLDataTypeStrategy.ts"],"names":[],"mappings":";;;;;;AAAA,kDAAgD;AAEhD,4CAA+C;AAC/C,oDAA4B;AAE5B,MAAa,mBAAmB;IAI5B,YAAY,QAAsB;QAC9B,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAC;QACjC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;IAC5B,CAAC;IAEM,mBAAmB,CAAC,QAAsB;QAC7C,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAC;IACrC,CAAC;IAEM,SAAS,CAAC,cAAkC;QAC/C,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC;IACjC,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,IAAS;QAC5B,IAAI;YACA,IAAI,MAAM,GAAG,MAAM,gBAAM,CAAC,kBAAkB,CAAC,IAAI,EAAE,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;YACvF,MAAM,GAAG,IAAA,sBAAc,EAAC,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACnE,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aACvC;YACD,OAAO,MAAM,CAAC;SACjB;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,IAAI,qBAAY,CAAC,kCAAkC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;SAC1F;IACL,CAAC;CACJ;AA7BD,kDA6BC"}
|
|
@@ -1,13 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
2
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
4
|
};
|
|
@@ -31,43 +22,40 @@ class ZipFileDataTypeStrategy {
|
|
|
31
22
|
setFilter(filterFunction) {
|
|
32
23
|
this.filter = filterFunction;
|
|
33
24
|
}
|
|
34
|
-
parseData(data) {
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
if (data instanceof ArrayBuffer) {
|
|
39
|
-
data = Buffer.from(data);
|
|
40
|
-
}
|
|
41
|
-
const { extractionPath, files } = yield this.fileCompressor.inflateBuffer(data, (filename) => {
|
|
42
|
-
if (this.settings.whitelistedFiles) {
|
|
43
|
-
return this.settings.whitelistedFiles.indexOf(filename) >= 0;
|
|
44
|
-
}
|
|
45
|
-
return !!filename;
|
|
46
|
-
});
|
|
47
|
-
const redisModel = new RedisModel_1.RedisModel("HTTPProtocolStrategy" + "Model", {
|
|
48
|
-
isKeyConstructedFromData: false,
|
|
49
|
-
prefix: "files",
|
|
50
|
-
}, null);
|
|
51
|
-
const prefix = this.settings.name;
|
|
52
|
-
const encoding = (_a = this.settings.encoding) !== null && _a !== void 0 ? _a : "utf8";
|
|
53
|
-
const body = yield Promise.all(files.map((file) => __awaiter(this, void 0, void 0, function* () {
|
|
54
|
-
const data = yield promises_1.default.readFile(file.path, encoding);
|
|
55
|
-
const relativePath = path_1.default.join(prefix, file.name);
|
|
56
|
-
yield redisModel.hset(relativePath, data.toString(encoding));
|
|
57
|
-
return {
|
|
58
|
-
filepath: relativePath,
|
|
59
|
-
mtime: file.mtime,
|
|
60
|
-
name: path_1.default.parse(relativePath).name,
|
|
61
|
-
path: file.path,
|
|
62
|
-
};
|
|
63
|
-
})));
|
|
64
|
-
yield promises_1.default.rm(extractionPath, { recursive: true, force: true });
|
|
65
|
-
return body;
|
|
25
|
+
async parseData(data) {
|
|
26
|
+
try {
|
|
27
|
+
if (data instanceof ArrayBuffer) {
|
|
28
|
+
data = Buffer.from(data);
|
|
66
29
|
}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
30
|
+
const { extractionPath, files } = await this.fileCompressor.inflateBuffer(data, (filename) => {
|
|
31
|
+
if (this.settings.whitelistedFiles) {
|
|
32
|
+
return this.settings.whitelistedFiles.indexOf(filename) >= 0;
|
|
33
|
+
}
|
|
34
|
+
return !!filename;
|
|
35
|
+
});
|
|
36
|
+
const redisModel = new RedisModel_1.RedisModel("HTTPProtocolStrategy" + "Model", {
|
|
37
|
+
isKeyConstructedFromData: false,
|
|
38
|
+
prefix: "files",
|
|
39
|
+
}, null);
|
|
40
|
+
const prefix = this.settings.name;
|
|
41
|
+
const encoding = this.settings.encoding ?? "utf8";
|
|
42
|
+
const body = await Promise.all(files.map(async (file) => {
|
|
43
|
+
const data = await promises_1.default.readFile(file.path, encoding);
|
|
44
|
+
const relativePath = path_1.default.join(prefix, file.name);
|
|
45
|
+
await redisModel.hset(relativePath, data.toString(encoding));
|
|
46
|
+
return {
|
|
47
|
+
filepath: relativePath,
|
|
48
|
+
mtime: file.mtime,
|
|
49
|
+
name: path_1.default.parse(relativePath).name,
|
|
50
|
+
path: file.path,
|
|
51
|
+
};
|
|
52
|
+
}));
|
|
53
|
+
await promises_1.default.rm(extractionPath, { recursive: true, force: true });
|
|
54
|
+
return body;
|
|
55
|
+
}
|
|
56
|
+
catch (err) {
|
|
57
|
+
throw new errors_1.GeneralError("Error unzipping source file.", this.constructor.name, err);
|
|
58
|
+
}
|
|
71
59
|
}
|
|
72
60
|
}
|
|
73
61
|
exports.ZipFileDataTypeStrategy = ZipFileDataTypeStrategy;
|
package/dist/integration-engine/datasources/datatype-strategy/ZipFileDataTypeStrategy.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ZipFileDataTypeStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/ZipFileDataTypeStrategy.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"ZipFileDataTypeStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/ZipFileDataTypeStrategy.ts"],"names":[],"mappings":";;;;;;AACA,iEAA4D;AAC5D,wDAAmD;AACnD,4CAA+C;AAC/C,2DAA6B;AAC7B,4CAAoB;AACpB,gDAAwB;AAExB,MAAa,uBAAuB;IAIhC,YAAoB,QAAsB;QAAtB,aAAQ,GAAR,QAAQ,CAAc;QACtC,IAAI,CAAC,cAAc,GAAG,IAAI,+BAAc,CAAC,YAAE,CAAC,MAAM,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;IAC5B,CAAC;IAEM,mBAAmB,CAAC,QAAsB;QAC7C,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC7B,CAAC;IAEM,SAAS,CAAC,cAAkC;QAC/C,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC;IACjC,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,IAA0B;QAC7C,IAAI;YACA,IAAI,IAAI,YAAY,WAAW,EAAE;gBAC7B,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aAC5B;YAED,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,IAAc,EAAE,CAAC,QAAQ,EAAE,EAAE;gBACnG,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE;oBAChC,OAAO,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;iBAChE;gBAED,OAAO,CAAC,CAAC,QAAQ,CAAC;YACtB,CAAC,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,IAAI,uBAAU,CAC7B,sBAAsB,GAAG,OAAO,EAChC;gBACI,wBAAwB,EAAE,KAAK;gBAC/B,MAAM,EAAE,OAAO;aAClB,EACD,IAAI,CACP,CAAC;YACF,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,MAAM,CAAC;YAElD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAC1B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;gBACrB,MAAM,IAAI,GAAG,MAAM,kBAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,QAAe,CAAC,CAAC;gBAC3D,MAAM,YAAY,GAAG,cAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBAElD,MAAM,UAAU,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,QAA0B,CAAC,CAAC,CAAC;gBAC/E,OAAO;oBACH,QAAQ,EAAE,YAAY;oBACtB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,IAAI,EAAE,cAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI;oBACnC,IAAI,EAAE,IAAI,CAAC,IAAI;iBAClB,CAAC;YACN,CAAC,CAAC,CACL,CAAC;YAEF,MAAM,kBAAE,CAAC,EAAE,CAAC,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAE9D,OAAO,IAAI,CAAC;SACf;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,IAAI,qBAAY,CAAC,8BAA8B,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;SACtF;IACL,CAAC;CACJ;AAhED,0DAgEC"}
|