@golemio/core 1.18.0 → 1.18.1-dev.1622172793
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/helpers/DateTime.js +4 -5
- package/dist/helpers/DateTime.js.map +1 -1
- package/dist/helpers/HealthCheck.js +5 -14
- package/dist/helpers/HealthCheck.js.map +1 -1
- package/dist/helpers/configuration/SimpleConfig.js +4 -1
- package/dist/helpers/configuration/SimpleConfig.js.map +1 -1
- package/dist/helpers/data-access/amqp/AlternateExchangeCreator.js +4 -13
- package/dist/helpers/data-access/amqp/AlternateExchangeCreator.js.map +1 -1
- package/dist/helpers/data-access/amqp/AmqpConnector.js +28 -40
- package/dist/helpers/data-access/amqp/AmqpConnector.js.map +1 -1
- package/dist/helpers/data-access/feature-flags/helpers/AbstractFeatureFlagServiceWrapper.js +2 -13
- package/dist/helpers/data-access/feature-flags/helpers/AbstractFeatureFlagServiceWrapper.js.map +1 -1
- package/dist/helpers/data-access/feature-flags/providers/NoopFeatureFlagService.js +9 -24
- package/dist/helpers/data-access/feature-flags/providers/NoopFeatureFlagService.js.map +1 -1
- package/dist/helpers/data-access/postgres/DatabaseConnector.js +23 -25
- package/dist/helpers/data-access/postgres/DatabaseConnector.js.map +1 -1
- package/dist/helpers/data-access/postgres/repositories/AbstractCachedRepository.js +3 -12
- package/dist/helpers/data-access/postgres/repositories/AbstractCachedRepository.js.map +1 -1
- package/dist/helpers/data-access/pubsub/subscribers/RedisSubscriber.js +15 -30
- package/dist/helpers/data-access/pubsub/subscribers/RedisSubscriber.js.map +1 -1
- package/dist/helpers/data-access/redis/IoRedisConnector.js +8 -17
- package/dist/helpers/data-access/redis/IoRedisConnector.js.map +1 -1
- package/dist/helpers/data-access/storage/AzureBlobStorageService.js +64 -81
- package/dist/helpers/data-access/storage/AzureBlobStorageService.js.map +1 -1
- package/dist/helpers/data-access/storage/NoopStorageService.js +9 -26
- package/dist/helpers/data-access/storage/NoopStorageService.js.map +1 -1
- package/dist/helpers/data-access/table-storage/TableStorageServiceFactory.js +1 -2
- package/dist/helpers/data-access/table-storage/TableStorageServiceFactory.js.map +1 -1
- package/dist/helpers/data-access/table-storage/providers/AzureTableStorageService.js +39 -68
- package/dist/helpers/data-access/table-storage/providers/AzureTableStorageService.js.map +1 -1
- package/dist/helpers/data-access/table-storage/providers/NoopTableStorageService.js +4 -17
- package/dist/helpers/data-access/table-storage/providers/NoopTableStorageService.js.map +1 -1
- package/dist/helpers/logger/LoggerProvider.js +33 -14
- package/dist/helpers/logger/LoggerProvider.js.map +1 -1
- package/dist/helpers/logger/RequestLoggerProvider.js +1 -2
- package/dist/helpers/logger/RequestLoggerProvider.js.map +1 -1
- package/dist/helpers/routing/AbstractGeoJsonRouter.js +16 -34
- package/dist/helpers/routing/AbstractGeoJsonRouter.js.map +1 -1
- package/dist/input-gateway/config/InputGatewayConfiguration.js +1 -2
- package/dist/input-gateway/config/InputGatewayConfiguration.js.map +1 -1
- package/dist/input-gateway/connectors/AMQPConnector.js +52 -68
- package/dist/input-gateway/connectors/AMQPConnector.js.map +1 -1
- package/dist/input-gateway/controllers/BaseController.js +4 -13
- package/dist/input-gateway/controllers/BaseController.js.map +1 -1
- package/dist/integration-engine/connectors/AMQPConnector.js +28 -40
- package/dist/integration-engine/connectors/AMQPConnector.js.map +1 -1
- package/dist/integration-engine/data-access/RawDataStorage.js +7 -16
- package/dist/integration-engine/data-access/RawDataStorage.js.map +1 -1
- package/dist/integration-engine/data-access/pubsub/RedisPubSubChannel.js +3 -14
- package/dist/integration-engine/data-access/pubsub/RedisPubSubChannel.js.map +1 -1
- package/dist/integration-engine/data-access/pubsub/subscribers/RedisSubscriber.js +12 -25
- package/dist/integration-engine/data-access/pubsub/subscribers/RedisSubscriber.js.map +1 -1
- package/dist/integration-engine/datasources/DataSource.js +41 -60
- package/dist/integration-engine/datasources/DataSource.js.map +1 -1
- package/dist/integration-engine/datasources/DataSourceStream.js +12 -21
- package/dist/integration-engine/datasources/DataSourceStream.js.map +1 -1
- package/dist/integration-engine/datasources/DataSourceStreamManager.js +4 -13
- package/dist/integration-engine/datasources/DataSourceStreamManager.js.map +1 -1
- package/dist/integration-engine/datasources/DataSourceStreamed.js +37 -51
- package/dist/integration-engine/datasources/DataSourceStreamed.js.map +1 -1
- package/dist/integration-engine/datasources/RetryDataSource.js +25 -40
- package/dist/integration-engine/datasources/RetryDataSource.js.map +1 -1
- package/dist/integration-engine/datasources/datatype-strategy/CSVDataTypeStrategy.js +22 -33
- package/dist/integration-engine/datasources/datatype-strategy/CSVDataTypeStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/datatype-strategy/JSONDataTypeStrategy.js +14 -25
- package/dist/integration-engine/datasources/datatype-strategy/JSONDataTypeStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/datatype-strategy/XMLDataTypeStrategy.js +11 -22
- package/dist/integration-engine/datasources/datatype-strategy/XMLDataTypeStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/datatype-strategy/ZipFileDataTypeStrategy.js +33 -45
- package/dist/integration-engine/datasources/datatype-strategy/ZipFileDataTypeStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/FTPProtocolStrategy.js +17 -26
- package/dist/integration-engine/datasources/protocol-strategy/FTPProtocolStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/GoogleCloudStorageProtocolStrategy.js +10 -19
- package/dist/integration-engine/datasources/protocol-strategy/GoogleCloudStorageProtocolStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/HTTPFetchProtocolStrategy.js +42 -51
- package/dist/integration-engine/datasources/protocol-strategy/HTTPFetchProtocolStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategy.js +49 -62
- package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.js +67 -81
- package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/PaginatedHTTPProtocolStrategy.js +29 -38
- package/dist/integration-engine/datasources/protocol-strategy/PaginatedHTTPProtocolStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/PostgresProtocolStrategy.js +37 -42
- package/dist/integration-engine/datasources/protocol-strategy/PostgresProtocolStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/PostgresProtocolStrategyStreamed.js +29 -32
- package/dist/integration-engine/datasources/protocol-strategy/PostgresProtocolStrategyStreamed.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/ProtocolStrategy.js +73 -88
- package/dist/integration-engine/datasources/protocol-strategy/ProtocolStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/SFTPProtocolStrategy.js +17 -26
- package/dist/integration-engine/datasources/protocol-strategy/SFTPProtocolStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/helpers/FTPFileProcessor.js +37 -49
- package/dist/integration-engine/datasources/protocol-strategy/helpers/FTPFileProcessor.js.map +1 -1
- package/dist/integration-engine/helpers/FileCompressor.js +59 -76
- package/dist/integration-engine/helpers/FileCompressor.js.map +1 -1
- package/dist/integration-engine/helpers/GeocodeApi.js +20 -30
- package/dist/integration-engine/helpers/GeocodeApi.js.map +1 -1
- package/dist/integration-engine/helpers/IntegrationErrorHandler.js +3 -3
- package/dist/integration-engine/helpers/IntegrationErrorHandler.js.map +1 -1
- package/dist/integration-engine/helpers/PaginationHelper.js +1 -2
- package/dist/integration-engine/helpers/PaginationHelper.js.map +1 -1
- package/dist/integration-engine/models/PostgresModel.js +70 -75
- package/dist/integration-engine/models/PostgresModel.js.map +1 -1
- package/dist/integration-engine/models/RedisModel.js +28 -39
- package/dist/integration-engine/models/RedisModel.js.map +1 -1
- package/dist/integration-engine/queueprocessors/QueueProcessor.js +23 -28
- package/dist/integration-engine/queueprocessors/QueueProcessor.js.map +1 -1
- package/dist/integration-engine/transformations/BaseTransformation.js +8 -17
- package/dist/integration-engine/transformations/BaseTransformation.js.map +1 -1
- package/dist/integration-engine/workers/AbstractTask.js +7 -18
- package/dist/integration-engine/workers/AbstractTask.js.map +1 -1
- package/dist/integration-engine/workers/AbstractTaskJsonSchema.js +25 -38
- package/dist/integration-engine/workers/AbstractTaskJsonSchema.js.map +1 -1
- package/dist/integration-engine/workers/AbstractTaskWithoutValidation.js +2 -13
- package/dist/integration-engine/workers/AbstractTaskWithoutValidation.js.map +1 -1
- package/dist/integration-engine/workers/BaseWorker.js +4 -13
- package/dist/integration-engine/workers/BaseWorker.js.map +1 -1
- package/dist/integration-engine/workers/helpers/MessageDataParser.js +1 -2
- package/dist/integration-engine/workers/helpers/MessageDataParser.js.map +1 -1
- package/dist/integration-engine/workers/helpers/MessageDataValidator.js +18 -29
- package/dist/integration-engine/workers/helpers/MessageDataValidator.js.map +1 -1
- package/dist/integration-engine/workers/helpers/WebhookDecorators.js +1 -10
- package/dist/integration-engine/workers/helpers/WebhookDecorators.js.map +1 -1
- package/dist/integration-engine/workers/webhooks/CloudflareCachePurgeWebhook.js +13 -24
- package/dist/integration-engine/workers/webhooks/CloudflareCachePurgeWebhook.js.map +1 -1
- package/dist/monitoring/metrics/metrics-builder.js +21 -17
- package/dist/monitoring/metrics/metrics-builder.js.map +1 -1
- package/dist/monitoring/metrics/metrics-service.js +4 -14
- package/dist/monitoring/metrics/metrics-service.js.map +1 -1
- package/dist/monitoring/opentelemetry/instrumentation/configs/amqplib.js +1 -2
- package/dist/monitoring/opentelemetry/instrumentation/configs/amqplib.js.map +1 -1
- package/dist/monitoring/opentelemetry/instrumentation/configs/aws-sdk.js +1 -1
- package/dist/monitoring/opentelemetry/instrumentation/configs/aws-sdk.js.map +1 -1
- package/dist/monitoring/opentelemetry/trace-provider.js +8 -2
- package/dist/monitoring/opentelemetry/trace-provider.js.map +1 -1
- package/dist/monitoring/sentry/sentry-provider.js +1 -1
- package/dist/monitoring/sentry/sentry-provider.js.map +1 -1
- package/dist/monitoring/sentry/sentry-transactions.js +9 -19
- package/dist/monitoring/sentry/sentry-transactions.js.map +1 -1
- package/dist/output-gateway/Geo.js +3 -24
- package/dist/output-gateway/Geo.js.map +1 -1
- package/dist/output-gateway/config/OutputGatewayConfiguration.js +1 -2
- package/dist/output-gateway/config/OutputGatewayConfiguration.js.map +1 -1
- package/dist/output-gateway/database/index.js +3 -12
- package/dist/output-gateway/database/index.js.map +1 -1
- package/dist/output-gateway/ioc/Di.js +1 -10
- package/dist/output-gateway/ioc/Di.js.map +1 -1
- package/dist/output-gateway/models/SequelizeModel.js +4 -2
- package/dist/output-gateway/models/SequelizeModel.js.map +1 -1
- package/dist/output-gateway/routes/GeoJsonRouter.js +14 -32
- package/dist/output-gateway/routes/GeoJsonRouter.js.map +1 -1
- package/package.json +6 -5
- package/tsconfig.base.json +2 -2
|
@@ -1,13 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
2
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
4
|
};
|
|
@@ -29,24 +20,24 @@ const defaultSettings = {
|
|
|
29
20
|
};
|
|
30
21
|
class FTPProtocolStrategy extends ProtocolStrategy_1.ProtocolStrategy {
|
|
31
22
|
constructor(settings) {
|
|
32
|
-
super(
|
|
23
|
+
super({ ...defaultSettings, ...settings });
|
|
33
24
|
this.setConnectionSettings = (settings) => {
|
|
34
|
-
this.connectionSettings =
|
|
25
|
+
this.connectionSettings = { ...defaultSettings, ...settings };
|
|
35
26
|
this.downloadPath = this.connectionSettings.tmpDir;
|
|
36
27
|
this.fileProcessor = new FTPFileProcessor_1.FTPFileProcessor(this.connectionSettings, this.downloadPath);
|
|
37
28
|
};
|
|
38
|
-
this.getRawData = () =>
|
|
29
|
+
this.getRawData = async () => {
|
|
39
30
|
try {
|
|
40
|
-
const dirHandle =
|
|
41
|
-
|
|
31
|
+
const dirHandle = await fs_1.default.promises.open(this.downloadPath, fs_1.default.constants.O_DIRECTORY);
|
|
32
|
+
await dirHandle.close();
|
|
42
33
|
}
|
|
43
34
|
catch (err) {
|
|
44
35
|
throw new errors_1.GeneralError("FTP download path is not a directory", this.constructor.name, err);
|
|
45
36
|
}
|
|
46
37
|
try {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
38
|
+
await this.ftpClient.access(this.connectionSettings.url);
|
|
39
|
+
await this.ftpClient.cd(this.connectionSettings.path);
|
|
40
|
+
await this.ftpClient.downloadTo(fs_1.default.createWriteStream(path_1.default.join(this.downloadPath, this.connectionSettings.filename)), this.connectionSettings.filename);
|
|
50
41
|
}
|
|
51
42
|
catch (err) {
|
|
52
43
|
throw new errors_1.GeneralError("Error while downloading FTP files", this.constructor.name, err);
|
|
@@ -58,25 +49,25 @@ class FTPProtocolStrategy extends ProtocolStrategy_1.ProtocolStrategy {
|
|
|
58
49
|
let result;
|
|
59
50
|
try {
|
|
60
51
|
if (this.connectionSettings.targetType === IProtocolStrategy_1.FTPTargetType.COMPRESSED) {
|
|
61
|
-
result =
|
|
52
|
+
result = await this.fileProcessor.processTargetCompressed(prefix);
|
|
62
53
|
}
|
|
63
54
|
else if (this.connectionSettings.targetType === IProtocolStrategy_1.FTPTargetType.FOLDER) {
|
|
64
|
-
result =
|
|
55
|
+
result = await this.fileProcessor.processTargetFolder(prefix);
|
|
65
56
|
}
|
|
66
57
|
else {
|
|
67
|
-
result =
|
|
58
|
+
result = await this.fileProcessor.processTargetFile();
|
|
68
59
|
}
|
|
69
60
|
}
|
|
70
61
|
catch (err) {
|
|
71
62
|
throw new errors_1.GeneralError("Error while processing downloaded FTP files", this.constructor.name, err);
|
|
72
63
|
}
|
|
73
64
|
return { data: result };
|
|
74
|
-
}
|
|
75
|
-
this.getLastModified = () =>
|
|
65
|
+
};
|
|
66
|
+
this.getLastModified = async () => {
|
|
76
67
|
try {
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
const lastModified =
|
|
68
|
+
await this.ftpClient.access(this.connectionSettings.url);
|
|
69
|
+
await this.ftpClient.cd(this.connectionSettings.path);
|
|
70
|
+
const lastModified = await this.ftpClient.lastMod(this.connectionSettings.filename);
|
|
80
71
|
if (!lastModified) {
|
|
81
72
|
throw new errors_1.GeneralError("Error while getting last modified date", this.constructor.name);
|
|
82
73
|
}
|
|
@@ -85,7 +76,7 @@ class FTPProtocolStrategy extends ProtocolStrategy_1.ProtocolStrategy {
|
|
|
85
76
|
finally {
|
|
86
77
|
this.ftpClient.close();
|
|
87
78
|
}
|
|
88
|
-
}
|
|
79
|
+
};
|
|
89
80
|
this.ftpClient = new basic_ftp_1.Client();
|
|
90
81
|
this.ftpClient.ftp.log = helpers_1.log.silly.bind(helpers_1.log);
|
|
91
82
|
this.ftpClient.ftp.verbose = true;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FTPProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/FTPProtocolStrategy.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"FTPProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/FTPProtocolStrategy.ts"],"names":[],"mappings":";;;;;;AAAA,2DAAwH;AACxH,yDAAsF;AACtF,2CAAkC;AAClC,4CAA+C;AAC/C,yCAAmC;AACnC,4CAAoB;AACpB,4CAAoB;AACpB,gDAAwB;AACxB,iEAA8D;AAM9D,MAAM,eAAe,GAA0B;IAC3C,MAAM,EAAE,YAAE,CAAC,MAAM,EAAE;IACnB,UAAU,EAAE,iCAAa,CAAC,IAAI;IAC9B,UAAU,EAAE,iCAAa,CAAC,IAAI;CACjC,CAAC;AAEF,MAAa,mBAAoB,SAAQ,mCAAgB;IAMrD,YAAY,QAAsB;QAC9B,KAAK,CAAC,EAAE,GAAG,eAAe,EAAE,GAAG,QAAQ,EAAE,CAAC,CAAC;QAQxC,0BAAqB,GAAG,CAAC,QAAsB,EAAQ,EAAE;YAC5D,IAAI,CAAC,kBAAkB,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,QAAQ,EAAE,CAAC;YAC9D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAO,CAAC;YACpD,IAAI,CAAC,aAAa,GAAG,IAAI,mCAAgB,CAAC,IAAI,CAAC,kBAAkB,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAC1F,CAAC,CAAC;QAEK,eAAU,GAAG,KAAK,IAA6B,EAAE;YACpD,IAAI;gBACA,MAAM,SAAS,GAAG,MAAM,YAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,YAAE,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;gBACtF,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;aAC3B;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,qBAAY,CAAC,sCAAsC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aAC9F;YAED,IAAI;gBACA,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;gBACzD,MAAM,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;gBACtD,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAC3B,YAAE,CAAC,iBAAiB,CAAC,cAAI,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,EACpF,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CACnC,CAAC;aACL;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,qBAAY,CAAC,mCAAmC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aAC3F;oBAAS;gBACN,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;aAC1B;YAED,MAAM,MAAM,GAAG,cAAI,CAAC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC;YACvE,IAAI,MAAuB,CAAC;YAE5B,IAAI;gBACA,IAAI,IAAI,CAAC,kBAAkB,CAAC,UAAU,KAAK,iCAAa,CAAC,UAAU,EAAE;oBACjE,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC;iBACrE;qBAAM,IAAI,IAAI,CAAC,kBAAkB,CAAC,UAAU,KAAK,iCAAa,CAAC,MAAM,EAAE;oBACpE,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;iBACjE;qBAAM;oBACH,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,iBAAiB,EAAE,CAAC;iBACzD;aACJ;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,qBAAY,CAAC,6CAA6C,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aACrG;YAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAC5B,CAAC,CAAC;QAEK,oBAAe,GAAG,KAAK,IAA4B,EAAE;YACxD,IAAI;gBACA,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;gBACzD,MAAM,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;gBACtD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;gBAEpF,IAAI,CAAC,YAAY,EAAE;oBACf,MAAM,IAAI,qBAAY,CAAC,wCAAwC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;iBAC3F;gBAED,OAAO,YAAY,CAAC,WAAW,EAAE,CAAC;aACrC;oBAAS;gBACN,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;aAC1B;QACL,CAAC,CAAC;QAlEE,IAAI,CAAC,SAAS,GAAG,IAAI,kBAAM,EAAE,CAAC;QAC9B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,GAAG,aAAG,CAAC,KAAK,CAAC,IAAI,CAAC,aAAG,CAAC,CAAC;QAC7C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC;QAClC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAO,CAAC;QACpD,IAAI,CAAC,aAAa,GAAG,IAAI,mCAAgB,CAAC,IAAI,CAAC,kBAAkB,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAC1F,CAAC;CA8DJ;AA3ED,kDA2EC"}
|
package/dist/integration-engine/datasources/protocol-strategy/GoogleCloudStorageProtocolStrategy.js
CHANGED
|
@@ -1,13 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
3
|
exports.GoogleCloudStorageProtocolStrategy = void 0;
|
|
13
4
|
const errors_1 = require("@golemio/errors");
|
|
@@ -21,10 +12,10 @@ class GoogleCloudStorageProtocolStrategy extends ProtocolStrategy_1.ProtocolStra
|
|
|
21
12
|
this.connectionSettings = settings;
|
|
22
13
|
this.storage = new storage_1.Storage({ keyFilename: settings.keyFilename });
|
|
23
14
|
};
|
|
24
|
-
this.getRawData = () =>
|
|
15
|
+
this.getRawData = async () => {
|
|
25
16
|
try {
|
|
26
17
|
// Lists files in the bucket
|
|
27
|
-
let [files] =
|
|
18
|
+
let [files] = await this.storage.bucket(this.connectionSettings.bucketName).getFiles({
|
|
28
19
|
prefix: this.connectionSettings.filesPrefix,
|
|
29
20
|
});
|
|
30
21
|
// Filter files by filter function
|
|
@@ -35,26 +26,26 @@ class GoogleCloudStorageProtocolStrategy extends ProtocolStrategy_1.ProtocolStra
|
|
|
35
26
|
isKeyConstructedFromData: false,
|
|
36
27
|
prefix: "files",
|
|
37
28
|
}, null);
|
|
38
|
-
const result = files.map((file) =>
|
|
39
|
-
const fileBuffer =
|
|
29
|
+
const result = files.map(async (file) => {
|
|
30
|
+
const fileBuffer = await this.storage.bucket(this.connectionSettings.bucketName).file(file.name).download();
|
|
40
31
|
const filepath = `${this.connectionSettings.bucketName}/${file.name}`;
|
|
41
|
-
|
|
32
|
+
await redisModel.hset(filepath, fileBuffer[0].toString("utf16le"));
|
|
42
33
|
return {
|
|
43
34
|
filepath,
|
|
44
35
|
name: file.name,
|
|
45
36
|
};
|
|
46
|
-
})
|
|
37
|
+
});
|
|
47
38
|
return {
|
|
48
|
-
data:
|
|
39
|
+
data: await Promise.all(result),
|
|
49
40
|
};
|
|
50
41
|
}
|
|
51
42
|
catch (err) {
|
|
52
43
|
throw new errors_1.GeneralError("Error while getting data from server.", this.constructor.name, err);
|
|
53
44
|
}
|
|
54
|
-
}
|
|
55
|
-
this.getLastModified = () =>
|
|
45
|
+
};
|
|
46
|
+
this.getLastModified = async () => {
|
|
56
47
|
throw new Error("Method not implemented.");
|
|
57
|
-
}
|
|
48
|
+
};
|
|
58
49
|
this.storage = new storage_1.Storage({ keyFilename: settings.keyFilename });
|
|
59
50
|
}
|
|
60
51
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"GoogleCloudStorageProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/GoogleCloudStorageProtocolStrategy.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"GoogleCloudStorageProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/GoogleCloudStorageProtocolStrategy.ts"],"names":[],"mappings":";;;AAAA,4CAA+C;AAC/C,mDAAsD;AACtD,yCAAwC;AAExC,yDAAsF;AAEtF,MAAa,kCAAmC,SAAQ,mCAAgB;IAIpE,YAAY,QAAqC;QAC7C,KAAK,CAAC,QAAQ,CAAC,CAAC;QAIb,0BAAqB,GAAG,CAAC,QAAqC,EAAQ,EAAE;YAC3E,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;YACnC,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAO,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACtE,CAAC,CAAC;QAEK,eAAU,GAAG,KAAK,IAAkB,EAAE;YACzC,IAAI;gBACA,4BAA4B;gBAC5B,IAAI,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC;oBACjF,MAAM,EAAE,IAAI,CAAC,kBAAkB,CAAC,WAAW;iBAC9C,CAAC,CAAC;gBAEH,kCAAkC;gBAClC,IAAI,IAAI,CAAC,kBAAkB,CAAC,WAAW,EAAE;oBACrC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,CAAC;iBAC7D;gBAED,MAAM,UAAU,GAAG,IAAI,mBAAU,CAC7B,oCAAoC,GAAG,OAAO,EAC9C;oBACI,wBAAwB,EAAE,KAAK;oBAC/B,MAAM,EAAE,OAAO;iBAClB,EACD,IAAI,CACP,CAAC;gBAEF,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAU,EAAE,EAAE;oBAC1C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;oBAE5G,MAAM,QAAQ,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,UAAU,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;oBACtE,MAAM,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;oBACnE,OAAO;wBACH,QAAQ;wBACR,IAAI,EAAE,IAAI,CAAC,IAAI;qBAClB,CAAC;gBACN,CAAC,CAAC,CAAC;gBAEH,OAAO;oBACH,IAAI,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;iBAClC,CAAC;aACL;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,qBAAY,CAAC,uCAAuC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aAC/F;QACL,CAAC,CAAC;QAEK,oBAAe,GAAG,KAAK,IAA4B,EAAE;YACxD,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC/C,CAAC,CAAC;QAlDE,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAO,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IACtE,CAAC;CAkDJ;AAzDD,gFAyDC"}
|
|
@@ -1,13 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
3
|
exports.HTTPFetchProtocolStrategy = void 0;
|
|
13
4
|
const ProtocolStrategy_1 = require("./ProtocolStrategy");
|
|
@@ -23,56 +14,56 @@ class HTTPFetchProtocolStrategy extends ProtocolStrategy_1.ProtocolStrategy {
|
|
|
23
14
|
return headersObject;
|
|
24
15
|
};
|
|
25
16
|
}
|
|
26
|
-
getRawData() {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
data: null,
|
|
42
|
-
meta: {
|
|
43
|
-
headers,
|
|
44
|
-
statusCode,
|
|
45
|
-
},
|
|
46
|
-
};
|
|
47
|
-
}
|
|
48
|
-
let data;
|
|
49
|
-
if (this.connectionSettings.responseType === "json") {
|
|
50
|
-
data = (yield result.json());
|
|
51
|
-
}
|
|
52
|
-
else if (this.connectionSettings.responseType === "text") {
|
|
53
|
-
data = (yield result.text());
|
|
54
|
-
}
|
|
55
|
-
else if (this.connectionSettings.responseType === "blob") {
|
|
56
|
-
data = (yield result.blob());
|
|
57
|
-
}
|
|
58
|
-
else if (this.connectionSettings.responseType === "arrayBuffer") {
|
|
59
|
-
data = (yield result.arrayBuffer());
|
|
60
|
-
}
|
|
61
|
-
else {
|
|
62
|
-
data = (yield result.json());
|
|
63
|
-
}
|
|
17
|
+
async getRawData() {
|
|
18
|
+
let statusCode;
|
|
19
|
+
try {
|
|
20
|
+
const abortController = this.prepareAbortController();
|
|
21
|
+
const result = await fetch(this.connectionSettings.url, {
|
|
22
|
+
body: this.connectionSettings.body,
|
|
23
|
+
headers: {
|
|
24
|
+
...this.connectionSettings.headers,
|
|
25
|
+
},
|
|
26
|
+
method: this.connectionSettings.method,
|
|
27
|
+
signal: abortController?.signal,
|
|
28
|
+
});
|
|
29
|
+
const headers = this.getHeaders(result.headers);
|
|
30
|
+
statusCode = result.status;
|
|
31
|
+
if (statusCode === 304) {
|
|
64
32
|
return {
|
|
65
|
-
data,
|
|
33
|
+
data: null,
|
|
66
34
|
meta: {
|
|
67
35
|
headers,
|
|
68
36
|
statusCode,
|
|
69
37
|
},
|
|
70
38
|
};
|
|
71
39
|
}
|
|
72
|
-
|
|
73
|
-
|
|
40
|
+
let data;
|
|
41
|
+
if (this.connectionSettings.responseType === "json") {
|
|
42
|
+
data = (await result.json());
|
|
43
|
+
}
|
|
44
|
+
else if (this.connectionSettings.responseType === "text") {
|
|
45
|
+
data = (await result.text());
|
|
46
|
+
}
|
|
47
|
+
else if (this.connectionSettings.responseType === "blob") {
|
|
48
|
+
data = (await result.blob());
|
|
49
|
+
}
|
|
50
|
+
else if (this.connectionSettings.responseType === "arrayBuffer") {
|
|
51
|
+
data = (await result.arrayBuffer());
|
|
74
52
|
}
|
|
75
|
-
|
|
53
|
+
else {
|
|
54
|
+
data = (await result.json());
|
|
55
|
+
}
|
|
56
|
+
return {
|
|
57
|
+
data,
|
|
58
|
+
meta: {
|
|
59
|
+
headers,
|
|
60
|
+
statusCode,
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
catch (err) {
|
|
65
|
+
throw new errors_1.GeneralError(`Error while getting data from server. Response status code: ${statusCode}`, this.constructor.name, err);
|
|
66
|
+
}
|
|
76
67
|
}
|
|
77
68
|
prepareAbortController() {
|
|
78
69
|
if (!this.connectionSettings.timeoutInSeconds) {
|
package/dist/integration-engine/datasources/protocol-strategy/HTTPFetchProtocolStrategy.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"HTTPFetchProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/HTTPFetchProtocolStrategy.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"HTTPFetchProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/HTTPFetchProtocolStrategy.ts"],"names":[],"mappings":";;;AAAA,yDAAsF;AACtF,4CAA+C;AAI/C,MAAa,yBAAmC,SAAQ,mCAAgB;IAGpE,YAAY,QAA4B;QACpC,KAAK,CAAC,QAAQ,CAAC,CAAC;QA2DZ,eAAU,GAAG,CAAC,OAAgB,EAA0B,EAAE;YAC9D,MAAM,aAAa,GAA2B,EAAE,CAAC;YAEjD,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;gBAC3B,aAAa,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YAC/B,CAAC,CAAC,CAAC;YAEH,OAAO,aAAa,CAAC;QACzB,CAAC,CAAC;IAlEF,CAAC;IAEM,KAAK,CAAC,UAAU;QACnB,IAAI,UAA8B,CAAC;QACnC,IAAI;YACA,MAAM,eAAe,GAAG,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAEtD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE;gBACpD,IAAI,EAAE,IAAI,CAAC,kBAAkB,CAAC,IAAI;gBAClC,OAAO,EAAE;oBACL,GAAI,IAAI,CAAC,kBAAkB,CAAC,OAAkC;iBACjE;gBACD,MAAM,EAAE,IAAI,CAAC,kBAAkB,CAAC,MAAM;gBACtC,MAAM,EAAE,eAAe,EAAE,MAAM;aAClC,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAChD,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;YAE3B,IAAI,UAAU,KAAK,GAAG,EAAE;gBACpB,OAAO;oBACH,IAAI,EAAE,IAAI;oBACV,IAAI,EAAE;wBACF,OAAO;wBACP,UAAU;qBACb;iBACJ,CAAC;aACL;YAED,IAAI,IAAO,CAAC;YACZ,IAAI,IAAI,CAAC,kBAAkB,CAAC,YAAY,KAAK,MAAM,EAAE;gBACjD,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAM,CAAC;aACrC;iBAAM,IAAI,IAAI,CAAC,kBAAkB,CAAC,YAAY,KAAK,MAAM,EAAE;gBACxD,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAM,CAAC;aACrC;iBAAM,IAAI,IAAI,CAAC,kBAAkB,CAAC,YAAY,KAAK,MAAM,EAAE;gBACxD,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAM,CAAC;aACrC;iBAAM,IAAI,IAAI,CAAC,kBAAkB,CAAC,YAAY,KAAK,aAAa,EAAE;gBAC/D,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,WAAW,EAAE,CAAM,CAAC;aAC5C;iBAAM;gBACH,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAM,CAAC;aACrC;YAED,OAAO;gBACH,IAAI;gBACJ,IAAI,EAAE;oBACF,OAAO;oBACP,UAAU;iBACb;aACJ,CAAC;SACL;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,IAAI,qBAAY,CAClB,+DAA+D,UAAU,EAAE,EAC3E,IAAI,CAAC,WAAW,CAAC,IAAI,EACrB,GAAG,CACN,CAAC;SACL;IACL,CAAC;IAYO,sBAAsB;QAC1B,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;YAC3C,OAAO,IAAI,CAAC;SACf;QAED,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;QAC9C,UAAU,CAAC,GAAG,EAAE,CAAC,eAAe,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC;QAE3F,OAAO,eAAe,CAAC;IAC3B,CAAC;CACJ;AAnFD,8DAmFC"}
|
package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategy.js
CHANGED
|
@@ -1,13 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
3
|
exports.HTTPRequestProtocolStrategy = void 0;
|
|
13
4
|
const ProtocolStrategy_1 = require("./ProtocolStrategy");
|
|
@@ -27,64 +18,60 @@ class HTTPRequestProtocolStrategy extends ProtocolStrategy_1.ProtocolStrategy {
|
|
|
27
18
|
return headersObject;
|
|
28
19
|
};
|
|
29
20
|
}
|
|
30
|
-
getRawData() {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
data = (0, zlib_1.gunzipSync)(data);
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
else {
|
|
54
|
-
data = (yield result.body.json());
|
|
21
|
+
async getRawData() {
|
|
22
|
+
let statusCode;
|
|
23
|
+
try {
|
|
24
|
+
const result = await this.getResponseInternal();
|
|
25
|
+
const headers = result.headers;
|
|
26
|
+
statusCode = result.statusCode;
|
|
27
|
+
let data;
|
|
28
|
+
if (this.connectionSettings.responseType === "json") {
|
|
29
|
+
data = (await result.body.json());
|
|
30
|
+
}
|
|
31
|
+
else if (this.connectionSettings.responseType === "text") {
|
|
32
|
+
data = (await result.body.text());
|
|
33
|
+
}
|
|
34
|
+
else if (this.connectionSettings.responseType === "blob") {
|
|
35
|
+
data = (await result.body.blob());
|
|
36
|
+
}
|
|
37
|
+
else if (this.connectionSettings.responseType === "arrayBuffer") {
|
|
38
|
+
data = (await result.body.arrayBuffer());
|
|
39
|
+
if (this.connectionSettings.compression === "gzip") {
|
|
40
|
+
data = (0, zlib_1.gunzipSync)(data);
|
|
55
41
|
}
|
|
56
|
-
return {
|
|
57
|
-
data,
|
|
58
|
-
meta: {
|
|
59
|
-
headers: this.getHeaders(headers),
|
|
60
|
-
statusCode,
|
|
61
|
-
},
|
|
62
|
-
};
|
|
63
42
|
}
|
|
64
|
-
|
|
65
|
-
|
|
43
|
+
else {
|
|
44
|
+
data = (await result.body.json());
|
|
66
45
|
}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
const clientOptions = {
|
|
73
|
-
connect: {
|
|
74
|
-
timeout: this.connectionSettings.timeoutInSeconds ? this.connectionSettings.timeoutInSeconds * 1000 : undefined,
|
|
75
|
-
rejectUnauthorized: this.connectionSettings.rejectUnauthorized,
|
|
46
|
+
return {
|
|
47
|
+
data,
|
|
48
|
+
meta: {
|
|
49
|
+
headers: this.getHeaders(headers),
|
|
50
|
+
statusCode,
|
|
76
51
|
},
|
|
77
52
|
};
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
53
|
+
}
|
|
54
|
+
catch (err) {
|
|
55
|
+
throw new errors_1.GeneralError(`Error while getting data from server. Response status code: ${statusCode}`, this.constructor.name, err);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
async getResponseInternal() {
|
|
59
|
+
const url = new URL(this.connectionSettings.url);
|
|
60
|
+
const clientOptions = {
|
|
61
|
+
connect: {
|
|
62
|
+
timeout: this.connectionSettings.timeoutInSeconds ? this.connectionSettings.timeoutInSeconds * 1000 : undefined,
|
|
63
|
+
rejectUnauthorized: this.connectionSettings.rejectUnauthorized,
|
|
64
|
+
},
|
|
65
|
+
};
|
|
66
|
+
const client = new undici_1.Client(url.origin, clientOptions);
|
|
67
|
+
const requestOptions = {
|
|
68
|
+
path: url.pathname + url.search,
|
|
69
|
+
method: this.connectionSettings.method.toUpperCase(),
|
|
70
|
+
body: this.connectionSettings.body,
|
|
71
|
+
headers: this.connectionSettings.headers,
|
|
72
|
+
};
|
|
73
|
+
const result = await client.request(requestOptions);
|
|
74
|
+
return result;
|
|
88
75
|
}
|
|
89
76
|
}
|
|
90
77
|
exports.HTTPRequestProtocolStrategy = HTTPRequestProtocolStrategy;
|
package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategy.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"HTTPRequestProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategy.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"HTTPRequestProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategy.ts"],"names":[],"mappings":";;;AAAA,yDAAsF;AACtF,4CAA+C;AAC/C,mCAA4C;AAE5C,+BAAkC;AAIlC,MAAa,2BAAqC,SAAQ,mCAAgB;IAGtE,YAAY,QAA8B;QACtC,KAAK,CAAC,QAAQ,CAAC,CAAC;QA6CZ,eAAU,GAAG,CAAC,OAA4B,EAA0B,EAAE;YAC1E,MAAM,aAAa,GAA2B,EAAE,CAAC;YAEjD,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE;gBACvB,IAAI,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE;oBACpD,aAAa,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAW,CAAC;iBAC/C;aACJ;YAED,OAAO,aAAa,CAAC;QACzB,CAAC,CAAC;IAtDF,CAAC;IAEM,KAAK,CAAC,UAAU;QACnB,IAAI,UAA8B,CAAC;QAEnC,IAAI;YACA,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAEhD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;YAC/B,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;YAC/B,IAAI,IAAO,CAAC;YAEZ,IAAI,IAAI,CAAC,kBAAkB,CAAC,YAAY,KAAK,MAAM,EAAE;gBACjD,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAM,CAAC;aAC1C;iBAAM,IAAI,IAAI,CAAC,kBAAkB,CAAC,YAAY,KAAK,MAAM,EAAE;gBACxD,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAM,CAAC;aAC1C;iBAAM,IAAI,IAAI,CAAC,kBAAkB,CAAC,YAAY,KAAK,MAAM,EAAE;gBACxD,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAM,CAAC;aAC1C;iBAAM,IAAI,IAAI,CAAC,kBAAkB,CAAC,YAAY,KAAK,aAAa,EAAE;gBAC/D,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,CAAM,CAAC;gBAE9C,IAAI,IAAI,CAAC,kBAAkB,CAAC,WAAW,KAAK,MAAM,EAAE;oBAChD,IAAI,GAAG,IAAA,iBAAU,EAAC,IAAmB,CAAM,CAAC;iBAC/C;aACJ;iBAAM;gBACH,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAM,CAAC;aAC1C;YAED,OAAO;gBACH,IAAI;gBACJ,IAAI,EAAE;oBACF,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;oBACjC,UAAU;iBACb;aACJ,CAAC;SACL;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,IAAI,qBAAY,CAClB,+DAA+D,UAAU,EAAE,EAC3E,IAAI,CAAC,WAAW,CAAC,IAAI,EACrB,GAAG,CACN,CAAC;SACL;IACL,CAAC;IAcS,KAAK,CAAC,mBAAmB;QAC/B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;QACjD,MAAM,aAAa,GAAmB;YAClC,OAAO,EAAE;gBACL,OAAO,EAAE,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC,CAAC,SAAS;gBAC/G,kBAAkB,EAAE,IAAI,CAAC,kBAAkB,CAAC,kBAAkB;aACjE;SACJ,CAAC;QACF,MAAM,MAAM,GAAG,IAAI,eAAM,CAAC,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QACrD,MAAM,cAAc,GAA8B;YAC9C,IAAI,EAAE,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM;YAC/B,MAAM,EAAE,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,WAAW,EAA2B;YAC7E,IAAI,EAAE,IAAI,CAAC,kBAAkB,CAAC,IAAI;YAClC,OAAO,EAAE,IAAI,CAAC,kBAAkB,CAAC,OAAO;SAC3C,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAEpD,OAAO,MAAM,CAAC;IAClB,CAAC;CACJ;AAjFD,kEAiFC"}
|
package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.js
CHANGED
|
@@ -1,13 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
3
|
exports.HTTPRequestProtocolStrategyStreamed = void 0;
|
|
13
4
|
const datasources_1 = require("..");
|
|
@@ -23,12 +14,12 @@ class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy_1.
|
|
|
23
14
|
this.streamTransform = streamTransform;
|
|
24
15
|
return this;
|
|
25
16
|
};
|
|
26
|
-
this.getRawData = () =>
|
|
17
|
+
this.getRawData = async () => {
|
|
27
18
|
if (this.connectionSettings.compression) {
|
|
28
19
|
throw new errors_1.GeneralError("compressed resources are not supported in HTTPRequestProtocolStrategyStreamed yet", this.constructor.name);
|
|
29
20
|
}
|
|
30
21
|
try {
|
|
31
|
-
const result =
|
|
22
|
+
const result = await this.getResponseInternal();
|
|
32
23
|
return {
|
|
33
24
|
data: result.body,
|
|
34
25
|
};
|
|
@@ -36,80 +27,75 @@ class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy_1.
|
|
|
36
27
|
catch (err) {
|
|
37
28
|
throw new errors_1.GeneralError("Error while getting data from server.", this.constructor.name, err);
|
|
38
29
|
}
|
|
39
|
-
}
|
|
30
|
+
};
|
|
40
31
|
}
|
|
41
|
-
getData() {
|
|
42
|
-
const
|
|
43
|
-
|
|
32
|
+
async getData() {
|
|
33
|
+
const dataStream = await super.getData();
|
|
34
|
+
const outStream = new datasources_1.DataSourceStream({
|
|
35
|
+
objectMode: true,
|
|
36
|
+
read: () => {
|
|
37
|
+
return;
|
|
38
|
+
},
|
|
44
39
|
});
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
40
|
+
let dataStreamEnd = false;
|
|
41
|
+
if (this.streamTransform) {
|
|
42
|
+
dataStream.on("data", (data) => {
|
|
43
|
+
this.streamTransform.write(data);
|
|
44
|
+
});
|
|
45
|
+
dataStream.on("end", () => {
|
|
46
|
+
this.streamTransform.end();
|
|
47
|
+
});
|
|
48
|
+
this.streamTransform.on("data", (data) => {
|
|
49
|
+
outStream.push(data);
|
|
50
|
+
});
|
|
51
|
+
this.streamTransform.on("end", () => {
|
|
52
|
+
outStream.push(null);
|
|
53
|
+
});
|
|
54
|
+
this.streamTransform.on("error", (error) => {
|
|
55
|
+
if (error instanceof errors_1.AbstractGolemioError) {
|
|
56
|
+
outStream.emit("error", error);
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
outStream.emit("error", new errors_1.GeneralError("Transformation stream error", this.constructor.name, error));
|
|
60
|
+
}
|
|
61
|
+
});
|
|
62
|
+
dataStream.on("error", (error) => {
|
|
63
|
+
if (error instanceof errors_1.AbstractGolemioError) {
|
|
64
|
+
this.streamTransform.emit("error", error);
|
|
65
|
+
}
|
|
66
|
+
else {
|
|
67
|
+
this.streamTransform.emit("error", new errors_1.GeneralError("Source stream error", this.constructor.name, error));
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
dataStream.on("data", (data) => {
|
|
73
|
+
outStream.push(data);
|
|
52
74
|
});
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
this.streamTransform.write(data);
|
|
57
|
-
});
|
|
58
|
-
dataStream.on("end", () => {
|
|
59
|
-
this.streamTransform.end();
|
|
60
|
-
});
|
|
61
|
-
this.streamTransform.on("data", (data) => {
|
|
62
|
-
outStream.push(data);
|
|
63
|
-
});
|
|
64
|
-
this.streamTransform.on("end", () => {
|
|
75
|
+
dataStream.on("close", () => {
|
|
76
|
+
// some streams do not emit end event
|
|
77
|
+
if (!dataStreamEnd) {
|
|
65
78
|
outStream.push(null);
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
outStream.push(data);
|
|
87
|
-
});
|
|
88
|
-
dataStream.on("close", () => {
|
|
89
|
-
// some streams do not emit end event
|
|
90
|
-
if (!dataStreamEnd) {
|
|
91
|
-
outStream.push(null);
|
|
92
|
-
dataStreamEnd = true;
|
|
93
|
-
}
|
|
94
|
-
});
|
|
95
|
-
dataStream.on("end", () => {
|
|
96
|
-
// some streams do not emit close event
|
|
97
|
-
if (!dataStreamEnd) {
|
|
98
|
-
outStream.push(null);
|
|
99
|
-
dataStreamEnd = true;
|
|
100
|
-
}
|
|
101
|
-
});
|
|
102
|
-
dataStream.on("error", (error) => {
|
|
103
|
-
if (error instanceof errors_1.AbstractGolemioError) {
|
|
104
|
-
outStream.emit("error", error);
|
|
105
|
-
}
|
|
106
|
-
else {
|
|
107
|
-
outStream.emit("error", new errors_1.GeneralError("Source stream error", this.constructor.name, error));
|
|
108
|
-
}
|
|
109
|
-
});
|
|
110
|
-
}
|
|
111
|
-
return outStream;
|
|
112
|
-
});
|
|
79
|
+
dataStreamEnd = true;
|
|
80
|
+
}
|
|
81
|
+
});
|
|
82
|
+
dataStream.on("end", () => {
|
|
83
|
+
// some streams do not emit close event
|
|
84
|
+
if (!dataStreamEnd) {
|
|
85
|
+
outStream.push(null);
|
|
86
|
+
dataStreamEnd = true;
|
|
87
|
+
}
|
|
88
|
+
});
|
|
89
|
+
dataStream.on("error", (error) => {
|
|
90
|
+
if (error instanceof errors_1.AbstractGolemioError) {
|
|
91
|
+
outStream.emit("error", error);
|
|
92
|
+
}
|
|
93
|
+
else {
|
|
94
|
+
outStream.emit("error", new errors_1.GeneralError("Source stream error", this.constructor.name, error));
|
|
95
|
+
}
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
return outStream;
|
|
113
99
|
}
|
|
114
100
|
}
|
|
115
101
|
exports.HTTPRequestProtocolStrategyStreamed = HTTPRequestProtocolStrategyStreamed;
|