@golemio/core 1.18.0 → 1.18.1-dev.1622172793

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/dist/helpers/DateTime.js +4 -5
  2. package/dist/helpers/DateTime.js.map +1 -1
  3. package/dist/helpers/HealthCheck.js +5 -14
  4. package/dist/helpers/HealthCheck.js.map +1 -1
  5. package/dist/helpers/configuration/SimpleConfig.js +4 -1
  6. package/dist/helpers/configuration/SimpleConfig.js.map +1 -1
  7. package/dist/helpers/data-access/amqp/AlternateExchangeCreator.js +4 -13
  8. package/dist/helpers/data-access/amqp/AlternateExchangeCreator.js.map +1 -1
  9. package/dist/helpers/data-access/amqp/AmqpConnector.js +28 -40
  10. package/dist/helpers/data-access/amqp/AmqpConnector.js.map +1 -1
  11. package/dist/helpers/data-access/feature-flags/helpers/AbstractFeatureFlagServiceWrapper.js +2 -13
  12. package/dist/helpers/data-access/feature-flags/helpers/AbstractFeatureFlagServiceWrapper.js.map +1 -1
  13. package/dist/helpers/data-access/feature-flags/providers/NoopFeatureFlagService.js +9 -24
  14. package/dist/helpers/data-access/feature-flags/providers/NoopFeatureFlagService.js.map +1 -1
  15. package/dist/helpers/data-access/postgres/DatabaseConnector.js +23 -25
  16. package/dist/helpers/data-access/postgres/DatabaseConnector.js.map +1 -1
  17. package/dist/helpers/data-access/postgres/repositories/AbstractCachedRepository.js +3 -12
  18. package/dist/helpers/data-access/postgres/repositories/AbstractCachedRepository.js.map +1 -1
  19. package/dist/helpers/data-access/pubsub/subscribers/RedisSubscriber.js +15 -30
  20. package/dist/helpers/data-access/pubsub/subscribers/RedisSubscriber.js.map +1 -1
  21. package/dist/helpers/data-access/redis/IoRedisConnector.js +8 -17
  22. package/dist/helpers/data-access/redis/IoRedisConnector.js.map +1 -1
  23. package/dist/helpers/data-access/storage/AzureBlobStorageService.js +64 -81
  24. package/dist/helpers/data-access/storage/AzureBlobStorageService.js.map +1 -1
  25. package/dist/helpers/data-access/storage/NoopStorageService.js +9 -26
  26. package/dist/helpers/data-access/storage/NoopStorageService.js.map +1 -1
  27. package/dist/helpers/data-access/table-storage/TableStorageServiceFactory.js +1 -2
  28. package/dist/helpers/data-access/table-storage/TableStorageServiceFactory.js.map +1 -1
  29. package/dist/helpers/data-access/table-storage/providers/AzureTableStorageService.js +39 -68
  30. package/dist/helpers/data-access/table-storage/providers/AzureTableStorageService.js.map +1 -1
  31. package/dist/helpers/data-access/table-storage/providers/NoopTableStorageService.js +4 -17
  32. package/dist/helpers/data-access/table-storage/providers/NoopTableStorageService.js.map +1 -1
  33. package/dist/helpers/logger/LoggerProvider.js +33 -14
  34. package/dist/helpers/logger/LoggerProvider.js.map +1 -1
  35. package/dist/helpers/logger/RequestLoggerProvider.js +1 -2
  36. package/dist/helpers/logger/RequestLoggerProvider.js.map +1 -1
  37. package/dist/helpers/routing/AbstractGeoJsonRouter.js +16 -34
  38. package/dist/helpers/routing/AbstractGeoJsonRouter.js.map +1 -1
  39. package/dist/input-gateway/config/InputGatewayConfiguration.js +1 -2
  40. package/dist/input-gateway/config/InputGatewayConfiguration.js.map +1 -1
  41. package/dist/input-gateway/connectors/AMQPConnector.js +52 -68
  42. package/dist/input-gateway/connectors/AMQPConnector.js.map +1 -1
  43. package/dist/input-gateway/controllers/BaseController.js +4 -13
  44. package/dist/input-gateway/controllers/BaseController.js.map +1 -1
  45. package/dist/integration-engine/connectors/AMQPConnector.js +28 -40
  46. package/dist/integration-engine/connectors/AMQPConnector.js.map +1 -1
  47. package/dist/integration-engine/data-access/RawDataStorage.js +7 -16
  48. package/dist/integration-engine/data-access/RawDataStorage.js.map +1 -1
  49. package/dist/integration-engine/data-access/pubsub/RedisPubSubChannel.js +3 -14
  50. package/dist/integration-engine/data-access/pubsub/RedisPubSubChannel.js.map +1 -1
  51. package/dist/integration-engine/data-access/pubsub/subscribers/RedisSubscriber.js +12 -25
  52. package/dist/integration-engine/data-access/pubsub/subscribers/RedisSubscriber.js.map +1 -1
  53. package/dist/integration-engine/datasources/DataSource.js +41 -60
  54. package/dist/integration-engine/datasources/DataSource.js.map +1 -1
  55. package/dist/integration-engine/datasources/DataSourceStream.js +12 -21
  56. package/dist/integration-engine/datasources/DataSourceStream.js.map +1 -1
  57. package/dist/integration-engine/datasources/DataSourceStreamManager.js +4 -13
  58. package/dist/integration-engine/datasources/DataSourceStreamManager.js.map +1 -1
  59. package/dist/integration-engine/datasources/DataSourceStreamed.js +37 -51
  60. package/dist/integration-engine/datasources/DataSourceStreamed.js.map +1 -1
  61. package/dist/integration-engine/datasources/RetryDataSource.js +25 -40
  62. package/dist/integration-engine/datasources/RetryDataSource.js.map +1 -1
  63. package/dist/integration-engine/datasources/datatype-strategy/CSVDataTypeStrategy.js +22 -33
  64. package/dist/integration-engine/datasources/datatype-strategy/CSVDataTypeStrategy.js.map +1 -1
  65. package/dist/integration-engine/datasources/datatype-strategy/JSONDataTypeStrategy.js +14 -25
  66. package/dist/integration-engine/datasources/datatype-strategy/JSONDataTypeStrategy.js.map +1 -1
  67. package/dist/integration-engine/datasources/datatype-strategy/XMLDataTypeStrategy.js +11 -22
  68. package/dist/integration-engine/datasources/datatype-strategy/XMLDataTypeStrategy.js.map +1 -1
  69. package/dist/integration-engine/datasources/datatype-strategy/ZipFileDataTypeStrategy.js +33 -45
  70. package/dist/integration-engine/datasources/datatype-strategy/ZipFileDataTypeStrategy.js.map +1 -1
  71. package/dist/integration-engine/datasources/protocol-strategy/FTPProtocolStrategy.js +17 -26
  72. package/dist/integration-engine/datasources/protocol-strategy/FTPProtocolStrategy.js.map +1 -1
  73. package/dist/integration-engine/datasources/protocol-strategy/GoogleCloudStorageProtocolStrategy.js +10 -19
  74. package/dist/integration-engine/datasources/protocol-strategy/GoogleCloudStorageProtocolStrategy.js.map +1 -1
  75. package/dist/integration-engine/datasources/protocol-strategy/HTTPFetchProtocolStrategy.js +42 -51
  76. package/dist/integration-engine/datasources/protocol-strategy/HTTPFetchProtocolStrategy.js.map +1 -1
  77. package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategy.js +49 -62
  78. package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategy.js.map +1 -1
  79. package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.js +67 -81
  80. package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.js.map +1 -1
  81. package/dist/integration-engine/datasources/protocol-strategy/PaginatedHTTPProtocolStrategy.js +29 -38
  82. package/dist/integration-engine/datasources/protocol-strategy/PaginatedHTTPProtocolStrategy.js.map +1 -1
  83. package/dist/integration-engine/datasources/protocol-strategy/PostgresProtocolStrategy.js +37 -42
  84. package/dist/integration-engine/datasources/protocol-strategy/PostgresProtocolStrategy.js.map +1 -1
  85. package/dist/integration-engine/datasources/protocol-strategy/PostgresProtocolStrategyStreamed.js +29 -32
  86. package/dist/integration-engine/datasources/protocol-strategy/PostgresProtocolStrategyStreamed.js.map +1 -1
  87. package/dist/integration-engine/datasources/protocol-strategy/ProtocolStrategy.js +73 -88
  88. package/dist/integration-engine/datasources/protocol-strategy/ProtocolStrategy.js.map +1 -1
  89. package/dist/integration-engine/datasources/protocol-strategy/SFTPProtocolStrategy.js +17 -26
  90. package/dist/integration-engine/datasources/protocol-strategy/SFTPProtocolStrategy.js.map +1 -1
  91. package/dist/integration-engine/datasources/protocol-strategy/helpers/FTPFileProcessor.js +37 -49
  92. package/dist/integration-engine/datasources/protocol-strategy/helpers/FTPFileProcessor.js.map +1 -1
  93. package/dist/integration-engine/helpers/FileCompressor.js +59 -76
  94. package/dist/integration-engine/helpers/FileCompressor.js.map +1 -1
  95. package/dist/integration-engine/helpers/GeocodeApi.js +20 -30
  96. package/dist/integration-engine/helpers/GeocodeApi.js.map +1 -1
  97. package/dist/integration-engine/helpers/IntegrationErrorHandler.js +3 -3
  98. package/dist/integration-engine/helpers/IntegrationErrorHandler.js.map +1 -1
  99. package/dist/integration-engine/helpers/PaginationHelper.js +1 -2
  100. package/dist/integration-engine/helpers/PaginationHelper.js.map +1 -1
  101. package/dist/integration-engine/models/PostgresModel.js +70 -75
  102. package/dist/integration-engine/models/PostgresModel.js.map +1 -1
  103. package/dist/integration-engine/models/RedisModel.js +28 -39
  104. package/dist/integration-engine/models/RedisModel.js.map +1 -1
  105. package/dist/integration-engine/queueprocessors/QueueProcessor.js +23 -28
  106. package/dist/integration-engine/queueprocessors/QueueProcessor.js.map +1 -1
  107. package/dist/integration-engine/transformations/BaseTransformation.js +8 -17
  108. package/dist/integration-engine/transformations/BaseTransformation.js.map +1 -1
  109. package/dist/integration-engine/workers/AbstractTask.js +7 -18
  110. package/dist/integration-engine/workers/AbstractTask.js.map +1 -1
  111. package/dist/integration-engine/workers/AbstractTaskJsonSchema.js +25 -38
  112. package/dist/integration-engine/workers/AbstractTaskJsonSchema.js.map +1 -1
  113. package/dist/integration-engine/workers/AbstractTaskWithoutValidation.js +2 -13
  114. package/dist/integration-engine/workers/AbstractTaskWithoutValidation.js.map +1 -1
  115. package/dist/integration-engine/workers/BaseWorker.js +4 -13
  116. package/dist/integration-engine/workers/BaseWorker.js.map +1 -1
  117. package/dist/integration-engine/workers/helpers/MessageDataParser.js +1 -2
  118. package/dist/integration-engine/workers/helpers/MessageDataParser.js.map +1 -1
  119. package/dist/integration-engine/workers/helpers/MessageDataValidator.js +18 -29
  120. package/dist/integration-engine/workers/helpers/MessageDataValidator.js.map +1 -1
  121. package/dist/integration-engine/workers/helpers/WebhookDecorators.js +1 -10
  122. package/dist/integration-engine/workers/helpers/WebhookDecorators.js.map +1 -1
  123. package/dist/integration-engine/workers/webhooks/CloudflareCachePurgeWebhook.js +13 -24
  124. package/dist/integration-engine/workers/webhooks/CloudflareCachePurgeWebhook.js.map +1 -1
  125. package/dist/monitoring/metrics/metrics-builder.js +21 -17
  126. package/dist/monitoring/metrics/metrics-builder.js.map +1 -1
  127. package/dist/monitoring/metrics/metrics-service.js +4 -14
  128. package/dist/monitoring/metrics/metrics-service.js.map +1 -1
  129. package/dist/monitoring/opentelemetry/instrumentation/configs/amqplib.js +1 -2
  130. package/dist/monitoring/opentelemetry/instrumentation/configs/amqplib.js.map +1 -1
  131. package/dist/monitoring/opentelemetry/instrumentation/configs/aws-sdk.js +1 -1
  132. package/dist/monitoring/opentelemetry/instrumentation/configs/aws-sdk.js.map +1 -1
  133. package/dist/monitoring/opentelemetry/trace-provider.js +8 -2
  134. package/dist/monitoring/opentelemetry/trace-provider.js.map +1 -1
  135. package/dist/monitoring/sentry/sentry-provider.js +1 -1
  136. package/dist/monitoring/sentry/sentry-provider.js.map +1 -1
  137. package/dist/monitoring/sentry/sentry-transactions.js +9 -19
  138. package/dist/monitoring/sentry/sentry-transactions.js.map +1 -1
  139. package/dist/output-gateway/Geo.js +3 -24
  140. package/dist/output-gateway/Geo.js.map +1 -1
  141. package/dist/output-gateway/config/OutputGatewayConfiguration.js +1 -2
  142. package/dist/output-gateway/config/OutputGatewayConfiguration.js.map +1 -1
  143. package/dist/output-gateway/database/index.js +3 -12
  144. package/dist/output-gateway/database/index.js.map +1 -1
  145. package/dist/output-gateway/ioc/Di.js +1 -10
  146. package/dist/output-gateway/ioc/Di.js.map +1 -1
  147. package/dist/output-gateway/models/SequelizeModel.js +4 -2
  148. package/dist/output-gateway/models/SequelizeModel.js.map +1 -1
  149. package/dist/output-gateway/routes/GeoJsonRouter.js +14 -32
  150. package/dist/output-gateway/routes/GeoJsonRouter.js.map +1 -1
  151. package/package.json +6 -5
  152. package/tsconfig.base.json +2 -2
@@ -1,13 +1,4 @@
1
1
  "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
2
  Object.defineProperty(exports, "__esModule", { value: true });
12
3
  exports.DataSourceStreamed = void 0;
13
4
  const logger_1 = require("../../helpers/logger");
@@ -19,13 +10,13 @@ class DataSourceStreamed extends datasources_1.DataSource {
19
10
  constructor() {
20
11
  super(...arguments);
21
12
  this.dataBuffer = [];
22
- this.proceed = () => __awaiter(this, void 0, void 0, function* () {
13
+ this.proceed = async () => {
23
14
  return this.dataStream.proceed();
24
- });
15
+ };
25
16
  /**
26
17
  * @param {boolean} useDataBuffer data is buffered and sent to output stream in `config.DATA_BATCH_SIZE` batches
27
18
  */
28
- this.getOutputStream = (useDataBuffer = false) => __awaiter(this, void 0, void 0, function* () {
19
+ this.getOutputStream = async (useDataBuffer = false) => {
29
20
  let inputStreamEndedAttempts = 0;
30
21
  this.dataStream = new datasources_1.DataSourceStream({
31
22
  objectMode: true,
@@ -33,7 +24,7 @@ class DataSourceStreamed extends datasources_1.DataSource {
33
24
  return;
34
25
  },
35
26
  });
36
- const inputStream = yield this.protocolStrategy.getData();
27
+ const inputStream = await this.protocolStrategy.getData();
37
28
  inputStream.on("error", (error) => {
38
29
  if (error instanceof errors_1.AbstractGolemioError) {
39
30
  this.dataStream.emit("error", error);
@@ -42,20 +33,20 @@ class DataSourceStreamed extends datasources_1.DataSource {
42
33
  this.dataStream.emit("error", new errors_1.GeneralError("DataSourceStreamed general error", this.constructor.name, error));
43
34
  }
44
35
  });
45
- inputStream.onDataListeners.push((data) => __awaiter(this, void 0, void 0, function* () {
36
+ inputStream.onDataListeners.push(async (data) => {
46
37
  inputStream.pause();
47
38
  if (useDataBuffer) {
48
39
  this.dataBuffer.push(data);
49
- yield this.processData();
40
+ await this.processData();
50
41
  }
51
42
  else {
52
- yield this.processData(false, data);
43
+ await this.processData(false, data);
53
44
  }
54
45
  inputStream.resume();
55
- }));
56
- inputStream.on("end", () => __awaiter(this, void 0, void 0, function* () {
46
+ });
47
+ inputStream.on("end", async () => {
57
48
  if (useDataBuffer) {
58
- yield this.processData(true);
49
+ await this.processData(true);
59
50
  }
60
51
  if (!inputStream.isPaused()) {
61
52
  this.dataStream.push(null);
@@ -75,17 +66,16 @@ class DataSourceStreamed extends datasources_1.DataSource {
75
66
  }
76
67
  }, config_1.config.stream.wait_for_end_interval);
77
68
  }
78
- }));
69
+ });
79
70
  inputStream.proceed().catch((error) => this.dataStream.emit("error", error));
80
71
  return this.dataStream;
81
- });
82
- this.processData = (force = false, data = null) => __awaiter(this, void 0, void 0, function* () {
83
- var _a;
72
+ };
73
+ this.processData = async (force = false, data = null) => {
84
74
  if (this.dataBuffer.length >= config_1.config.DATA_BATCH_SIZE || force || data) {
85
75
  try {
86
76
  let content;
87
- if ((_a = this.dataTypeStrategy) === null || _a === void 0 ? void 0 : _a.parseData) {
88
- content = yield this.dataTypeStrategy.parseData(data || this.dataBuffer);
77
+ if (this.dataTypeStrategy?.parseData) {
78
+ content = await this.dataTypeStrategy.parseData(data || this.dataBuffer);
89
79
  }
90
80
  else {
91
81
  content = data || this.dataBuffer;
@@ -106,37 +96,33 @@ class DataSourceStreamed extends datasources_1.DataSource {
106
96
  this.dataStream.emit("error", new errors_1.GeneralError("Retrieving of the source data failed.", this.name, err));
107
97
  }
108
98
  }
109
- });
99
+ };
110
100
  }
111
- getAll(useDataBuffer = false) {
112
- return __awaiter(this, void 0, void 0, function* () {
113
- const { data } = yield this.getAllWithMetadata(useDataBuffer);
114
- return data;
115
- });
101
+ async getAll(useDataBuffer = false) {
102
+ const { data } = await this.getAllWithMetadata(useDataBuffer);
103
+ return data;
116
104
  }
117
- getAllWithMetadata(useDataBuffer = false) {
118
- return __awaiter(this, void 0, void 0, function* () {
119
- this.dataStream = yield this.getOutputStream(useDataBuffer);
120
- this.dataStream.on("end", () => {
121
- this.dataStream.destroy();
122
- });
123
- this.dataStream.onDataListeners.push((data) => __awaiter(this, void 0, void 0, function* () {
124
- if (this.validator) {
125
- try {
126
- this.dataStream.pause();
127
- yield this.validator.Validate(data);
128
- this.dataStream.resume();
129
- }
130
- catch (err) {
131
- this.dataStream.destroy(new errors_1.GeneralError("Error while validating source data.", this.name, err));
132
- }
105
+ async getAllWithMetadata(useDataBuffer = false) {
106
+ this.dataStream = await this.getOutputStream(useDataBuffer);
107
+ this.dataStream.on("end", () => {
108
+ this.dataStream.destroy();
109
+ });
110
+ this.dataStream.onDataListeners.push(async (data) => {
111
+ if (this.validator) {
112
+ try {
113
+ this.dataStream.pause();
114
+ await this.validator.Validate(data);
115
+ this.dataStream.resume();
133
116
  }
134
- else {
135
- helpers_1.log.warn("DataSource validator is not set.");
117
+ catch (err) {
118
+ this.dataStream.destroy(new errors_1.GeneralError("Error while validating source data.", this.name, err));
136
119
  }
137
- }));
138
- return { data: this.dataStream, metadata: undefined };
120
+ }
121
+ else {
122
+ helpers_1.log.warn("DataSource validator is not set.");
123
+ }
139
124
  });
125
+ return { data: this.dataStream, metadata: undefined };
140
126
  }
141
127
  }
142
128
  exports.DataSourceStreamed = DataSourceStreamed;
@@ -1 +1 @@
1
- {"version":3,"file":"DataSourceStreamed.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/DataSourceStreamed.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,iDAAkD;AAClD,sCAAoC;AACpC,oCAA4E;AAC5E,wCAAkC;AAClC,4CAAqE;AAErE,MAAa,kBAAmB,SAAQ,wBAAU;IAAlD;;QACY,eAAU,GAAU,EAAE,CAAC;QAExB,YAAO,GAAG,GAAwB,EAAE;YACvC,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QACrC,CAAC,CAAA,CAAC;QA+BF;;WAEG;QACO,oBAAe,GAAG,CAAO,gBAAyB,KAAK,EAA6B,EAAE;YAC5F,IAAI,wBAAwB,GAAG,CAAC,CAAC;YACjC,IAAI,CAAC,UAAU,GAAG,IAAI,8BAAgB,CAAC;gBACnC,UAAU,EAAE,IAAI;gBAChB,IAAI,EAAE,GAAG,EAAE;oBACP,OAAO;gBACX,CAAC;aACJ,CAAC,CAAC;YAEH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC;YAE1D,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBACnC,IAAI,KAAK,YAAY,6BAAoB,EAAE;oBACvC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;iBACxC;qBAAM;oBACH,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,kCAAkC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;iBACrH;YACL,CAAC,CAAC,CAAC;YAEH,WAAW,CAAC,eAAe,CAAC,IAAI,CAAC,CAAO,IAAS,EAAiB,EAAE;gBAChE,WAAW,CAAC,KAAK,EAAE,CAAC;gBAEpB,IAAI,aAAa,EAAE;oBACf,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC3B,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;iBAC5B;qBAAM;oBACH,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;iBACvC;gBAED,WAAW,CAAC,MAAM,EAAE,CAAC;YACzB,CAAC,CAAA,CAAC,CAAC;YAEH,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,GAAwB,EAAE;gBAC5C,IAAI,aAAa,EAAE;oBACf,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;iBAChC;gBAED,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE;oBACzB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBAC9B;qBAAM;oBACH,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE;wBAC7B,wBAAwB,EAAE,CAAC;wBAC3B,kCAAkC;wBAClC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE;4BACzB,aAAa,CAAC,OAAO,CAAC,CAAC;4BACvB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;yBAC9B;6BAAM,IAAI,wBAAwB,GAAG,eAAM,CAAC,MAAM,CAAC,qBAAqB,EAAE;4BACvE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,kCAAkC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;4BAC/F,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;4BAC3B,aAAa,CAAC,OAAO,CAAC,CAAC;yBAC1B;oBACL,CAAC,EAAE,eAAM,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;iBAC3C;YACL,CAAC,CAAA,CAAC,CAAC;YAEH,WAAW,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAEpF,OAAO,IAAI,CAAC,UAAU,CAAC;QAC3B,CAAC,CAAA,CAAC;QAEM,gBAAW,GAAG,CAAO,KAAK,GAAG,KAAK,EAAE,IAAI,GAAG,IAAI,EAAiB,EAAE;;YACtE,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,eAAM,CAAC,eAAe,IAAI,KAAK,IAAI,IAAI,EAAE;gBACnE,IAAI;oBACA,IAAI,OAAY,CAAC;oBAEjB,IAAI,MAAA,IAAI,CAAC,gBAAgB,0CAAE,SAAS,EAAE;wBAClC,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC;qBAC5E;yBAAM;wBACH,OAAO,GAAG,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC;qBACrC;oBAED,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;wBACvB,aAAG,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,IAAI,oCAAoC,CAAC,CAAC;wBAC/D,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,wBAAe,CAAC,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC,CAAC;qBACrG;yBAAM;wBACH,MAAM,eAAe,GAAG,OAAO,YAAY,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;wBACtE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,wBAAe,CAAC,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;wBAE/F,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;wBAC9B,mBAAmB;wBACnB,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;qBAC9B;iBACJ;gBAAC,OAAO,GAAG,EAAE;oBACV,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,uCAAuC,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;iBAC5G;aACJ;QACL,CAAC,CAAA,CAAC;IACN,CAAC;IAvHyB,MAAM,CAAC,aAAa,GAAG,KAAK;;YAC9C,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC;YAC9D,OAAO,IAAI,CAAC;QAChB,CAAC;KAAA;IAEqB,kBAAkB,CAAC,aAAa,GAAG,KAAK;;YAC1D,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC;YAE5D,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBAC3B,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;YAC9B,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAO,IAAS,EAAE,EAAE;gBACrD,IAAI,IAAI,CAAC,SAAS,EAAE;oBAChB,IAAI;wBACA,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;wBACxB,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;wBACpC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;qBAC5B;oBAAC,OAAO,GAAG,EAAE;wBACV,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,qBAAY,CAAC,qCAAqC,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;qBACpG;iBACJ;qBAAM;oBACH,aAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;iBAChD;YACL,CAAC,CAAA,CAAC,CAAC;YAEH,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;QAC1D,CAAC;KAAA;CA4FJ;AA9HD,gDA8HC"}
1
+ {"version":3,"file":"DataSourceStreamed.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/DataSourceStreamed.ts"],"names":[],"mappings":";;;AAAA,iDAAkD;AAClD,sCAAoC;AACpC,oCAA4E;AAC5E,wCAAkC;AAClC,4CAAqE;AAErE,MAAa,kBAAmB,SAAQ,wBAAU;IAAlD;;QACY,eAAU,GAAU,EAAE,CAAC;QAExB,YAAO,GAAG,KAAK,IAAmB,EAAE;YACvC,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QACrC,CAAC,CAAC;QA+BF;;WAEG;QACO,oBAAe,GAAG,KAAK,EAAE,gBAAyB,KAAK,EAA6B,EAAE;YAC5F,IAAI,wBAAwB,GAAG,CAAC,CAAC;YACjC,IAAI,CAAC,UAAU,GAAG,IAAI,8BAAgB,CAAC;gBACnC,UAAU,EAAE,IAAI;gBAChB,IAAI,EAAE,GAAG,EAAE;oBACP,OAAO;gBACX,CAAC;aACJ,CAAC,CAAC;YAEH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC;YAE1D,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBACnC,IAAI,KAAK,YAAY,6BAAoB,EAAE;oBACvC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;iBACxC;qBAAM;oBACH,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,kCAAkC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;iBACrH;YACL,CAAC,CAAC,CAAC;YAEH,WAAW,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,IAAS,EAAiB,EAAE;gBAChE,WAAW,CAAC,KAAK,EAAE,CAAC;gBAEpB,IAAI,aAAa,EAAE;oBACf,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC3B,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;iBAC5B;qBAAM;oBACH,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;iBACvC;gBAED,WAAW,CAAC,MAAM,EAAE,CAAC;YACzB,CAAC,CAAC,CAAC;YAEH,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,IAAmB,EAAE;gBAC5C,IAAI,aAAa,EAAE;oBACf,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;iBAChC;gBAED,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE;oBACzB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBAC9B;qBAAM;oBACH,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE;wBAC7B,wBAAwB,EAAE,CAAC;wBAC3B,kCAAkC;wBAClC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE;4BACzB,aAAa,CAAC,OAAO,CAAC,CAAC;4BACvB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;yBAC9B;6BAAM,IAAI,wBAAwB,GAAG,eAAM,CAAC,MAAM,CAAC,qBAAqB,EAAE;4BACvE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,kCAAkC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;4BAC/F,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;4BAC3B,aAAa,CAAC,OAAO,CAAC,CAAC;yBAC1B;oBACL,CAAC,EAAE,eAAM,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;iBAC3C;YACL,CAAC,CAAC,CAAC;YAEH,WAAW,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAEpF,OAAO,IAAI,CAAC,UAAU,CAAC;QAC3B,CAAC,CAAC;QAEM,gBAAW,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE,IAAI,GAAG,IAAI,EAAiB,EAAE;YACtE,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,eAAM,CAAC,eAAe,IAAI,KAAK,IAAI,IAAI,EAAE;gBACnE,IAAI;oBACA,IAAI,OAAY,CAAC;oBAEjB,IAAI,IAAI,CAAC,gBAAgB,EAAE,SAAS,EAAE;wBAClC,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC;qBAC5E;yBAAM;wBACH,OAAO,GAAG,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC;qBACrC;oBAED,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;wBACvB,aAAG,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,IAAI,oCAAoC,CAAC,CAAC;wBAC/D,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,wBAAe,CAAC,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC,CAAC;qBACrG;yBAAM;wBACH,MAAM,eAAe,GAAG,OAAO,YAAY,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;wBACtE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,wBAAe,CAAC,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;wBAE/F,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;wBAC9B,mBAAmB;wBACnB,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;qBAC9B;iBACJ;gBAAC,OAAO,GAAG,EAAE;oBACV,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,uCAAuC,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;iBAC5G;aACJ;QACL,CAAC,CAAC;IACN,CAAC;IAvHmB,KAAK,CAAC,MAAM,CAAC,aAAa,GAAG,KAAK;QAC9C,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAC9D,OAAO,IAAI,CAAC;IAChB,CAAC;IAEe,KAAK,CAAC,kBAAkB,CAAC,aAAa,GAAG,KAAK;QAC1D,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC;QAE5D,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YAC3B,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QAC9B,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,IAAS,EAAE,EAAE;YACrD,IAAI,IAAI,CAAC,SAAS,EAAE;gBAChB,IAAI;oBACA,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;oBACxB,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;oBACpC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;iBAC5B;gBAAC,OAAO,GAAG,EAAE;oBACV,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,qBAAY,CAAC,qCAAqC,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;iBACpG;aACJ;iBAAM;gBACH,aAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;aAChD;QACL,CAAC,CAAC,CAAC;QAEH,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;IAC1D,CAAC;CA4FJ;AA9HD,gDA8HC"}
@@ -1,13 +1,4 @@
1
1
  "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
2
  Object.defineProperty(exports, "__esModule", { value: true });
12
3
  exports.RetryDataSource = void 0;
13
4
  const promises_1 = require("timers/promises");
@@ -17,41 +8,35 @@ class RetryDataSource {
17
8
  this.datasource = new DataSource_1.DataSource(name, protocolStrategy, dataTypeStrategy, validator);
18
9
  this.retryCount = retryCount;
19
10
  }
20
- getAll() {
21
- return __awaiter(this, void 0, void 0, function* () {
22
- try {
23
- return yield this.datasource.getAll();
11
+ async getAll() {
12
+ try {
13
+ return await this.datasource.getAll();
14
+ }
15
+ catch (err) {
16
+ if (this.retryCount > 0) {
17
+ this.retryCount--;
18
+ await this.sleep();
19
+ return await this.getAll();
24
20
  }
25
- catch (err) {
26
- if (this.retryCount > 0) {
27
- this.retryCount--;
28
- yield this.sleep();
29
- return yield this.getAll();
30
- }
31
- throw err;
32
- }
33
- });
21
+ throw err;
22
+ }
34
23
  }
35
- getLastModified() {
36
- return __awaiter(this, void 0, void 0, function* () {
37
- try {
38
- return yield this.datasource.getLastModified();
39
- }
40
- catch (err) {
41
- if (this.retryCount > 0) {
42
- this.retryCount--;
43
- yield this.sleep();
44
- return yield this.getLastModified();
45
- }
46
- throw err;
24
+ async getLastModified() {
25
+ try {
26
+ return await this.datasource.getLastModified();
27
+ }
28
+ catch (err) {
29
+ if (this.retryCount > 0) {
30
+ this.retryCount--;
31
+ await this.sleep();
32
+ return await this.getLastModified();
47
33
  }
48
- });
34
+ throw err;
35
+ }
49
36
  }
50
- sleep() {
51
- return __awaiter(this, void 0, void 0, function* () {
52
- // sleep random time between 5 seconds to 60 seconds
53
- yield (0, promises_1.setTimeout)(Math.random() * 55000 + 5000);
54
- });
37
+ async sleep() {
38
+ // sleep random time between 5 seconds to 60 seconds
39
+ await (0, promises_1.setTimeout)(Math.random() * 55000 + 5000);
55
40
  }
56
41
  }
57
42
  exports.RetryDataSource = RetryDataSource;
@@ -1 +1 @@
1
- {"version":3,"file":"RetryDataSource.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/RetryDataSource.ts"],"names":[],"mappings":";;;;;;;;;;;;AACA,8CAAsD;AACtD,6CAA0C;AAI1C,MAAa,eAAe;IAIxB,YACI,IAAY,EACZ,gBAAmC,EACnC,mBAA6C,IAAI,EACjD,YAA+B,IAAI,EACnC,aAAqB,CAAC;QAEtB,IAAI,CAAC,UAAU,GAAG,IAAI,uBAAU,CAAI,IAAI,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,SAAS,CAAC,CAAC;QACzF,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IACjC,CAAC;IAEY,MAAM;;YACf,IAAI;gBACA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;aACzC;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE;oBACrB,IAAI,CAAC,UAAU,EAAE,CAAC;oBAClB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;oBACnB,OAAO,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;iBAC9B;gBACD,MAAM,GAAG,CAAC;aACb;QACL,CAAC;KAAA;IAEY,eAAe;;YACxB,IAAI;gBACA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,eAAe,EAAE,CAAC;aAClD;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE;oBACrB,IAAI,CAAC,UAAU,EAAE,CAAC;oBAClB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;oBACnB,OAAO,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;iBACvC;gBACD,MAAM,GAAG,CAAC;aACb;QACL,CAAC;KAAA;IAEa,KAAK;;YACf,oDAAoD;YACpD,MAAM,IAAA,qBAAK,EAAC,IAAI,CAAC,MAAM,EAAE,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC;QAC9C,CAAC;KAAA;CACJ;AA7CD,0CA6CC"}
1
+ {"version":3,"file":"RetryDataSource.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/RetryDataSource.ts"],"names":[],"mappings":";;;AACA,8CAAsD;AACtD,6CAA0C;AAI1C,MAAa,eAAe;IAIxB,YACI,IAAY,EACZ,gBAAmC,EACnC,mBAA6C,IAAI,EACjD,YAA+B,IAAI,EACnC,aAAqB,CAAC;QAEtB,IAAI,CAAC,UAAU,GAAG,IAAI,uBAAU,CAAI,IAAI,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,SAAS,CAAC,CAAC;QACzF,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IACjC,CAAC;IAEM,KAAK,CAAC,MAAM;QACf,IAAI;YACA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;SACzC;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE;gBACrB,IAAI,CAAC,UAAU,EAAE,CAAC;gBAClB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACnB,OAAO,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;aAC9B;YACD,MAAM,GAAG,CAAC;SACb;IACL,CAAC;IAEM,KAAK,CAAC,eAAe;QACxB,IAAI;YACA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,eAAe,EAAE,CAAC;SAClD;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE;gBACrB,IAAI,CAAC,UAAU,EAAE,CAAC;gBAClB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACnB,OAAO,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;aACvC;YACD,MAAM,GAAG,CAAC;SACb;IACL,CAAC;IAEO,KAAK,CAAC,KAAK;QACf,oDAAoD;QACpD,MAAM,IAAA,qBAAK,EAAC,IAAI,CAAC,MAAM,EAAE,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC;IAC9C,CAAC;CACJ;AA7CD,0CA6CC"}
@@ -1,13 +1,4 @@
1
1
  "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
2
  Object.defineProperty(exports, "__esModule", { value: true });
12
3
  exports.CSVDataTypeStrategy = void 0;
13
4
  const errors_1 = require("@golemio/errors");
@@ -24,30 +15,28 @@ class CSVDataTypeStrategy {
24
15
  setFilter(filterFunction) {
25
16
  this.filter = filterFunction;
26
17
  }
27
- parseData(data) {
28
- return __awaiter(this, void 0, void 0, function* () {
29
- const readable = new stream_1.Readable();
30
- readable._read = () => {
31
- // _read is required but you can noop it
32
- };
33
- readable.push(data);
34
- readable.push(null);
35
- return new Promise((resolve, reject) => {
36
- let resultsArray = [];
37
- readable
38
- .pipe((0, fast_csv_1.parse)(this.dataTypeSettings.fastcsvParams))
39
- .on("error", (error) => {
40
- reject(new errors_1.GeneralError("Error while parsing source data.", this.constructor.name, error));
41
- })
42
- .on("data", (row) => {
43
- resultsArray.push(this.dataTypeSettings.subscribe(row));
44
- })
45
- .on("end", (rowCount) => {
46
- if (this.filter) {
47
- resultsArray = resultsArray.filter(this.filter);
48
- }
49
- return resolve(resultsArray);
50
- });
18
+ async parseData(data) {
19
+ const readable = new stream_1.Readable();
20
+ readable._read = () => {
21
+ // _read is required but you can noop it
22
+ };
23
+ readable.push(data);
24
+ readable.push(null);
25
+ return new Promise((resolve, reject) => {
26
+ let resultsArray = [];
27
+ readable
28
+ .pipe((0, fast_csv_1.parse)(this.dataTypeSettings.fastcsvParams))
29
+ .on("error", (error) => {
30
+ reject(new errors_1.GeneralError("Error while parsing source data.", this.constructor.name, error));
31
+ })
32
+ .on("data", (row) => {
33
+ resultsArray.push(this.dataTypeSettings.subscribe(row));
34
+ })
35
+ .on("end", (rowCount) => {
36
+ if (this.filter) {
37
+ resultsArray = resultsArray.filter(this.filter);
38
+ }
39
+ return resolve(resultsArray);
51
40
  });
52
41
  });
53
42
  }
@@ -1 +1 @@
1
- {"version":3,"file":"CSVDataTypeStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/CSVDataTypeStrategy.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,4CAA+C;AAC/C,uCAAiD;AACjD,mCAAkC;AAGlC,MAAa,mBAAmB;IAI5B,YAAY,QAAsB;QAC9B,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAC;QACjC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;IAC5B,CAAC;IAEM,mBAAmB,CAAC,QAAsB;QAC7C,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAC;IACrC,CAAC;IAEM,SAAS,CAAC,cAAkC;QAC/C,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC;IACjC,CAAC;IAEY,SAAS,CAAC,IAAS;;YAC5B,MAAM,QAAQ,GAAG,IAAI,iBAAQ,EAAE,CAAC;YAChC,QAAQ,CAAC,KAAK,GAAG,GAAG,EAAE;gBAClB,wCAAwC;YAC5C,CAAC,CAAC;YACF,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEpB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACnC,IAAI,YAAY,GAAU,EAAE,CAAC;gBAC7B,QAAQ;qBACH,IAAI,CAAC,IAAA,gBAAY,EAAC,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;qBACvD,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;oBACnB,MAAM,CAAC,IAAI,qBAAY,CAAC,kCAAkC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC/F,CAAC,CAAC;qBACD,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE;oBAChB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC5D,CAAC,CAAC;qBACD,EAAE,CAAC,KAAK,EAAE,CAAC,QAAgB,EAAE,EAAE;oBAC5B,IAAI,IAAI,CAAC,MAAM,EAAE;wBACb,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;qBACnD;oBACD,OAAO,OAAO,CAAC,YAAY,CAAC,CAAC;gBACjC,CAAC,CAAC,CAAC;YACX,CAAC,CAAC,CAAC;QACP,CAAC;KAAA;CACJ;AA3CD,kDA2CC"}
1
+ {"version":3,"file":"CSVDataTypeStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/CSVDataTypeStrategy.ts"],"names":[],"mappings":";;;AAAA,4CAA+C;AAC/C,uCAAiD;AACjD,mCAAkC;AAGlC,MAAa,mBAAmB;IAI5B,YAAY,QAAsB;QAC9B,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAC;QACjC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;IAC5B,CAAC;IAEM,mBAAmB,CAAC,QAAsB;QAC7C,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAC;IACrC,CAAC;IAEM,SAAS,CAAC,cAAkC;QAC/C,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC;IACjC,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,IAAS;QAC5B,MAAM,QAAQ,GAAG,IAAI,iBAAQ,EAAE,CAAC;QAChC,QAAQ,CAAC,KAAK,GAAG,GAAG,EAAE;YAClB,wCAAwC;QAC5C,CAAC,CAAC;QACF,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,YAAY,GAAU,EAAE,CAAC;YAC7B,QAAQ;iBACH,IAAI,CAAC,IAAA,gBAAY,EAAC,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;iBACvD,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBACnB,MAAM,CAAC,IAAI,qBAAY,CAAC,kCAAkC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;YAC/F,CAAC,CAAC;iBACD,EAAE,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE;gBAChB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;YAC5D,CAAC,CAAC;iBACD,EAAE,CAAC,KAAK,EAAE,CAAC,QAAgB,EAAE,EAAE;gBAC5B,IAAI,IAAI,CAAC,MAAM,EAAE;oBACb,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;iBACnD;gBACD,OAAO,OAAO,CAAC,YAAY,CAAC,CAAC;YACjC,CAAC,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;IACP,CAAC;CACJ;AA3CD,kDA2CC"}
@@ -1,13 +1,4 @@
1
1
  "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
2
  Object.defineProperty(exports, "__esModule", { value: true });
12
3
  exports.JSONDataTypeStrategy = void 0;
13
4
  const utils_1 = require("../../../helpers/utils");
@@ -23,24 +14,22 @@ class JSONDataTypeStrategy {
23
14
  setFilter(filterFunction) {
24
15
  this.filter = filterFunction;
25
16
  }
26
- parseData(data) {
27
- return __awaiter(this, void 0, void 0, function* () {
28
- try {
29
- if (typeof data === "string") {
30
- data = JSON.parse(data);
31
- }
32
- if (this.resultsPath) {
33
- data = (0, utils_1.getSubProperty)(this.resultsPath, data);
34
- }
35
- if (this.filter) {
36
- data = data.filter(this.filter);
37
- }
38
- return data;
17
+ async parseData(data) {
18
+ try {
19
+ if (typeof data === "string") {
20
+ data = JSON.parse(data);
39
21
  }
40
- catch (err) {
41
- throw new errors_1.GeneralError("Error while parsing source data.", this.constructor.name, err);
22
+ if (this.resultsPath) {
23
+ data = (0, utils_1.getSubProperty)(this.resultsPath, data);
42
24
  }
43
- });
25
+ if (this.filter) {
26
+ data = data.filter(this.filter);
27
+ }
28
+ return data;
29
+ }
30
+ catch (err) {
31
+ throw new errors_1.GeneralError("Error while parsing source data.", this.constructor.name, err);
32
+ }
44
33
  }
45
34
  }
46
35
  exports.JSONDataTypeStrategy = JSONDataTypeStrategy;
@@ -1 +1 @@
1
- {"version":3,"file":"JSONDataTypeStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/JSONDataTypeStrategy.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,kDAAgD;AAEhD,4CAA+C;AAE/C,MAAa,oBAAoB;IAI7B,YAAY,QAAuB;QAC/B,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC;QACxC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;IAC5B,CAAC;IAEM,mBAAmB,CAAC,QAAuB;QAC9C,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC;IAC5C,CAAC;IAEM,SAAS,CAAC,cAAkC;QAC/C,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC;IACjC,CAAC;IAEY,SAAS,CAAC,IAAS;;YAC5B,IAAI;gBACA,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;oBAC1B,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;iBAC3B;gBAED,IAAI,IAAI,CAAC,WAAW,EAAE;oBAClB,IAAI,GAAG,IAAA,sBAAc,EAAM,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;iBACtD;gBAED,IAAI,IAAI,CAAC,MAAM,EAAE;oBACb,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;iBACnC;gBACD,OAAO,IAAI,CAAC;aACf;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,qBAAY,CAAC,kCAAkC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aAC1F;QACL,CAAC;KAAA;CACJ;AAnCD,oDAmCC"}
1
+ {"version":3,"file":"JSONDataTypeStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/JSONDataTypeStrategy.ts"],"names":[],"mappings":";;;AAAA,kDAAgD;AAEhD,4CAA+C;AAE/C,MAAa,oBAAoB;IAI7B,YAAY,QAAuB;QAC/B,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC;QACxC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;IAC5B,CAAC;IAEM,mBAAmB,CAAC,QAAuB;QAC9C,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC;IAC5C,CAAC;IAEM,SAAS,CAAC,cAAkC;QAC/C,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC;IACjC,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,IAAS;QAC5B,IAAI;YACA,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;gBAC1B,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;aAC3B;YAED,IAAI,IAAI,CAAC,WAAW,EAAE;gBAClB,IAAI,GAAG,IAAA,sBAAc,EAAM,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;aACtD;YAED,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aACnC;YACD,OAAO,IAAI,CAAC;SACf;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,IAAI,qBAAY,CAAC,kCAAkC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;SAC1F;IACL,CAAC;CACJ;AAnCD,oDAmCC"}
@@ -1,13 +1,4 @@
1
1
  "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
2
  var __importDefault = (this && this.__importDefault) || function (mod) {
12
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
13
4
  };
@@ -27,20 +18,18 @@ class XMLDataTypeStrategy {
27
18
  setFilter(filterFunction) {
28
19
  this.filter = filterFunction;
29
20
  }
30
- parseData(data) {
31
- return __awaiter(this, void 0, void 0, function* () {
32
- try {
33
- let parsed = yield xml2js_1.default.parseStringPromise(data, this.dataTypeSettings.xml2jsParams);
34
- parsed = (0, utils_1.getSubProperty)(this.dataTypeSettings.resultsPath, parsed);
35
- if (this.filter) {
36
- parsed = parsed.filter(this.filter);
37
- }
38
- return parsed;
39
- }
40
- catch (err) {
41
- throw new errors_1.GeneralError("Error while parsing source data.", this.constructor.name, err);
21
+ async parseData(data) {
22
+ try {
23
+ let parsed = await xml2js_1.default.parseStringPromise(data, this.dataTypeSettings.xml2jsParams);
24
+ parsed = (0, utils_1.getSubProperty)(this.dataTypeSettings.resultsPath, parsed);
25
+ if (this.filter) {
26
+ parsed = parsed.filter(this.filter);
42
27
  }
43
- });
28
+ return parsed;
29
+ }
30
+ catch (err) {
31
+ throw new errors_1.GeneralError("Error while parsing source data.", this.constructor.name, err);
32
+ }
44
33
  }
45
34
  }
46
35
  exports.XMLDataTypeStrategy = XMLDataTypeStrategy;
@@ -1 +1 @@
1
- {"version":3,"file":"XMLDataTypeStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/XMLDataTypeStrategy.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,kDAAgD;AAEhD,4CAA+C;AAC/C,oDAA4B;AAE5B,MAAa,mBAAmB;IAI5B,YAAY,QAAsB;QAC9B,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAC;QACjC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;IAC5B,CAAC;IAEM,mBAAmB,CAAC,QAAsB;QAC7C,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAC;IACrC,CAAC;IAEM,SAAS,CAAC,cAAkC;QAC/C,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC;IACjC,CAAC;IAEY,SAAS,CAAC,IAAS;;YAC5B,IAAI;gBACA,IAAI,MAAM,GAAG,MAAM,gBAAM,CAAC,kBAAkB,CAAC,IAAI,EAAE,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;gBACvF,MAAM,GAAG,IAAA,sBAAc,EAAC,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;gBACnE,IAAI,IAAI,CAAC,MAAM,EAAE;oBACb,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;iBACvC;gBACD,OAAO,MAAM,CAAC;aACjB;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,qBAAY,CAAC,kCAAkC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aAC1F;QACL,CAAC;KAAA;CACJ;AA7BD,kDA6BC"}
1
+ {"version":3,"file":"XMLDataTypeStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/XMLDataTypeStrategy.ts"],"names":[],"mappings":";;;;;;AAAA,kDAAgD;AAEhD,4CAA+C;AAC/C,oDAA4B;AAE5B,MAAa,mBAAmB;IAI5B,YAAY,QAAsB;QAC9B,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAC;QACjC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;IAC5B,CAAC;IAEM,mBAAmB,CAAC,QAAsB;QAC7C,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAC;IACrC,CAAC;IAEM,SAAS,CAAC,cAAkC;QAC/C,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC;IACjC,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,IAAS;QAC5B,IAAI;YACA,IAAI,MAAM,GAAG,MAAM,gBAAM,CAAC,kBAAkB,CAAC,IAAI,EAAE,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;YACvF,MAAM,GAAG,IAAA,sBAAc,EAAC,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACnE,IAAI,IAAI,CAAC,MAAM,EAAE;gBACb,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aACvC;YACD,OAAO,MAAM,CAAC;SACjB;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,IAAI,qBAAY,CAAC,kCAAkC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;SAC1F;IACL,CAAC;CACJ;AA7BD,kDA6BC"}
@@ -1,13 +1,4 @@
1
1
  "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
2
  var __importDefault = (this && this.__importDefault) || function (mod) {
12
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
13
4
  };
@@ -31,43 +22,40 @@ class ZipFileDataTypeStrategy {
31
22
  setFilter(filterFunction) {
32
23
  this.filter = filterFunction;
33
24
  }
34
- parseData(data) {
35
- var _a;
36
- return __awaiter(this, void 0, void 0, function* () {
37
- try {
38
- if (data instanceof ArrayBuffer) {
39
- data = Buffer.from(data);
40
- }
41
- const { extractionPath, files } = yield this.fileCompressor.inflateBuffer(data, (filename) => {
42
- if (this.settings.whitelistedFiles) {
43
- return this.settings.whitelistedFiles.indexOf(filename) >= 0;
44
- }
45
- return !!filename;
46
- });
47
- const redisModel = new RedisModel_1.RedisModel("HTTPProtocolStrategy" + "Model", {
48
- isKeyConstructedFromData: false,
49
- prefix: "files",
50
- }, null);
51
- const prefix = this.settings.name;
52
- const encoding = (_a = this.settings.encoding) !== null && _a !== void 0 ? _a : "utf8";
53
- const body = yield Promise.all(files.map((file) => __awaiter(this, void 0, void 0, function* () {
54
- const data = yield promises_1.default.readFile(file.path, encoding);
55
- const relativePath = path_1.default.join(prefix, file.name);
56
- yield redisModel.hset(relativePath, data.toString(encoding));
57
- return {
58
- filepath: relativePath,
59
- mtime: file.mtime,
60
- name: path_1.default.parse(relativePath).name,
61
- path: file.path,
62
- };
63
- })));
64
- yield promises_1.default.rm(extractionPath, { recursive: true, force: true });
65
- return body;
25
+ async parseData(data) {
26
+ try {
27
+ if (data instanceof ArrayBuffer) {
28
+ data = Buffer.from(data);
66
29
  }
67
- catch (err) {
68
- throw new errors_1.GeneralError("Error unzipping source file.", this.constructor.name, err);
69
- }
70
- });
30
+ const { extractionPath, files } = await this.fileCompressor.inflateBuffer(data, (filename) => {
31
+ if (this.settings.whitelistedFiles) {
32
+ return this.settings.whitelistedFiles.indexOf(filename) >= 0;
33
+ }
34
+ return !!filename;
35
+ });
36
+ const redisModel = new RedisModel_1.RedisModel("HTTPProtocolStrategy" + "Model", {
37
+ isKeyConstructedFromData: false,
38
+ prefix: "files",
39
+ }, null);
40
+ const prefix = this.settings.name;
41
+ const encoding = this.settings.encoding ?? "utf8";
42
+ const body = await Promise.all(files.map(async (file) => {
43
+ const data = await promises_1.default.readFile(file.path, encoding);
44
+ const relativePath = path_1.default.join(prefix, file.name);
45
+ await redisModel.hset(relativePath, data.toString(encoding));
46
+ return {
47
+ filepath: relativePath,
48
+ mtime: file.mtime,
49
+ name: path_1.default.parse(relativePath).name,
50
+ path: file.path,
51
+ };
52
+ }));
53
+ await promises_1.default.rm(extractionPath, { recursive: true, force: true });
54
+ return body;
55
+ }
56
+ catch (err) {
57
+ throw new errors_1.GeneralError("Error unzipping source file.", this.constructor.name, err);
58
+ }
71
59
  }
72
60
  }
73
61
  exports.ZipFileDataTypeStrategy = ZipFileDataTypeStrategy;
@@ -1 +1 @@
1
- {"version":3,"file":"ZipFileDataTypeStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/ZipFileDataTypeStrategy.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AACA,iEAA4D;AAC5D,wDAAmD;AACnD,4CAA+C;AAC/C,2DAA6B;AAC7B,4CAAoB;AACpB,gDAAwB;AAExB,MAAa,uBAAuB;IAIhC,YAAoB,QAAsB;QAAtB,aAAQ,GAAR,QAAQ,CAAc;QACtC,IAAI,CAAC,cAAc,GAAG,IAAI,+BAAc,CAAC,YAAE,CAAC,MAAM,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;IAC5B,CAAC;IAEM,mBAAmB,CAAC,QAAsB;QAC7C,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC7B,CAAC;IAEM,SAAS,CAAC,cAAkC;QAC/C,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC;IACjC,CAAC;IAEY,SAAS,CAAC,IAA0B;;;YAC7C,IAAI;gBACA,IAAI,IAAI,YAAY,WAAW,EAAE;oBAC7B,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBAC5B;gBAED,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,IAAc,EAAE,CAAC,QAAQ,EAAE,EAAE;oBACnG,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE;wBAChC,OAAO,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;qBAChE;oBAED,OAAO,CAAC,CAAC,QAAQ,CAAC;gBACtB,CAAC,CAAC,CAAC;gBAEH,MAAM,UAAU,GAAG,IAAI,uBAAU,CAC7B,sBAAsB,GAAG,OAAO,EAChC;oBACI,wBAAwB,EAAE,KAAK;oBAC/B,MAAM,EAAE,OAAO;iBAClB,EACD,IAAI,CACP,CAAC;gBACF,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAClC,MAAM,QAAQ,GAAG,MAAA,IAAI,CAAC,QAAQ,CAAC,QAAQ,mCAAI,MAAM,CAAC;gBAElD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAC1B,KAAK,CAAC,GAAG,CAAC,CAAO,IAAI,EAAE,EAAE;oBACrB,MAAM,IAAI,GAAG,MAAM,kBAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,QAAe,CAAC,CAAC;oBAC3D,MAAM,YAAY,GAAG,cAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;oBAElD,MAAM,UAAU,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,QAA0B,CAAC,CAAC,CAAC;oBAC/E,OAAO;wBACH,QAAQ,EAAE,YAAY;wBACtB,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,IAAI,EAAE,cAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI;wBACnC,IAAI,EAAE,IAAI,CAAC,IAAI;qBAClB,CAAC;gBACN,CAAC,CAAA,CAAC,CACL,CAAC;gBAEF,MAAM,kBAAE,CAAC,EAAE,CAAC,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;gBAE9D,OAAO,IAAI,CAAC;aACf;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,qBAAY,CAAC,8BAA8B,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aACtF;;KACJ;CACJ;AAhED,0DAgEC"}
1
+ {"version":3,"file":"ZipFileDataTypeStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/ZipFileDataTypeStrategy.ts"],"names":[],"mappings":";;;;;;AACA,iEAA4D;AAC5D,wDAAmD;AACnD,4CAA+C;AAC/C,2DAA6B;AAC7B,4CAAoB;AACpB,gDAAwB;AAExB,MAAa,uBAAuB;IAIhC,YAAoB,QAAsB;QAAtB,aAAQ,GAAR,QAAQ,CAAc;QACtC,IAAI,CAAC,cAAc,GAAG,IAAI,+BAAc,CAAC,YAAE,CAAC,MAAM,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;IAC5B,CAAC;IAEM,mBAAmB,CAAC,QAAsB;QAC7C,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC7B,CAAC;IAEM,SAAS,CAAC,cAAkC;QAC/C,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC;IACjC,CAAC;IAEM,KAAK,CAAC,SAAS,CAAC,IAA0B;QAC7C,IAAI;YACA,IAAI,IAAI,YAAY,WAAW,EAAE;gBAC7B,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aAC5B;YAED,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,IAAc,EAAE,CAAC,QAAQ,EAAE,EAAE;gBACnG,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE;oBAChC,OAAO,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;iBAChE;gBAED,OAAO,CAAC,CAAC,QAAQ,CAAC;YACtB,CAAC,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,IAAI,uBAAU,CAC7B,sBAAsB,GAAG,OAAO,EAChC;gBACI,wBAAwB,EAAE,KAAK;gBAC/B,MAAM,EAAE,OAAO;aAClB,EACD,IAAI,CACP,CAAC;YACF,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,MAAM,CAAC;YAElD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAC1B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;gBACrB,MAAM,IAAI,GAAG,MAAM,kBAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,QAAe,CAAC,CAAC;gBAC3D,MAAM,YAAY,GAAG,cAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBAElD,MAAM,UAAU,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,QAA0B,CAAC,CAAC,CAAC;gBAC/E,OAAO;oBACH,QAAQ,EAAE,YAAY;oBACtB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,IAAI,EAAE,cAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI;oBACnC,IAAI,EAAE,IAAI,CAAC,IAAI;iBAClB,CAAC;YACN,CAAC,CAAC,CACL,CAAC;YAEF,MAAM,kBAAE,CAAC,EAAE,CAAC,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAE9D,OAAO,IAAI,CAAC;SACf;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,IAAI,qBAAY,CAAC,8BAA8B,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;SACtF;IACL,CAAC;CACJ;AAhED,0DAgEC"}