@golemio/core 1.16.0-dev.1542548484 → 1.16.0-rc.1542535168
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/helpers/data-access/storage/AzureBlobStorageService.d.ts +1 -4
- package/dist/helpers/data-access/storage/AzureBlobStorageService.js +1 -3
- package/dist/helpers/data-access/storage/AzureBlobStorageService.js.map +1 -1
- package/dist/input-gateway/config/InputGatewayConfiguration.d.ts +9 -0
- package/dist/input-gateway/config/InputGatewayConfiguration.js +10 -1
- package/dist/input-gateway/config/InputGatewayConfiguration.js.map +1 -1
- package/dist/input-gateway/config/abstract.d.ts +2 -0
- package/dist/input-gateway/data-access/StorageServiceProvider.d.ts +3 -3
- package/dist/input-gateway/data-access/StorageServiceProvider.js +6 -17
- package/dist/input-gateway/data-access/StorageServiceProvider.js.map +1 -1
- package/dist/input-gateway/helpers/SaveRawDataMiddleware.d.ts +2 -3
- package/dist/input-gateway/helpers/SaveRawDataMiddleware.js +8 -10
- package/dist/input-gateway/helpers/SaveRawDataMiddleware.js.map +1 -1
- package/dist/integration-engine/config/IntegrationEngineConfiguration.d.ts +9 -0
- package/dist/integration-engine/config/IntegrationEngineConfiguration.js +10 -0
- package/dist/integration-engine/config/IntegrationEngineConfiguration.js.map +1 -1
- package/dist/integration-engine/config/abstract.d.ts +2 -0
- package/dist/integration-engine/data-access/RawDataStorage.d.ts +1 -2
- package/dist/integration-engine/data-access/RawDataStorage.js +4 -6
- package/dist/integration-engine/data-access/RawDataStorage.js.map +1 -1
- package/dist/integration-engine/data-access/StorageServiceProvider.d.ts +3 -3
- package/dist/integration-engine/data-access/StorageServiceProvider.js +4 -16
- package/dist/integration-engine/data-access/StorageServiceProvider.js.map +1 -1
- package/dist/integration-engine/datasources/DataSource.d.ts +1 -10
- package/dist/integration-engine/datasources/DataSource.js +23 -45
- package/dist/integration-engine/datasources/DataSource.js.map +1 -1
- package/dist/integration-engine/datasources/DataSourceStreamed.d.ts +1 -5
- package/dist/integration-engine/datasources/DataSourceStreamed.js +22 -30
- package/dist/integration-engine/datasources/DataSourceStreamed.js.map +1 -1
- package/dist/integration-engine/datasources/IDataSource.d.ts +2 -11
- package/dist/integration-engine/datasources/protocol-strategy/HTTPFetchProtocolStrategy.d.ts +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/HTTPFetchProtocolStrategy.js +0 -9
- package/dist/integration-engine/datasources/protocol-strategy/HTTPFetchProtocolStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/IProtocolStrategy.d.ts +0 -6
- package/dist/integration-engine/datasources/protocol-strategy/PaginatedHTTPProtocolStrategy.js +2 -10
- package/dist/integration-engine/datasources/protocol-strategy/PaginatedHTTPProtocolStrategy.js.map +1 -1
- package/dist/integration-engine/datasources/protocol-strategy/ProtocolStrategy.d.ts +0 -8
- package/dist/integration-engine/datasources/protocol-strategy/ProtocolStrategy.js +33 -26
- package/dist/integration-engine/datasources/protocol-strategy/ProtocolStrategy.js.map +1 -1
- package/dist/integration-engine/helpers/RawDataStore.d.ts +4 -0
- package/dist/integration-engine/helpers/RawDataStore.js +25 -0
- package/dist/integration-engine/helpers/RawDataStore.js.map +1 -0
- package/package.json +4 -5
- package/dist/integration-engine/helpers/ResourceNotModified.d.ts +0 -3
- package/dist/integration-engine/helpers/ResourceNotModified.js +0 -8
- package/dist/integration-engine/helpers/ResourceNotModified.js.map +0 -1
|
@@ -5,10 +5,7 @@ import { Readable } from "stream";
|
|
|
5
5
|
import { ILogger } from "../../logger";
|
|
6
6
|
import { AbstractStorageService } from "./AbstractStorageService";
|
|
7
7
|
export interface IAzureBlobStorageConfig {
|
|
8
|
-
|
|
9
|
-
tenantId: string;
|
|
10
|
-
clientSecret: string;
|
|
11
|
-
account: string;
|
|
8
|
+
connectionString: string;
|
|
12
9
|
containerName: string;
|
|
13
10
|
}
|
|
14
11
|
export declare class AzureBlobStorageService extends AbstractStorageService {
|
|
@@ -13,13 +13,11 @@ exports.AzureBlobStorageService = void 0;
|
|
|
13
13
|
const storage_blob_1 = require("@azure/storage-blob");
|
|
14
14
|
const AbstractStorageService_1 = require("./AbstractStorageService");
|
|
15
15
|
const _helpers_1 = require("../..");
|
|
16
|
-
const identity_1 = require("@azure/identity");
|
|
17
16
|
class AzureBlobStorageService extends AbstractStorageService_1.AbstractStorageService {
|
|
18
17
|
constructor(config, log) {
|
|
19
18
|
super();
|
|
20
19
|
this.log = log;
|
|
21
|
-
const
|
|
22
|
-
const blobServiceClient = new storage_blob_1.BlobServiceClient(`https://${config.account}.blob.core.windows.net`, credentials);
|
|
20
|
+
const blobServiceClient = storage_blob_1.BlobServiceClient.fromConnectionString(config.connectionString);
|
|
23
21
|
this.containerClient = blobServiceClient.getContainerClient(config.containerName);
|
|
24
22
|
this.containerClient.createIfNotExists().catch((err) => {
|
|
25
23
|
// This is fine in the production environment
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AzureBlobStorageService.js","sourceRoot":"","sources":["../../../../src/helpers/data-access/storage/AzureBlobStorageService.ts"],"names":[],"mappings":";;;;;;;;;;;;AACA,sDAA4H;AAG5H,qEAAkE;AAClE,oCAAoC;
|
|
1
|
+
{"version":3,"file":"AzureBlobStorageService.js","sourceRoot":"","sources":["../../../../src/helpers/data-access/storage/AzureBlobStorageService.ts"],"names":[],"mappings":";;;;;;;;;;;;AACA,sDAA4H;AAG5H,qEAAkE;AAClE,oCAAoC;AAOpC,MAAa,uBAAwB,SAAQ,+CAAsB;IAG/D,YAAY,MAA+B,EAAmB,GAAY;QACtE,KAAK,EAAE,CAAC;QADkD,QAAG,GAAH,GAAG,CAAS;QAGtE,MAAM,iBAAiB,GAAG,gCAAiB,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAC1F,IAAI,CAAC,eAAe,GAAG,iBAAiB,CAAC,kBAAkB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAClF,IAAI,CAAC,eAAe,CAAC,iBAAiB,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACnD,6CAA6C;YAC7C,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,qCAAqC,MAAM,CAAC,aAAa,GAAG,EAAE,GAAG,CAAC,CAAC;QAC/G,CAAC,CAAC,CAAC;IACP,CAAC;IAEY,YAAY,CAAC,MAAgB,EAAE,UAAkB,EAAE,MAAc,EAAE,EAAE,QAAiB;;YAC/F,MAAM,GAAG,GAAG,IAAA,mBAAQ,EAAC,IAAI,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC,CAAC;YAChE,MAAM,cAAc,GAAG,QAAQ,aAAR,QAAQ,cAAR,QAAQ,GAAI,GAAG,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC;YAC/F,MAAM,WAAW,GAAG,GAAG,UAAU,IAAI,cAAc,EAAE,CAAC;YACtD,MAAM,WAAW,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAEzC,IAAI;gBACA,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,kBAAkB,CAAC,GAAG,WAAW,GAAG,WAAW,EAAE,CAAC,CAAC;gBAE3F,MAAM,OAAO,GAAmC;oBAC5C,eAAe,EAAE,EAAE,eAAe,EAAE,0BAA0B,EAAE;oBAChE,UAAU,EAAE,IAAI,CAAC,oBAAoB,CAAC,WAAW,GAAG,WAAW,CAAC;iBACnE,CAAC;gBAEF,MAAM,UAAU,CAAC,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;aACxE;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,mCAAmC,WAAW,GAAG,WAAW,GAAG,EAAE,GAAG,CAAC,CAAC;aACjH;QACL,CAAC;KAAA;IAEY,UAAU,CAAC,IAAqB,EAAE,UAAkB,EAAE,MAAc,EAAE,EAAE,QAAiB;;YAClG,MAAM,GAAG,GAAG,IAAA,mBAAQ,EAAC,IAAI,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC,CAAC;YAChE,MAAM,cAAc,GAAG,QAAQ,aAAR,QAAQ,cAAR,QAAQ,GAAI,GAAG,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC;YAC/F,MAAM,WAAW,GAAG,GAAG,UAAU,IAAI,cAAc,EAAE,CAAC;YACtD,MAAM,WAAW,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAEzC,IAAI;gBACA,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,kBAAkB,CAAC,GAAG,WAAW,GAAG,WAAW,EAAE,CAAC,CAAC;gBAE3F,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;oBACxB,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBAC5B;gBAED,MAAM,OAAO,GAAmC;oBAC5C,eAAe,EAAE,EAAE,eAAe,EAAE,0BAA0B,EAAE;oBAChE,UAAU,EAAE,IAAI,CAAC,oBAAoB,CAAC,WAAW,GAAG,WAAW,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;iBAC5F,CAAC;gBAEF,MAAM,UAAU,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;aAC9C;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,4BAA4B,WAAW,GAAG,WAAW,GAAG,EAAE,GAAG,CAAC,CAAC;aAC1G;QACL,CAAC;KAAA;IAEY,UAAU,CAAC,OAAe;;YACnC,IAAI;gBACA,MAAM,OAAO,GAAsB;oBAC/B,eAAe,EAAE,SAAS;iBAC7B,CAAC;gBAEF,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;gBAEpE,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;aACpC;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,8BAA8B,OAAO,GAAG,EAAE,GAAG,CAAC,CAAC;aAC1F;QACL,CAAC;KAAA;IAEY,aAAa,CAAC,OAAe;;YACtC,IAAI;gBACA,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;gBAEpE,MAAM,gBAAgB,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC;gBACrD,OAAO,gBAAgB,CAAC,kBAAkB,CAAC;aAC9C;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,0BAA0B,OAAO,GAAG,EAAE,GAAG,CAAC,CAAC;aACtF;QACL,CAAC;KAAA;IAEO,oBAAoB,CAAC,QAAgB,EAAE,cAAuB;QAClE,OAAO,CAAC,QAA+B,EAAE,EAAE;YACvC,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,gBAAgB;gBACrC,GAAG,QAAQ,KAAK,QAAQ,CAAC,WAAW,MAAM,cAAc,aAAd,cAAc,cAAd,cAAc,GAAI,SAAS,EAAE,CAC9E,CAAC;QACN,CAAC,CAAC;IACN,CAAC;CACJ;AA3FD,0DA2FC"}
|
|
@@ -11,6 +11,15 @@ export declare class InputGatewayConfiguration implements IConfiguration {
|
|
|
11
11
|
port: string | undefined;
|
|
12
12
|
rabbit_connections: string[] | undefined;
|
|
13
13
|
rabbit_exchange_name: string | undefined;
|
|
14
|
+
storage: {
|
|
15
|
+
enabled: boolean;
|
|
16
|
+
provider: {
|
|
17
|
+
azure: {
|
|
18
|
+
connectionString: string;
|
|
19
|
+
containerName: string;
|
|
20
|
+
};
|
|
21
|
+
};
|
|
22
|
+
};
|
|
14
23
|
saveRawDataWhitelist: any;
|
|
15
24
|
sentry: {
|
|
16
25
|
enabled: boolean;
|
|
@@ -15,7 +15,7 @@ const utils_1 = require("../../helpers/utils");
|
|
|
15
15
|
const tsyringe_1 = require("tsyringe");
|
|
16
16
|
let InputGatewayConfiguration = exports.InputGatewayConfiguration = class InputGatewayConfiguration {
|
|
17
17
|
constructor() {
|
|
18
|
-
var _a;
|
|
18
|
+
var _a, _b, _c;
|
|
19
19
|
this.app_version = process.env.npm_package_version;
|
|
20
20
|
this.app_name = process.env.APP_NAME || "input-gateway";
|
|
21
21
|
this.colorful_logs = process.env.COLORFUL_LOGS;
|
|
@@ -27,6 +27,15 @@ let InputGatewayConfiguration = exports.InputGatewayConfiguration = class InputG
|
|
|
27
27
|
this.port = process.env.PORT;
|
|
28
28
|
this.rabbit_connections = (_a = process.env.RABBIT_CONN) === null || _a === void 0 ? void 0 : _a.split(",");
|
|
29
29
|
this.rabbit_exchange_name = process.env.RABBIT_EXCHANGE_NAME;
|
|
30
|
+
this.storage = {
|
|
31
|
+
enabled: process.env.STORAGE_ENABLED === "true",
|
|
32
|
+
provider: {
|
|
33
|
+
azure: {
|
|
34
|
+
connectionString: (_b = process.env.AZURE_BLOB_STORAGE_CONN) !== null && _b !== void 0 ? _b : "",
|
|
35
|
+
containerName: (_c = process.env.AZURE_BLOB_STORAGE_CONTAINER_NAME) !== null && _c !== void 0 ? _c : "",
|
|
36
|
+
},
|
|
37
|
+
},
|
|
38
|
+
};
|
|
30
39
|
this.sentry = {
|
|
31
40
|
enabled: process.env.SENTRY_ENABLED === "true",
|
|
32
41
|
dsn: process.env.SENTRY_DSN,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"InputGatewayConfiguration.js","sourceRoot":"","sources":["../../../src/input-gateway/config/InputGatewayConfiguration.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,4CAAwC;AACxC,+CAAyC;AACzC,uCAAsC;AAI/B,IAAM,yBAAyB,uCAA/B,MAAM,yBAAyB;
|
|
1
|
+
{"version":3,"file":"InputGatewayConfiguration.js","sourceRoot":"","sources":["../../../src/input-gateway/config/InputGatewayConfiguration.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,4CAAwC;AACxC,+CAAyC;AACzC,uCAAsC;AAI/B,IAAM,yBAAyB,uCAA/B,MAAM,yBAAyB;IAyClC;;QAxCO,gBAAW,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;QAC9C,aAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,eAAe,CAAC;QACnD,kBAAa,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;QAC1C,cAAS,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;QAClC,aAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,aAAa,CAAC;QACjD,sBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;QAClD,uBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;QACpD,sBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;QAClD,SAAI,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;QACxB,uBAAkB,GAAG,MAAA,OAAO,CAAC,GAAG,CAAC,WAAW,0CAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QACzD,yBAAoB,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;QACxD,YAAO,GAAG;YACb,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,KAAK,MAAM;YAC/C,QAAQ,EAAE;gBACN,KAAK,EAAE;oBACH,gBAAgB,EAAE,MAAA,OAAO,CAAC,GAAG,CAAC,uBAAuB,mCAAI,EAAE;oBAC3D,aAAa,EAAE,MAAA,OAAO,CAAC,GAAG,CAAC,iCAAiC,mCAAI,EAAE;iBACrE;aACJ;SACJ,CAAC;QAEK,WAAM,GAAG;YACZ,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,MAAM;YAC9C,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;YAC3B,OAAO,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE;YACrE,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;YAC3C,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,MAAM;YAC1C,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC;SAClG,CAAC;QACK,YAAO,GAAG;YACb,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,KAAK,MAAM;YAC/C,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,YAAsB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI;YACxF,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE;SAC3C,CAAC;QACK,cAAS,GAAG;YACf,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI;YAC/G,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK;YACtH,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI;SACrH,CAAC;QAGE,MAAM,oBAAoB,GAAG,IAAI,uBAAY,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC;QAC3E,IAAI,CAAC,oBAAoB,GAAG,IAAA,eAAO,EAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,oBAAoB,CAAC;IAC1F,CAAC;CACJ,CAAA;oCA7CY,yBAAyB;IADrC,IAAA,qBAAU,GAAE;;GACA,yBAAyB,CA6CrC"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { ISentryConfig } from "../../monitoring/sentry/sentry-provider";
|
|
2
2
|
import { IMetricsConfig } from "../../monitoring/metrics/metrics-interface";
|
|
3
|
+
import { IStorageConfig } from "../../helpers/data-access/storage";
|
|
3
4
|
export interface ISaveRawDataWhitelist {
|
|
4
5
|
route: string;
|
|
5
6
|
withHeaders: boolean;
|
|
@@ -16,6 +17,7 @@ export interface IConfiguration {
|
|
|
16
17
|
port: string | undefined;
|
|
17
18
|
rabbit_connections: string[] | undefined;
|
|
18
19
|
rabbit_exchange_name: string | undefined;
|
|
20
|
+
storage: IStorageConfig;
|
|
19
21
|
sentry: ISentryConfig;
|
|
20
22
|
saveRawDataWhitelist: ISaveRawDataWhitelist[];
|
|
21
23
|
metrics: IMetricsConfig;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { ILogger } from "../../helpers";
|
|
2
|
-
import { ISimpleConfig } from "../../helpers/configuration/ISimpleConfig";
|
|
3
1
|
import { AbstractStorageService } from "../../helpers/data-access/storage";
|
|
2
|
+
import { ILogger } from "../../helpers";
|
|
3
|
+
import { IConfiguration } from "../config";
|
|
4
4
|
export declare class StorageServiceProvider {
|
|
5
5
|
protected _service: AbstractStorageService;
|
|
6
|
-
constructor(config:
|
|
6
|
+
constructor(config: IConfiguration, log: ILogger);
|
|
7
7
|
get service(): AbstractStorageService;
|
|
8
8
|
}
|
|
@@ -13,24 +13,13 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
|
13
13
|
};
|
|
14
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
15
|
exports.StorageServiceProvider = void 0;
|
|
16
|
-
const storage_1 = require("../../helpers/data-access/storage");
|
|
17
|
-
const CoreToken_1 = require("../../helpers/ioc/CoreToken");
|
|
18
|
-
const errors_1 = require("@golemio/errors");
|
|
19
16
|
const tsyringe_1 = require("tsyringe");
|
|
17
|
+
const errors_1 = require("@golemio/errors");
|
|
18
|
+
const storage_1 = require("../../helpers/data-access/storage");
|
|
19
|
+
const ioc_1 = require("../ioc");
|
|
20
20
|
let StorageServiceProvider = exports.StorageServiceProvider = class StorageServiceProvider {
|
|
21
21
|
constructor(config, log) {
|
|
22
|
-
const service = new storage_1.StorageServiceFactory(
|
|
23
|
-
enabled: config.getValue("env.STORAGE_ENABLED", "false") === "true",
|
|
24
|
-
provider: {
|
|
25
|
-
azure: {
|
|
26
|
-
clientId: config.getValue("env.AZURE_BLOB_CLIENT_ID", ""),
|
|
27
|
-
tenantId: config.getValue("env.AZURE_BLOB_TENANT_ID", ""),
|
|
28
|
-
clientSecret: config.getValue("env.AZURE_BLOB_CLIENT_SECRET", ""),
|
|
29
|
-
account: config.getValue("env.AZURE_BLOB_ACCOUNT", ""),
|
|
30
|
-
containerName: config.getValue("env.AZURE_BLOB_STORAGE_CONTAINER_NAME", ""),
|
|
31
|
-
},
|
|
32
|
-
},
|
|
33
|
-
}, log).getService(storage_1.StorageProvider.Azure);
|
|
22
|
+
const service = new storage_1.StorageServiceFactory(config.storage, log).getService(storage_1.StorageProvider.Azure);
|
|
34
23
|
if (!service) {
|
|
35
24
|
throw new errors_1.FatalError(`StorageServiceProvider returned undefined for provider ${storage_1.StorageProvider.Azure}`, this.constructor.name);
|
|
36
25
|
}
|
|
@@ -42,8 +31,8 @@ let StorageServiceProvider = exports.StorageServiceProvider = class StorageServi
|
|
|
42
31
|
};
|
|
43
32
|
exports.StorageServiceProvider = StorageServiceProvider = __decorate([
|
|
44
33
|
(0, tsyringe_1.injectable)(),
|
|
45
|
-
__param(0, (0, tsyringe_1.inject)(
|
|
46
|
-
__param(1, (0, tsyringe_1.inject)(
|
|
34
|
+
__param(0, (0, tsyringe_1.inject)(ioc_1.ContainerToken.Config)),
|
|
35
|
+
__param(1, (0, tsyringe_1.inject)(ioc_1.ContainerToken.Logger)),
|
|
47
36
|
__metadata("design:paramtypes", [Object, Object])
|
|
48
37
|
], StorageServiceProvider);
|
|
49
38
|
//# sourceMappingURL=StorageServiceProvider.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"StorageServiceProvider.js","sourceRoot":"","sources":["../../../src/input-gateway/data-access/StorageServiceProvider.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"StorageServiceProvider.js","sourceRoot":"","sources":["../../../src/input-gateway/data-access/StorageServiceProvider.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,uCAA8C;AAC9C,4CAA6C;AAC7C,+DAA8G;AAC9G,gCAAyC;AAKlC,IAAM,sBAAsB,oCAA5B,MAAM,sBAAsB;IAG/B,YAA2C,MAAsB,EAAiC,GAAY;QAC1G,MAAM,OAAO,GAAG,IAAI,+BAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,yBAAe,CAAC,KAAK,CAAC,CAAC;QAEjG,IAAI,CAAC,OAAO,EAAE;YACV,MAAM,IAAI,mBAAU,CAChB,0DAA0D,yBAAe,CAAC,KAAK,EAAE,EACjF,IAAI,CAAC,WAAW,CAAC,IAAI,CACxB,CAAC;SACL;QAED,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;IAC5B,CAAC;IAED,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC;IACzB,CAAC;CACJ,CAAA;iCAnBY,sBAAsB;IADlC,IAAA,qBAAU,GAAE;IAII,WAAA,IAAA,iBAAM,EAAC,oBAAc,CAAC,MAAM,CAAC,CAAA;IAA0B,WAAA,IAAA,iBAAM,EAAC,oBAAc,CAAC,MAAM,CAAC,CAAA;;GAHxF,sBAAsB,CAmBlC"}
|
|
@@ -1,14 +1,13 @@
|
|
|
1
|
+
import { NextFunction, Request, Response } from "express";
|
|
1
2
|
import { ILogger } from "../../helpers";
|
|
2
|
-
import { ISimpleConfig } from "../../helpers/configuration/ISimpleConfig";
|
|
3
3
|
import { AbstractStorageService } from "../../helpers/data-access/storage";
|
|
4
4
|
import { IConfiguration } from "../config";
|
|
5
|
-
import { NextFunction, Request, Response } from "express";
|
|
6
5
|
export declare class SaveRawDataMiddleware {
|
|
7
6
|
private readonly log;
|
|
8
7
|
private storage;
|
|
9
8
|
private readonly storageEnabled;
|
|
10
9
|
private readonly whitelist;
|
|
11
|
-
constructor(config: IConfiguration,
|
|
10
|
+
constructor(config: IConfiguration, log: ILogger, storage: AbstractStorageService);
|
|
12
11
|
getMiddleware: () => (req: Request, res: Response, next: NextFunction) => void;
|
|
13
12
|
getWhitelistRegexp: () => {
|
|
14
13
|
routesRegexp: RegExp;
|
|
@@ -13,17 +13,16 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
|
13
13
|
};
|
|
14
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
15
|
exports.SaveRawDataMiddleware = void 0;
|
|
16
|
+
const tsyringe_1 = require("tsyringe");
|
|
17
|
+
const path_to_regexp_1 = require("path-to-regexp");
|
|
18
|
+
const errors_1 = require("@golemio/errors");
|
|
16
19
|
const _helpers_1 = require("../../helpers");
|
|
17
20
|
const storage_1 = require("../../helpers/data-access/storage");
|
|
18
|
-
const CoreToken_1 = require("../../helpers/ioc/CoreToken");
|
|
19
21
|
const CheckContentTypeMiddleware_1 = require("./CheckContentTypeMiddleware");
|
|
20
22
|
const ContentTypeHelper_1 = require("./ContentTypeHelper");
|
|
21
23
|
const ioc_1 = require("../ioc");
|
|
22
|
-
const errors_1 = require("@golemio/errors");
|
|
23
|
-
const path_to_regexp_1 = require("path-to-regexp");
|
|
24
|
-
const tsyringe_1 = require("tsyringe");
|
|
25
24
|
let SaveRawDataMiddleware = exports.SaveRawDataMiddleware = class SaveRawDataMiddleware {
|
|
26
|
-
constructor(config,
|
|
25
|
+
constructor(config, log, storage) {
|
|
27
26
|
this.log = log;
|
|
28
27
|
this.storage = storage;
|
|
29
28
|
this.getMiddleware = () => {
|
|
@@ -66,16 +65,15 @@ let SaveRawDataMiddleware = exports.SaveRawDataMiddleware = class SaveRawDataMid
|
|
|
66
65
|
headersRegexp: regexpHeaders.length ? RegExp(regexpHeaders.join("|")) : RegExp(/.^/),
|
|
67
66
|
};
|
|
68
67
|
};
|
|
69
|
-
this.storageEnabled =
|
|
68
|
+
this.storageEnabled = config.storage.enabled;
|
|
70
69
|
this.whitelist = config.saveRawDataWhitelist;
|
|
71
70
|
}
|
|
72
71
|
};
|
|
73
72
|
exports.SaveRawDataMiddleware = SaveRawDataMiddleware = __decorate([
|
|
74
73
|
(0, tsyringe_1.injectable)(),
|
|
75
74
|
__param(0, (0, tsyringe_1.inject)(ioc_1.ContainerToken.Config)),
|
|
76
|
-
__param(1, (0, tsyringe_1.inject)(
|
|
77
|
-
__param(2, (0, tsyringe_1.inject)(
|
|
78
|
-
|
|
79
|
-
__metadata("design:paramtypes", [Object, Object, Object, storage_1.AbstractStorageService])
|
|
75
|
+
__param(1, (0, tsyringe_1.inject)(ioc_1.ContainerToken.Logger)),
|
|
76
|
+
__param(2, (0, tsyringe_1.inject)(ioc_1.ContainerToken.StorageService)),
|
|
77
|
+
__metadata("design:paramtypes", [Object, Object, storage_1.AbstractStorageService])
|
|
80
78
|
], SaveRawDataMiddleware);
|
|
81
79
|
//# sourceMappingURL=SaveRawDataMiddleware.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SaveRawDataMiddleware.js","sourceRoot":"","sources":["../../../src/input-gateway/helpers/SaveRawDataMiddleware.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"SaveRawDataMiddleware.js","sourceRoot":"","sources":["../../../src/input-gateway/helpers/SaveRawDataMiddleware.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AACA,uCAA8C;AAC9C,mDAA8C;AAC9C,4CAA6D;AAC7D,4CAA6C;AAC7C,+DAAsE;AACtE,6EAAwE;AACxE,2DAAuE;AACvE,gCAAyC;AAIlC,IAAM,qBAAqB,mCAA3B,MAAM,qBAAqB;IAG9B,YACmC,MAAsB,EACtB,GAA6B,EACrB,OAAuC;QAD9B,QAAG,GAAH,GAAG,CAAS;QACb,YAAO,GAAP,OAAO,CAAwB;QAM3E,kBAAa,GAAG,GAAG,EAAE;YACxB,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAClD,OAAO,CAAC,GAAY,EAAE,GAAa,EAAE,IAAkB,EAAE,EAAE;gBACvD,MAAM,cAAc,GAAG,GAAG,CAAC,IAAI,CAAC;gBAEhC,IAAI,IAAI,CAAC,cAAc,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,eAAe,CAAC,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE;oBACnG,iCAAiC;oBACjC,MAAM,WAAW,GAAG,IAAA,2CAAc,EAAC,GAAG,CAAwC,CAAC;oBAC/E,MAAM,GAAG,GAAG,0CAAsB,CAAC,WAAW,CAAC,CAAC;oBAEhD,wBAAwB;oBACxB,IAAI,CAAC,GAAG,EAAE;wBACN,qBAAY,CAAC,MAAM,CACf,IAAI,qBAAY,CACZ,4BAA4B,cAAc,GAAG,EAC7C,uBAAuB,EACvB,IAAI,KAAK,CAAC,8BAA8B,WAAW,iBAAiB,CAAC,EACrE,GAAG,CACN,EACD,IAAI,CAAC,GAAG,CACX,CAAC;wBACF,OAAO,IAAI,EAAE,CAAC;qBACjB;oBAED,MAAM,GAAG,GAAG,IAAA,mBAAQ,EAAC,IAAI,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC,CAAC;oBAChE,MAAM,QAAQ,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC;oBAE7E,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;oBAE5E,8CAA8C;oBAC9C,IAAI,eAAe,CAAC,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE;wBACpD,MAAM,cAAc,GAAG,GAAG,QAAQ,UAAU,CAAC;wBAC7C,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,cAAc,CAAC,CAAC;qBAC9G;iBACJ;gBAED,IAAI,EAAE,CAAC;YACX,CAAC,CAAC;QACN,CAAC,CAAC;QAEK,uBAAkB,GAAG,GAAG,EAAE;YAC7B,MAAM,YAAY,GAAG,EAAE,CAAC;YACxB,MAAM,aAAa,GAAG,EAAE,CAAC;YAEzB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,SAAS,EAAE;gBAClC,MAAM,WAAW,GAAG,IAAI,IAAA,6BAAY,EAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;gBAErE,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBAC/B,IAAI,OAAO,CAAC,WAAW,EAAE;oBACrB,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;iBACnC;aACJ;YAED,OAAO;gBACH,YAAY,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;gBACjF,aAAa,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;aACvF,CAAC;QACN,CAAC,CAAC;QA7DE,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;QAC7C,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,oBAAoB,CAAC;IACjD,CAAC;CA4DJ,CAAA;gCAtEY,qBAAqB;IADjC,IAAA,qBAAU,GAAE;IAKJ,WAAA,IAAA,iBAAM,EAAC,oBAAc,CAAC,MAAM,CAAC,CAAA;IAC7B,WAAA,IAAA,iBAAM,EAAC,oBAAc,CAAC,MAAM,CAAC,CAAA;IAC7B,WAAA,IAAA,iBAAM,EAAC,oBAAc,CAAC,cAAc,CAAC,CAAA;qDAAkB,gCAAsB;GANzE,qBAAqB,CAsEjC"}
|
|
@@ -29,6 +29,15 @@ export declare class IntegrationEngineConfiguration implements IConfiguration {
|
|
|
29
29
|
datasources: Record<string, any>;
|
|
30
30
|
port: string | undefined;
|
|
31
31
|
queuesBlacklist: Record<string, string[]>;
|
|
32
|
+
storage: {
|
|
33
|
+
enabled: boolean;
|
|
34
|
+
provider: {
|
|
35
|
+
azure: {
|
|
36
|
+
connectionString: string;
|
|
37
|
+
containerName: string;
|
|
38
|
+
};
|
|
39
|
+
};
|
|
40
|
+
};
|
|
32
41
|
saveRawDataWhitelist: Record<string, any>;
|
|
33
42
|
sentry: {
|
|
34
43
|
enabled: boolean;
|
|
@@ -17,6 +17,7 @@ const tsyringe_1 = require("tsyringe");
|
|
|
17
17
|
*/
|
|
18
18
|
let IntegrationEngineConfiguration = exports.IntegrationEngineConfiguration = class IntegrationEngineConfiguration {
|
|
19
19
|
constructor() {
|
|
20
|
+
var _a, _b;
|
|
20
21
|
this.DATA_BATCH_SIZE = process.env.DATA_BATCH_SIZE ? Number(process.env.DATA_BATCH_SIZE) : 1000;
|
|
21
22
|
this.HOPPYGO_BASE_URL = process.env.HOPPYGO_BASE_URL;
|
|
22
23
|
this.LOG_LEVEL = process.env.LOG_LEVEL;
|
|
@@ -43,6 +44,15 @@ let IntegrationEngineConfiguration = exports.IntegrationEngineConfiguration = cl
|
|
|
43
44
|
this.app_version = process.env.npm_package_version;
|
|
44
45
|
this.app_name = process.env.APP_NAME || "integration-engine";
|
|
45
46
|
this.port = process.env.PORT;
|
|
47
|
+
this.storage = {
|
|
48
|
+
enabled: process.env.STORAGE_ENABLED === "true",
|
|
49
|
+
provider: {
|
|
50
|
+
azure: {
|
|
51
|
+
connectionString: (_a = process.env.AZURE_BLOB_STORAGE_CONN) !== null && _a !== void 0 ? _a : "",
|
|
52
|
+
containerName: (_b = process.env.AZURE_BLOB_STORAGE_CONTAINER_NAME) !== null && _b !== void 0 ? _b : "",
|
|
53
|
+
},
|
|
54
|
+
},
|
|
55
|
+
};
|
|
46
56
|
this.sentry = {
|
|
47
57
|
enabled: process.env.SENTRY_ENABLED === "true",
|
|
48
58
|
dsn: process.env.SENTRY_DSN,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"IntegrationEngineConfiguration.js","sourceRoot":"","sources":["../../../src/integration-engine/config/IntegrationEngineConfiguration.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,4CAAwC;AACxC,uCAAsC;AAGtC;;GAEG;AAGI,IAAM,8BAA8B,4CAApC,MAAM,8BAA8B;
|
|
1
|
+
{"version":3,"file":"IntegrationEngineConfiguration.js","sourceRoot":"","sources":["../../../src/integration-engine/config/IntegrationEngineConfiguration.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,4CAAwC;AACxC,uCAAsC;AAGtC;;GAEG;AAGI,IAAM,8BAA8B,4CAApC,MAAM,8BAA8B;IAkFvC;;QAjFO,oBAAe,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3F,qBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;QAChD,cAAS,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;QAClC,+BAA0B,GAAG,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC;QACpE,aAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,aAAa,CAAC;QACjD,oCAA+B,GAAG,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC;QAC9E,mCAA8B,GAAG,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC;QAC5E,+BAA0B,GAAG,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC;QACpE,uBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;QACpD,yBAAoB,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;QACxD,8BAAyB,GAAG,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC;QAClE,kBAAa,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;QAC1C,kCAA6B,GAAG,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC;QAC1E,kCAA6B,GAAG,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC;QAC1E,+BAA0B,GAAG,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC;QACpE,gBAAW,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;QACtC,yBAAoB,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;QACxD,sCAAiC,GAAG,OAAO,CAAC,GAAG,CAAC,iCAAiC;YACpF,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC;YACvD,CAAC,CAAC,CAAC,CAAC;QACD,eAAU,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;QACpC,yBAAoB,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;QACxD,wBAAmB,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;QACtD,gBAAW,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;QAC9C,aAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,oBAAoB,CAAC;QAExD,SAAI,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;QAExB,YAAO,GAAG;YACb,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,KAAK,MAAM;YAC/C,QAAQ,EAAE;gBACN,KAAK,EAAE;oBACH,gBAAgB,EAAE,MAAA,OAAO,CAAC,GAAG,CAAC,uBAAuB,mCAAI,EAAE;oBAC3D,aAAa,EAAE,MAAA,OAAO,CAAC,GAAG,CAAC,iCAAiC,mCAAI,EAAE;iBACrE;aACJ;SACJ,CAAC;QAEK,WAAM,GAAG;YACZ,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,MAAM;YAC9C,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;YAC3B,OAAO,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE;YACrE,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;YAC3C,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,MAAM;YAC1C,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC;SAClG,CAAC;QACK,WAAM,GAAG;YACZ,qBAAqB,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,4BAAsC,EAAE,EAAE,CAAC,IAAI,EAAE;YAC7F,qBAAqB,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,4BAAsC,EAAE,EAAE,CAAC,IAAI,IAAI;SAClG,CAAC;QACK,YAAO,GAAG;YACb,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,KAAK,MAAM;YAC/C,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,YAAsB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI;YACxF,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE;SAC3C,CAAC;QACK,cAAS,GAAG;YACf,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI;YAC/G,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK;YACtH,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI;SACrH,CAAC;QACK,qBAAgB,GAAG;YACtB,iBAAiB,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,IAAI,GAAG;YACnF,qHAAqH;YACrH,uBAAuB,EAAE,OAAO,CAAC,GAAG,CAAC,yCAAyC;gBAC1E,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC;gBAC/D,CAAC,CAAC,CAAC;YACP,yCAAyC;YACzC,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,mCAAmC,IAAI,OAAO;YAC3E,6FAA6F;YAC7F,sBAAsB,EAAE,OAAO,CAAC,GAAG,CAAC,0CAA0C,KAAK,MAAM;YACzF,wBAAwB,EAAE,OAAO,CAAC,GAAG,CAAC,6CAA6C;gBAC/E,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,6CAA6C,EAAE,EAAE,CAAC;gBACzE,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;YACxB,yBAAyB,EAAE,OAAO,CAAC,GAAG,CAAC,yDAAyD;gBAC5F,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC;gBACjF,CAAC,CAAC,EAAE;YACR,2BAA2B,EAAE,OAAO,CAAC,GAAG,CAAC,2DAA2D;gBAChG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,2DAA2D,CAAC;gBACnF,CAAC,CAAC,CAAC,CAAC;SACX,CAAC;QAGE,IAAI,CAAC,WAAW,GAAG,IAAI,uBAAY,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC;QACzD,IAAI,CAAC,eAAe,GAAG,IAAI,uBAAY,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC;QAChE,IAAI,CAAC,oBAAoB,GAAG,IAAI,uBAAY,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC;IAC9E,CAAC;CACJ,CAAA;yCAvFY,8BAA8B;IAD1C,IAAA,qBAAU,GAAE;;GACA,8BAA8B,CAuF1C"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { IMetricsConfig } from "../../monitoring";
|
|
2
2
|
import { ISentryConfig } from "../../monitoring/sentry/sentry-provider";
|
|
3
|
+
import { IStorageConfig } from "../../helpers/data-access/storage";
|
|
3
4
|
export interface IConfiguration {
|
|
4
5
|
DATA_BATCH_SIZE: number;
|
|
5
6
|
HOPPYGO_BASE_URL: string | undefined;
|
|
@@ -27,6 +28,7 @@ export interface IConfiguration {
|
|
|
27
28
|
datasources: Record<string, any>;
|
|
28
29
|
port: string | undefined;
|
|
29
30
|
queuesBlacklist: Record<string, string[]>;
|
|
31
|
+
storage: IStorageConfig;
|
|
30
32
|
sentry: ISentryConfig;
|
|
31
33
|
saveRawDataWhitelist: Record<string, any>;
|
|
32
34
|
stream: {
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { ILogger } from "../../helpers";
|
|
2
|
-
import { ISimpleConfig } from "../../helpers/configuration/ISimpleConfig";
|
|
3
2
|
import { AbstractStorageService } from "../../helpers/data-access/storage";
|
|
4
3
|
import { IConfiguration } from "..";
|
|
5
4
|
export default class RawDataStorage {
|
|
@@ -7,6 +6,6 @@ export default class RawDataStorage {
|
|
|
7
6
|
private readonly log;
|
|
8
7
|
private readonly storageEnabled;
|
|
9
8
|
private readonly rawDataWhitelist;
|
|
10
|
-
constructor(config: IConfiguration, storageService: AbstractStorageService,
|
|
9
|
+
constructor(config: IConfiguration, storageService: AbstractStorageService, log: ILogger);
|
|
11
10
|
save: (data: any, meta: any, name?: string) => Promise<void>;
|
|
12
11
|
}
|
|
@@ -26,13 +26,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
26
26
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
27
27
|
const _helpers_1 = require("../../helpers");
|
|
28
28
|
const storage_1 = require("../../helpers/data-access/storage");
|
|
29
|
-
const CoreToken_1 = require("../../helpers/ioc/CoreToken");
|
|
30
29
|
const ContentTypeHelper_1 = require("../helpers/ContentTypeHelper");
|
|
31
30
|
const ioc_1 = require("../ioc");
|
|
32
31
|
const content_type_1 = __importDefault(require("content-type"));
|
|
33
32
|
const tsyringe_1 = require("tsyringe");
|
|
34
33
|
let RawDataStorage = class RawDataStorage {
|
|
35
|
-
constructor(config, storageService,
|
|
34
|
+
constructor(config, storageService, log) {
|
|
36
35
|
this.storageService = storageService;
|
|
37
36
|
this.log = log;
|
|
38
37
|
this.save = (data, meta, name = "Unknown") => __awaiter(this, void 0, void 0, function* () {
|
|
@@ -68,7 +67,7 @@ let RawDataStorage = class RawDataStorage {
|
|
|
68
67
|
yield this.storageService.uploadFile(headers, name, "json", `${fileName}_headers`);
|
|
69
68
|
}
|
|
70
69
|
});
|
|
71
|
-
this.storageEnabled =
|
|
70
|
+
this.storageEnabled = config.storage.enabled;
|
|
72
71
|
this.rawDataWhitelist = config.saveRawDataWhitelist;
|
|
73
72
|
}
|
|
74
73
|
};
|
|
@@ -76,9 +75,8 @@ RawDataStorage = __decorate([
|
|
|
76
75
|
(0, tsyringe_1.injectable)(),
|
|
77
76
|
__param(0, (0, tsyringe_1.inject)(ioc_1.ContainerToken.Config)),
|
|
78
77
|
__param(1, (0, tsyringe_1.inject)(ioc_1.ContainerToken.StorageService)),
|
|
79
|
-
__param(2, (0, tsyringe_1.inject)(
|
|
80
|
-
|
|
81
|
-
__metadata("design:paramtypes", [Object, storage_1.AbstractStorageService, Object, Object])
|
|
78
|
+
__param(2, (0, tsyringe_1.inject)(ioc_1.ContainerToken.Logger)),
|
|
79
|
+
__metadata("design:paramtypes", [Object, storage_1.AbstractStorageService, Object])
|
|
82
80
|
], RawDataStorage);
|
|
83
81
|
exports.default = RawDataStorage;
|
|
84
82
|
//# sourceMappingURL=RawDataStorage.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RawDataStorage.js","sourceRoot":"","sources":["../../../src/integration-engine/data-access/RawDataStorage.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,4CAA6C;
|
|
1
|
+
{"version":3,"file":"RawDataStorage.js","sourceRoot":"","sources":["../../../src/integration-engine/data-access/RawDataStorage.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,4CAA6C;AAC7C,+DAAsE;AAEtE,oEAAuE;AACvE,gCAAyC;AACzC,gEAAuC;AACvC,uCAA8C;AAG/B,IAAM,cAAc,GAApB,MAAM,cAAc;IAI/B,YACmC,MAAsB,EACd,cAA8C,EACtD,GAA6B;QADb,mBAAc,GAAd,cAAc,CAAwB;QACrC,QAAG,GAAH,GAAG,CAAS;QAMzD,SAAI,GAAG,CAAO,IAAS,EAAE,IAAS,EAAE,IAAI,GAAG,SAAS,EAAE,EAAE;YAC3D,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE;gBACtD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,6CAA6C,IAAI,gBAAgB,CAAC,CAAC;gBACpF,OAAO;aACV;YAED,MAAM,GAAG,GAAG,IAAA,mBAAQ,EAAC,IAAI,IAAI,EAAE,CAAC,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC;YAC9D,IAAI,GAAG,GAAG,IAAI,CAAC;YACf,IAAI,OAAO,GAAG,IAAI,CAAC;YAEnB,IAAI,IAAI,CAAC,OAAO,EAAE;gBACd,IAAI;oBACA,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;iBACnD;gBAAC,WAAM;oBACJ,IAAI,CAAC;iBACR;gBACD,IAAI;oBACA,MAAM,eAAe,GAAG,sBAAW,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;oBACxE,IAAI,eAAe,aAAf,eAAe,uBAAf,eAAe,CAAE,IAAI,EAAE;wBACvB,GAAG,GAAG,0CAAsB,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;qBACtD;iBACJ;gBAAC,WAAM;oBACJ,IAAI,CAAC;iBACR;aACJ;YAED,MAAM,QAAQ,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC;YAE1E,YAAY;YACZ,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,aAAH,GAAG,cAAH,GAAG,GAAI,EAAE,EAAE,QAAQ,CAAC,CAAC;YAEtE,IAAI,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,WAAW,IAAI,OAAO,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,KAAK,GAAG,EAAE;gBAClG,eAAe;gBACf,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,QAAQ,UAAU,CAAC,CAAC;aACtF;QACL,CAAC,CAAA,CAAC;QAvCE,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;QAC7C,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,oBAAoB,CAAC;IACxD,CAAC;CAsCJ,CAAA;AAjDoB,cAAc;IADlC,IAAA,qBAAU,GAAE;IAMJ,WAAA,IAAA,iBAAM,EAAC,oBAAc,CAAC,MAAM,CAAC,CAAA;IAC7B,WAAA,IAAA,iBAAM,EAAC,oBAAc,CAAC,cAAc,CAAC,CAAA;IACrC,WAAA,IAAA,iBAAM,EAAC,oBAAc,CAAC,MAAM,CAAC,CAAA;6CADiC,gCAAsB;GANxE,cAAc,CAiDlC;kBAjDoB,cAAc"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { ILogger } from "../../helpers";
|
|
2
|
-
import { ISimpleConfig } from "../../helpers/configuration/ISimpleConfig";
|
|
3
1
|
import { AbstractStorageService } from "../../helpers/data-access/storage";
|
|
2
|
+
import { ILogger } from "../../helpers";
|
|
3
|
+
import { IConfiguration } from "../config";
|
|
4
4
|
export declare class StorageServiceProvider {
|
|
5
5
|
protected _service: AbstractStorageService;
|
|
6
|
-
constructor(config:
|
|
6
|
+
constructor(config: IConfiguration, log: ILogger);
|
|
7
7
|
get service(): AbstractStorageService;
|
|
8
8
|
}
|
|
@@ -13,25 +13,13 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
|
13
13
|
};
|
|
14
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
15
|
exports.StorageServiceProvider = void 0;
|
|
16
|
+
const tsyringe_1 = require("tsyringe");
|
|
17
|
+
const errors_1 = require("@golemio/errors");
|
|
16
18
|
const storage_1 = require("../../helpers/data-access/storage");
|
|
17
|
-
const CoreToken_1 = require("../../helpers/ioc/CoreToken");
|
|
18
19
|
const ioc_1 = require("../ioc");
|
|
19
|
-
const errors_1 = require("@golemio/errors");
|
|
20
|
-
const tsyringe_1 = require("tsyringe");
|
|
21
20
|
let StorageServiceProvider = exports.StorageServiceProvider = class StorageServiceProvider {
|
|
22
21
|
constructor(config, log) {
|
|
23
|
-
const service = new storage_1.StorageServiceFactory(
|
|
24
|
-
enabled: config.getValue("env.STORAGE_ENABLED", "false") === "true",
|
|
25
|
-
provider: {
|
|
26
|
-
azure: {
|
|
27
|
-
clientId: config.getValue("env.AZURE_BLOB_CLIENT_ID", ""),
|
|
28
|
-
tenantId: config.getValue("env.AZURE_BLOB_TENANT_ID", ""),
|
|
29
|
-
clientSecret: config.getValue("env.AZURE_BLOB_CLIENT_SECRET", ""),
|
|
30
|
-
account: config.getValue("env.AZURE_BLOB_ACCOUNT", ""),
|
|
31
|
-
containerName: config.getValue("env.AZURE_BLOB_STORAGE_CONTAINER_NAME", ""),
|
|
32
|
-
},
|
|
33
|
-
},
|
|
34
|
-
}, log).getService(storage_1.StorageProvider.Azure);
|
|
22
|
+
const service = new storage_1.StorageServiceFactory(config.storage, log).getService(storage_1.StorageProvider.Azure);
|
|
35
23
|
if (!service) {
|
|
36
24
|
throw new errors_1.FatalError(`StorageServiceProvider returned undefined for provider ${storage_1.StorageProvider.Azure}`, this.constructor.name);
|
|
37
25
|
}
|
|
@@ -43,7 +31,7 @@ let StorageServiceProvider = exports.StorageServiceProvider = class StorageServi
|
|
|
43
31
|
};
|
|
44
32
|
exports.StorageServiceProvider = StorageServiceProvider = __decorate([
|
|
45
33
|
(0, tsyringe_1.injectable)(),
|
|
46
|
-
__param(0, (0, tsyringe_1.inject)(
|
|
34
|
+
__param(0, (0, tsyringe_1.inject)(ioc_1.ContainerToken.Config)),
|
|
47
35
|
__param(1, (0, tsyringe_1.inject)(ioc_1.ContainerToken.Logger)),
|
|
48
36
|
__metadata("design:paramtypes", [Object, Object])
|
|
49
37
|
], StorageServiceProvider);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"StorageServiceProvider.js","sourceRoot":"","sources":["../../../src/integration-engine/data-access/StorageServiceProvider.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"StorageServiceProvider.js","sourceRoot":"","sources":["../../../src/integration-engine/data-access/StorageServiceProvider.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,uCAA8C;AAC9C,4CAA6C;AAC7C,+DAA8G;AAC9G,gCAAyC;AAKlC,IAAM,sBAAsB,oCAA5B,MAAM,sBAAsB;IAG/B,YAA2C,MAAsB,EAAiC,GAAY;QAC1G,MAAM,OAAO,GAAG,IAAI,+BAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,yBAAe,CAAC,KAAK,CAAC,CAAC;QAEjG,IAAI,CAAC,OAAO,EAAE;YACV,MAAM,IAAI,mBAAU,CAChB,0DAA0D,yBAAe,CAAC,KAAK,EAAE,EACjF,IAAI,CAAC,WAAW,CAAC,IAAI,CACxB,CAAC;SACL;QAED,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;IAC5B,CAAC;IAED,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC;IACzB,CAAC;CACJ,CAAA;iCAnBY,sBAAsB;IADlC,IAAA,qBAAU,GAAE;IAII,WAAA,IAAA,iBAAM,EAAC,oBAAc,CAAC,MAAM,CAAC,CAAA;IAA0B,WAAA,IAAA,iBAAM,EAAC,oBAAc,CAAC,MAAM,CAAC,CAAA;;GAHxF,sBAAsB,CAmBlC"}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { LoggerEmitter } from "../../helpers/logger";
|
|
2
2
|
import { DataSourceStream, IDataSource, IDataTypeStrategy, IProtocolStrategy } from "./";
|
|
3
|
-
import { ResourceNotModified } from "../helpers/ResourceNotModified";
|
|
4
3
|
import { IValidator } from "@golemio/validator";
|
|
5
4
|
export declare class DataSource<T = any> implements IDataSource<T> {
|
|
6
5
|
dataStream: DataSourceStream;
|
|
@@ -14,16 +13,8 @@ export declare class DataSource<T = any> implements IDataSource<T> {
|
|
|
14
13
|
setDataTypeStrategy: (strategy: IDataTypeStrategy | null) => void;
|
|
15
14
|
setValidator: (validator: IValidator | null) => void;
|
|
16
15
|
getAll(): Promise<T>;
|
|
17
|
-
getAllWithMetadata(): Promise<{
|
|
18
|
-
data: T | ResourceNotModified;
|
|
19
|
-
metadata: unknown;
|
|
20
|
-
}>;
|
|
21
16
|
getLastModified(): Promise<string>;
|
|
22
|
-
protected getRawData()
|
|
23
|
-
data: T | ResourceNotModified;
|
|
24
|
-
metadata: unknown;
|
|
25
|
-
}>;
|
|
26
|
-
private getResponseStatusCode;
|
|
17
|
+
protected getRawData: () => Promise<T>;
|
|
27
18
|
protected isEmpty: (content: T) => boolean;
|
|
28
19
|
private notifyNumberOfRecords;
|
|
29
20
|
private formatDataForError;
|
|
@@ -13,7 +13,6 @@ exports.DataSource = void 0;
|
|
|
13
13
|
const logger_1 = require("../../helpers/logger");
|
|
14
14
|
const datasources_1 = require("./");
|
|
15
15
|
const helpers_1 = require("../helpers");
|
|
16
|
-
const ResourceNotModified_1 = require("../helpers/ResourceNotModified");
|
|
17
16
|
const ioc_1 = require("../ioc");
|
|
18
17
|
const errors_1 = require("@golemio/errors");
|
|
19
18
|
class DataSource {
|
|
@@ -31,6 +30,19 @@ class DataSource {
|
|
|
31
30
|
this.setValidator = (validator) => {
|
|
32
31
|
this.validator = validator;
|
|
33
32
|
};
|
|
33
|
+
this.getRawData = () => __awaiter(this, void 0, void 0, function* () {
|
|
34
|
+
var _b;
|
|
35
|
+
try {
|
|
36
|
+
const body = yield this.protocolStrategy.getData();
|
|
37
|
+
const isFile = (0, datasources_1.isFileObject)(body);
|
|
38
|
+
const content = yield ((_b = this.dataTypeStrategy) === null || _b === void 0 ? void 0 : _b.parseData(isFile ? body.data : body));
|
|
39
|
+
this.notifyNumberOfRecords(content);
|
|
40
|
+
return isFile ? Object.assign(Object.assign({}, body), { data: content }) : content;
|
|
41
|
+
}
|
|
42
|
+
catch (err) {
|
|
43
|
+
throw new errors_1.GeneralError("Retrieving of the source data failed.", this.name, err);
|
|
44
|
+
}
|
|
45
|
+
});
|
|
34
46
|
this.isEmpty = (content) => {
|
|
35
47
|
if (!content) {
|
|
36
48
|
return true;
|
|
@@ -84,29 +96,19 @@ class DataSource {
|
|
|
84
96
|
}
|
|
85
97
|
getAll() {
|
|
86
98
|
return __awaiter(this, void 0, void 0, function* () {
|
|
87
|
-
const
|
|
88
|
-
if (
|
|
89
|
-
|
|
99
|
+
const data = yield this.getRawData();
|
|
100
|
+
if (this.validator) {
|
|
101
|
+
try {
|
|
102
|
+
yield this.validator.Validate((0, datasources_1.isFileObject)(data) ? data.data : data);
|
|
103
|
+
}
|
|
104
|
+
catch (err) {
|
|
105
|
+
throw new errors_1.GeneralError(`Error while validating source data: ${this.formatDataForError(data)}`, this.name, err);
|
|
106
|
+
}
|
|
90
107
|
}
|
|
91
|
-
|
|
92
|
-
});
|
|
93
|
-
}
|
|
94
|
-
getAllWithMetadata() {
|
|
95
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
96
|
-
const res = yield this.getRawData();
|
|
97
|
-
if (!this.validator) {
|
|
108
|
+
else {
|
|
98
109
|
helpers_1.log.warn("DataSource validator is not set.");
|
|
99
|
-
return res;
|
|
100
|
-
}
|
|
101
|
-
if (res.data instanceof ResourceNotModified_1.ResourceNotModified)
|
|
102
|
-
return res;
|
|
103
|
-
try {
|
|
104
|
-
yield this.validator.Validate((0, datasources_1.isFileObject)(res.data) ? res.data.data : res.data);
|
|
105
110
|
}
|
|
106
|
-
|
|
107
|
-
throw new errors_1.GeneralError(`Error while validating source data: ${this.formatDataForError(res.data)}`, this.name, err);
|
|
108
|
-
}
|
|
109
|
-
return res;
|
|
111
|
+
return data;
|
|
110
112
|
});
|
|
111
113
|
}
|
|
112
114
|
getLastModified() {
|
|
@@ -114,30 +116,6 @@ class DataSource {
|
|
|
114
116
|
return this.protocolStrategy.getLastModified();
|
|
115
117
|
});
|
|
116
118
|
}
|
|
117
|
-
getRawData() {
|
|
118
|
-
var _a;
|
|
119
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
120
|
-
try {
|
|
121
|
-
const res = yield this.protocolStrategy.getDataWithMetadata();
|
|
122
|
-
if (this.getResponseStatusCode(res.metadata) === 304) {
|
|
123
|
-
return { data: new ResourceNotModified_1.ResourceNotModified(), metadata: res.metadata };
|
|
124
|
-
}
|
|
125
|
-
const body = res.data;
|
|
126
|
-
const isFile = (0, datasources_1.isFileObject)(body);
|
|
127
|
-
const content = yield ((_a = this.dataTypeStrategy) === null || _a === void 0 ? void 0 : _a.parseData(isFile ? body.data : body));
|
|
128
|
-
this.notifyNumberOfRecords(content);
|
|
129
|
-
return isFile ? { data: Object.assign(Object.assign({}, body), { content }), metadata: res.metadata } : { data: content, metadata: res.metadata };
|
|
130
|
-
}
|
|
131
|
-
catch (err) {
|
|
132
|
-
throw new errors_1.GeneralError("Retrieving of the source data failed.", this.name, err);
|
|
133
|
-
}
|
|
134
|
-
});
|
|
135
|
-
}
|
|
136
|
-
getResponseStatusCode(metadata) {
|
|
137
|
-
if (typeof metadata === "object" && metadata !== null && "statusCode" in metadata && metadata.statusCode === 304) {
|
|
138
|
-
return metadata.statusCode;
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
119
|
}
|
|
142
120
|
exports.DataSource = DataSource;
|
|
143
121
|
//# sourceMappingURL=DataSource.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DataSource.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/DataSource.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,iDAAiE;AACjE,oCAAoH;AACpH,wCAAkC;AAClC,
|
|
1
|
+
{"version":3,"file":"DataSource.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/DataSource.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,iDAAiE;AACjE,oCAAoH;AACpH,wCAAkC;AAClC,gCAAqE;AACrE,4CAA+C;AAG/C,MAAa,UAAU;IAQnB,YACI,IAAY,EACZ,gBAAmC,EACnC,mBAA6C,IAAI,EACjD,YAA+B,IAAI;;QAkBhC,wBAAmB,GAAG,CAAC,QAA2B,EAAQ,EAAE;YAC/D,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAC;YACjC,IAAI,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE;gBACrC,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aAClD;QACL,CAAC,CAAC;QAEK,wBAAmB,GAAG,CAAC,QAAkC,EAAQ,EAAE;YACtE,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAC;QACrC,CAAC,CAAC;QAEK,iBAAY,GAAG,CAAC,SAA4B,EAAQ,EAAE;YACzD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC/B,CAAC,CAAC;QAqBQ,eAAU,GAAG,GAAqB,EAAE;;YAC1C,IAAI;gBACA,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC;gBAEnD,MAAM,MAAM,GAAG,IAAA,0BAAY,EAAC,IAAI,CAAC,CAAC;gBAElC,MAAM,OAAO,GAAG,MAAM,CAAA,MAAA,IAAI,CAAC,gBAAgB,0CAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA,CAAC;gBAClF,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO,MAAM,CAAC,CAAC,iCAAM,IAAI,KAAE,IAAI,EAAE,OAAO,IAAG,CAAC,CAAC,OAAO,CAAC;aACxD;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,qBAAY,CAAC,uCAAuC,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aACnF;QACL,CAAC,CAAA,CAAC;QAEQ,YAAO,GAAG,CAAC,OAAU,EAAW,EAAE;YACxC,IAAI,CAAC,OAAO,EAAE;gBACV,OAAO,IAAI,CAAC;aACf;iBAAM,IAAI,OAAO,YAAY,KAAK,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;gBACzD,OAAO,IAAI,CAAC;aACf;iBAAM,IAAI,OAAO,YAAY,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE;gBACvE,OAAO,IAAI,CAAC;aACf;YACD,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC;QAEM,0BAAqB,GAAG,CAAC,OAAU,EAAE,EAAE;YAC3C,IAAI,eAAe,GAAG,CAAC,CAAC;YAExB,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;gBACvB,aAAG,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,IAAI,oCAAoC,CAAC,CAAC;aAClE;iBAAM;gBACH,eAAe,GAAG,OAAO,YAAY,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;aACnE;YAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,wBAAe,CAAC,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;QACnG,CAAC,CAAC;QAEM,uBAAkB,GAAG,CAAC,IAAS,EAAU,EAAE;YAC/C,IAAI,OAAO,GAAG,EAAE,CAAC;YAEjB,IAAI;gBACA,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;aAClC;YAAC,WAAM;gBACJ,IAAI,IAAI,CAAC,QAAQ,EAAE;oBACf,IAAI;wBACA,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;qBAC7B;oBAAC,WAAM;wBACJ,IAAI,CAAC;qBACR;iBACJ;aACJ;YAED,OAAO,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC;QAC5E,CAAC,CAAC;QAvGE,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAE3B,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE;YAC9D,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SAClD;QAED,IAAI,MAAA,IAAI,CAAC,SAAS,0CAAE,SAAS,EAAE;YAC3B,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,aAAG,CAAC,CAAC;SACjC;QAED,IAAI,CAAC,aAAa,GAAG,gCAA0B,CAAC,OAAO,CAAgB,oBAAc,CAAC,aAAa,CAAC,CAAC;IACzG,CAAC;IAiBY,MAAM;;YACf,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;YAErC,IAAI,IAAI,CAAC,SAAS,EAAE;gBAChB,IAAI;oBACA,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAA,0BAAY,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;iBACxE;gBAAC,OAAO,GAAG,EAAE;oBACV,MAAM,IAAI,qBAAY,CAAC,uCAAuC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;iBAClH;aACJ;iBAAM;gBACH,aAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;aAChD;YACD,OAAO,IAAI,CAAC;QAChB,CAAC;KAAA;IAEY,eAAe;;YACxB,OAAO,IAAI,CAAC,gBAAgB,CAAC,eAAe,EAAE,CAAC;QACnD,CAAC;KAAA;CAwDJ;AAtHD,gCAsHC"}
|
|
@@ -2,11 +2,7 @@ import { DataSource, DataSourceStream, IDataSource } from "./";
|
|
|
2
2
|
export declare class DataSourceStreamed extends DataSource implements IDataSource {
|
|
3
3
|
private dataBuffer;
|
|
4
4
|
proceed: () => Promise<void>;
|
|
5
|
-
getAll(useDataBuffer?: boolean)
|
|
6
|
-
getAllWithMetadata(useDataBuffer?: boolean): Promise<{
|
|
7
|
-
data: DataSourceStream;
|
|
8
|
-
metadata: unknown;
|
|
9
|
-
}>;
|
|
5
|
+
getAll: (useDataBuffer?: boolean) => Promise<DataSourceStream>;
|
|
10
6
|
/**
|
|
11
7
|
* @param {boolean} useDataBuffer data is buffered and sent to output stream in `config.DATA_BATCH_SIZE` batches
|
|
12
8
|
*/
|
|
@@ -22,6 +22,28 @@ class DataSourceStreamed extends datasources_1.DataSource {
|
|
|
22
22
|
this.proceed = () => __awaiter(this, void 0, void 0, function* () {
|
|
23
23
|
return this.dataStream.proceed();
|
|
24
24
|
});
|
|
25
|
+
this.getAll = (useDataBuffer = false) => __awaiter(this, void 0, void 0, function* () {
|
|
26
|
+
this.dataStream = yield this.getOutputStream(useDataBuffer);
|
|
27
|
+
this.dataStream.on("end", () => {
|
|
28
|
+
this.dataStream.destroy();
|
|
29
|
+
});
|
|
30
|
+
this.dataStream.onDataListeners.push((data) => __awaiter(this, void 0, void 0, function* () {
|
|
31
|
+
if (this.validator) {
|
|
32
|
+
try {
|
|
33
|
+
this.dataStream.pause();
|
|
34
|
+
yield this.validator.Validate(data);
|
|
35
|
+
this.dataStream.resume();
|
|
36
|
+
}
|
|
37
|
+
catch (err) {
|
|
38
|
+
this.dataStream.destroy(new errors_1.GeneralError("Error while validating source data.", this.name, err));
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
helpers_1.log.warn("DataSource validator is not set.");
|
|
43
|
+
}
|
|
44
|
+
}));
|
|
45
|
+
return this.dataStream;
|
|
46
|
+
});
|
|
25
47
|
/**
|
|
26
48
|
* @param {boolean} useDataBuffer data is buffered and sent to output stream in `config.DATA_BATCH_SIZE` batches
|
|
27
49
|
*/
|
|
@@ -108,36 +130,6 @@ class DataSourceStreamed extends datasources_1.DataSource {
|
|
|
108
130
|
}
|
|
109
131
|
});
|
|
110
132
|
}
|
|
111
|
-
getAll(useDataBuffer = false) {
|
|
112
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
113
|
-
const { data } = yield this.getAllWithMetadata(useDataBuffer);
|
|
114
|
-
return data;
|
|
115
|
-
});
|
|
116
|
-
}
|
|
117
|
-
getAllWithMetadata(useDataBuffer = false) {
|
|
118
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
119
|
-
this.dataStream = yield this.getOutputStream(useDataBuffer);
|
|
120
|
-
this.dataStream.on("end", () => {
|
|
121
|
-
this.dataStream.destroy();
|
|
122
|
-
});
|
|
123
|
-
this.dataStream.onDataListeners.push((data) => __awaiter(this, void 0, void 0, function* () {
|
|
124
|
-
if (this.validator) {
|
|
125
|
-
try {
|
|
126
|
-
this.dataStream.pause();
|
|
127
|
-
yield this.validator.Validate(data);
|
|
128
|
-
this.dataStream.resume();
|
|
129
|
-
}
|
|
130
|
-
catch (err) {
|
|
131
|
-
this.dataStream.destroy(new errors_1.GeneralError("Error while validating source data.", this.name, err));
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
else {
|
|
135
|
-
helpers_1.log.warn("DataSource validator is not set.");
|
|
136
|
-
}
|
|
137
|
-
}));
|
|
138
|
-
return { data: this.dataStream, metadata: undefined };
|
|
139
|
-
});
|
|
140
|
-
}
|
|
141
133
|
}
|
|
142
134
|
exports.DataSourceStreamed = DataSourceStreamed;
|
|
143
135
|
//# sourceMappingURL=DataSourceStreamed.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DataSourceStreamed.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/DataSourceStreamed.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,iDAAkD;AAClD,sCAAoC;AACpC,oCAA4E;AAC5E,wCAAkC;AAClC,4CAAqE;AAErE,MAAa,kBAAmB,SAAQ,wBAAU;IAAlD;;QACY,eAAU,GAAU,EAAE,CAAC;QAExB,YAAO,GAAG,GAAwB,EAAE;YACvC,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QACrC,CAAC,CAAA,CAAC;
|
|
1
|
+
{"version":3,"file":"DataSourceStreamed.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/DataSourceStreamed.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,iDAAkD;AAClD,sCAAoC;AACpC,oCAA4E;AAC5E,wCAAkC;AAClC,4CAAqE;AAErE,MAAa,kBAAmB,SAAQ,wBAAU;IAAlD;;QACY,eAAU,GAAU,EAAE,CAAC;QAExB,YAAO,GAAG,GAAwB,EAAE;YACvC,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QACrC,CAAC,CAAA,CAAC;QAEK,WAAM,GAAG,CAAO,aAAa,GAAG,KAAK,EAA6B,EAAE;YACvE,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC;YAE5D,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBAC3B,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;YAC9B,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAO,IAAS,EAAE,EAAE;gBACrD,IAAI,IAAI,CAAC,SAAS,EAAE;oBAChB,IAAI;wBACA,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;wBACxB,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;wBACpC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;qBAC5B;oBAAC,OAAO,GAAG,EAAE;wBACV,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,qBAAY,CAAC,qCAAqC,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;qBACpG;iBACJ;qBAAM;oBACH,aAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;iBAChD;YACL,CAAC,CAAA,CAAC,CAAC;YAEH,OAAO,IAAI,CAAC,UAAU,CAAC;QAC3B,CAAC,CAAA,CAAC;QAEF;;WAEG;QACO,oBAAe,GAAG,CAAO,gBAAyB,KAAK,EAA6B,EAAE;YAC5F,IAAI,wBAAwB,GAAG,CAAC,CAAC;YACjC,IAAI,CAAC,UAAU,GAAG,IAAI,8BAAgB,CAAC;gBACnC,UAAU,EAAE,IAAI;gBAChB,IAAI,EAAE,GAAG,EAAE;oBACP,OAAO;gBACX,CAAC;aACJ,CAAC,CAAC;YAEH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC;YAE1D,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBACnC,IAAI,KAAK,YAAY,6BAAoB,EAAE;oBACvC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;iBACxC;qBAAM;oBACH,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,kCAAkC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;iBACrH;YACL,CAAC,CAAC,CAAC;YAEH,WAAW,CAAC,eAAe,CAAC,IAAI,CAAC,CAAO,IAAS,EAAiB,EAAE;gBAChE,WAAW,CAAC,KAAK,EAAE,CAAC;gBAEpB,IAAI,aAAa,EAAE;oBACf,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC3B,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;iBAC5B;qBAAM;oBACH,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;iBACvC;gBAED,WAAW,CAAC,MAAM,EAAE,CAAC;YACzB,CAAC,CAAA,CAAC,CAAC;YAEH,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,GAAwB,EAAE;gBAC5C,IAAI,aAAa,EAAE;oBACf,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;iBAChC;gBAED,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE;oBACzB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBAC9B;qBAAM;oBACH,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE;wBAC7B,wBAAwB,EAAE,CAAC;wBAC3B,kCAAkC;wBAClC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE;4BACzB,aAAa,CAAC,OAAO,CAAC,CAAC;4BACvB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;yBAC9B;6BAAM,IAAI,wBAAwB,GAAG,eAAM,CAAC,MAAM,CAAC,qBAAqB,EAAE;4BACvE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,kCAAkC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;4BAC/F,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;4BAC3B,aAAa,CAAC,OAAO,CAAC,CAAC;yBAC1B;oBACL,CAAC,EAAE,eAAM,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;iBAC3C;YACL,CAAC,CAAA,CAAC,CAAC;YAEH,WAAW,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAEpF,OAAO,IAAI,CAAC,UAAU,CAAC;QAC3B,CAAC,CAAA,CAAC;QAEM,gBAAW,GAAG,CAAO,KAAK,GAAG,KAAK,EAAE,IAAI,GAAG,IAAI,EAAiB,EAAE;;YACtE,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,eAAM,CAAC,eAAe,IAAI,KAAK,IAAI,IAAI,EAAE;gBACnE,IAAI;oBACA,IAAI,OAAY,CAAC;oBAEjB,IAAI,MAAA,IAAI,CAAC,gBAAgB,0CAAE,SAAS,EAAE;wBAClC,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC;qBAC5E;yBAAM;wBACH,OAAO,GAAG,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC;qBACrC;oBAED,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;wBACvB,aAAG,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,IAAI,oCAAoC,CAAC,CAAC;wBAC/D,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,wBAAe,CAAC,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC,CAAC;qBACrG;yBAAM;wBACH,MAAM,eAAe,GAAG,OAAO,YAAY,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;wBACtE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,wBAAe,CAAC,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;wBAE/F,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;wBAC9B,mBAAmB;wBACnB,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;qBAC9B;iBACJ;gBAAC,OAAO,GAAG,EAAE;oBACV,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,uCAAuC,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;iBAC5G;aACJ;QACL,CAAC,CAAA,CAAC;IACN,CAAC;CAAA;AAzHD,gDAyHC"}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import { DataSourceStream, IDataTypeStrategy, IProtocolStrategy } from "./";
|
|
2
|
-
import { ResourceNotModified } from "../helpers/ResourceNotModified";
|
|
3
1
|
import { IValidator } from "@golemio/validator";
|
|
2
|
+
import { IDataTypeStrategy, IProtocolStrategy, DataSourceStream } from "./";
|
|
4
3
|
export interface IDataSource<T = any> {
|
|
5
4
|
/** The name of the data source. */
|
|
6
5
|
name: string;
|
|
@@ -9,18 +8,10 @@ export interface IDataSource<T = any> {
|
|
|
9
8
|
setDataTypeStrategy(strategy: IDataTypeStrategy): void;
|
|
10
9
|
setValidator(validator: IValidator): void;
|
|
11
10
|
/**
|
|
12
|
-
*
|
|
11
|
+
* Gets raw data, validates them and sends as a response.
|
|
13
12
|
*
|
|
14
13
|
* @returns {Promise<T>} Promise with received data.
|
|
15
14
|
*/
|
|
16
15
|
getAll(): Promise<T>;
|
|
17
|
-
/**
|
|
18
|
-
* Get raw data, validate them and return as a response, including any metadata such as HTTP headers. If the data has not been
|
|
19
|
-
* modified (HTTP 304 Not Modified response was received), return `ResourceNotModified`.
|
|
20
|
-
*/
|
|
21
|
-
getAllWithMetadata(): Promise<{
|
|
22
|
-
data: T | ResourceNotModified;
|
|
23
|
-
metadata: unknown;
|
|
24
|
-
}>;
|
|
25
16
|
getLastModified(): Promise<string>;
|
|
26
17
|
}
|
package/dist/integration-engine/datasources/protocol-strategy/HTTPFetchProtocolStrategy.d.ts
CHANGED
|
@@ -4,7 +4,7 @@ import { IHTTPProtocolStrategyResult } from "./interfaces/IHTTPProtocolStrategyR
|
|
|
4
4
|
export declare class HTTPFetchProtocolStrategy<T = any> extends ProtocolStrategy {
|
|
5
5
|
protected connectionSettings: IHTTPFetchSettings;
|
|
6
6
|
constructor(settings: IHTTPFetchSettings);
|
|
7
|
-
getRawData(): Promise<IHTTPProtocolStrategyResult<T
|
|
7
|
+
getRawData(): Promise<IHTTPProtocolStrategyResult<T>>;
|
|
8
8
|
private getHeaders;
|
|
9
9
|
private prepareAbortController;
|
|
10
10
|
}
|
|
@@ -36,15 +36,6 @@ class HTTPFetchProtocolStrategy extends ProtocolStrategy_1.ProtocolStrategy {
|
|
|
36
36
|
});
|
|
37
37
|
const headers = this.getHeaders(result.headers);
|
|
38
38
|
statusCode = result.status;
|
|
39
|
-
if (statusCode === 304) {
|
|
40
|
-
return {
|
|
41
|
-
data: null,
|
|
42
|
-
meta: {
|
|
43
|
-
headers,
|
|
44
|
-
statusCode,
|
|
45
|
-
},
|
|
46
|
-
};
|
|
47
|
-
}
|
|
48
39
|
let data;
|
|
49
40
|
if (this.connectionSettings.responseType === "json") {
|
|
50
41
|
data = (yield result.json());
|
package/dist/integration-engine/datasources/protocol-strategy/HTTPFetchProtocolStrategy.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"HTTPFetchProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/HTTPFetchProtocolStrategy.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,yDAAsF;AACtF,4CAA+C;AAI/C,MAAa,yBAAmC,SAAQ,mCAAgB;IAGpE,YAAY,QAA4B;QACpC,KAAK,CAAC,QAAQ,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"HTTPFetchProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/HTTPFetchProtocolStrategy.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,yDAAsF;AACtF,4CAA+C;AAI/C,MAAa,yBAAmC,SAAQ,mCAAgB;IAGpE,YAAY,QAA4B;QACpC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAiDZ,eAAU,GAAG,CAAC,OAAgB,EAA0B,EAAE;YAC9D,MAAM,aAAa,GAA2B,EAAE,CAAC;YAEjD,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;gBAC3B,aAAa,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YAC/B,CAAC,CAAC,CAAC;YAEH,OAAO,aAAa,CAAC;QACzB,CAAC,CAAC;IAxDF,CAAC;IAEY,UAAU;;YACnB,IAAI,UAA8B,CAAC;YACnC,IAAI;gBACA,MAAM,eAAe,GAAG,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBAEtD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE;oBACpD,IAAI,EAAE,IAAI,CAAC,kBAAkB,CAAC,IAAI;oBAClC,OAAO,oBACC,IAAI,CAAC,kBAAkB,CAAC,OAAkC,CACjE;oBACD,MAAM,EAAE,IAAI,CAAC,kBAAkB,CAAC,MAAM;oBACtC,MAAM,EAAE,eAAe,aAAf,eAAe,uBAAf,eAAe,CAAE,MAAM;iBAClC,CAAC,CAAC;gBAEH,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBAChD,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;gBAC3B,IAAI,IAAO,CAAC;gBAEZ,IAAI,IAAI,CAAC,kBAAkB,CAAC,YAAY,KAAK,MAAM,EAAE;oBACjD,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAM,CAAC;iBACrC;qBAAM,IAAI,IAAI,CAAC,kBAAkB,CAAC,YAAY,KAAK,MAAM,EAAE;oBACxD,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAM,CAAC;iBACrC;qBAAM,IAAI,IAAI,CAAC,kBAAkB,CAAC,YAAY,KAAK,MAAM,EAAE;oBACxD,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAM,CAAC;iBACrC;qBAAM,IAAI,IAAI,CAAC,kBAAkB,CAAC,YAAY,KAAK,aAAa,EAAE;oBAC/D,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,WAAW,EAAE,CAAM,CAAC;iBAC5C;qBAAM;oBACH,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAM,CAAC;iBACrC;gBAED,OAAO;oBACH,IAAI;oBACJ,IAAI,EAAE;wBACF,OAAO;wBACP,UAAU;qBACb;iBACJ,CAAC;aACL;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,qBAAY,CAClB,+DAA+D,UAAU,EAAE,EAC3E,IAAI,CAAC,WAAW,CAAC,IAAI,EACrB,GAAG,CACN,CAAC;aACL;QACL,CAAC;KAAA;IAYO,sBAAsB;QAC1B,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;YAC3C,OAAO,IAAI,CAAC;SACf;QAED,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;QAC9C,UAAU,CAAC,GAAG,EAAE,CAAC,eAAe,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC;QAE3F,OAAO,eAAe,CAAC;IAC3B,CAAC;CACJ;AAzED,8DAyEC"}
|
|
@@ -110,12 +110,6 @@ export interface IProtocolStrategy {
|
|
|
110
110
|
setCallerName(caller: string): void;
|
|
111
111
|
getConnectionSettings(): ProtocolStrategySettings;
|
|
112
112
|
setConnectionSettings(settings: ProtocolStrategySettings): void;
|
|
113
|
-
/** Get data */
|
|
114
113
|
getData(): Promise<any | DataSourceStream>;
|
|
115
|
-
/** Get data including any metadata such as HTTP headers */
|
|
116
|
-
getDataWithMetadata(): Promise<{
|
|
117
|
-
data: unknown | DataSourceStream;
|
|
118
|
-
metadata: unknown;
|
|
119
|
-
}>;
|
|
120
114
|
getLastModified(): Promise<any>;
|
|
121
115
|
}
|
package/dist/integration-engine/datasources/protocol-strategy/PaginatedHTTPProtocolStrategy.js
CHANGED
|
@@ -24,19 +24,11 @@ class PaginatedHTTPProtocolStrategy extends HTTPFetchProtocolStrategy_1.HTTPFetc
|
|
|
24
24
|
var _a, _b;
|
|
25
25
|
return __awaiter(this, void 0, void 0, function* () {
|
|
26
26
|
try {
|
|
27
|
-
let
|
|
28
|
-
if (res.data === null) {
|
|
29
|
-
throw new errors_1.GeneralError("No data", this.constructor.name);
|
|
30
|
-
}
|
|
31
|
-
let resultPage = res;
|
|
27
|
+
let resultPage = yield _super.getRawData.call(this);
|
|
32
28
|
const result = resultPage;
|
|
33
29
|
while ((_a = resultPage.data.properties) === null || _a === void 0 ? void 0 : _a.exceededTransferLimit) {
|
|
34
30
|
this.setNextPageUrl();
|
|
35
|
-
|
|
36
|
-
if (res.data === null) {
|
|
37
|
-
throw new errors_1.GeneralError("No data", this.constructor.name);
|
|
38
|
-
}
|
|
39
|
-
resultPage = res;
|
|
31
|
+
resultPage = yield _super.getRawData.call(this);
|
|
40
32
|
result.data.features.push(...resultPage.data.features);
|
|
41
33
|
}
|
|
42
34
|
(_b = result.data.properties) === null || _b === void 0 ? true : delete _b.exceededTransferLimit;
|
package/dist/integration-engine/datasources/protocol-strategy/PaginatedHTTPProtocolStrategy.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PaginatedHTTPProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/PaginatedHTTPProtocolStrategy.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,qEAAgE;AAChE,4CAA+C;AAC/C,2EAAwE;AAYxE,MAAa,6BAAuC,SAAQ,qDAAuD;IAC/G,YAAY,QAA4B;QACpC,KAAK,iCAAM,QAAQ,KAAE,GAAG,EAAE,mCAAgB,CAAC,mBAAmB,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAG,CAAC;IACpF,CAAC;IAEY,UAAU;;;;;;YACnB,IAAI;gBACA,IAAI,
|
|
1
|
+
{"version":3,"file":"PaginatedHTTPProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/PaginatedHTTPProtocolStrategy.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,qEAAgE;AAChE,4CAA+C;AAC/C,2EAAwE;AAYxE,MAAa,6BAAuC,SAAQ,qDAAuD;IAC/G,YAAY,QAA4B;QACpC,KAAK,iCAAM,QAAQ,KAAE,GAAG,EAAE,mCAAgB,CAAC,mBAAmB,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAG,CAAC;IACpF,CAAC;IAEY,UAAU;;;;;;YACnB,IAAI;gBACA,IAAI,UAAU,GAA8D,MAAM,OAAM,UAAU,WAAE,CAAC;gBACrG,MAAM,MAAM,GAAG,UAAU,CAAC;gBAE1B,OAAO,MAAA,UAAU,CAAC,IAAI,CAAC,UAAU,0CAAE,qBAAqB,EAAE;oBACtD,IAAI,CAAC,cAAc,EAAE,CAAC;oBACtB,UAAU,GAAG,MAAM,OAAM,UAAU,WAAE,CAAC;oBACtC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;iBAC1D;gBAEM,MAAA,MAAM,CAAC,IAAI,CAAC,UAAU,+CAAE,qBAAqB,CAAC;gBACrD,IAAI,CAAC,cAAc,EAAE,CAAC;gBAEtB,OAAO,MAAM,CAAC;aACjB;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,qBAAY,CAAC,uCAAuC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aAC/F;;KACJ;IAEO,cAAc;QAClB,IAAI,CAAC,qBAAqB,iCACnB,IAAI,CAAC,kBAAkB,KAC1B,GAAG,EAAE,mCAAgB,CAAC,cAAc,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IACnE,CAAC;IACP,CAAC;IAEO,cAAc;QAClB,IAAI,CAAC,qBAAqB,iCACnB,IAAI,CAAC,kBAAkB,KAC1B,GAAG,EAAE,mCAAgB,CAAC,cAAc,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IACnE,CAAC;IACP,CAAC;CACJ;AAtCD,sEAsCC"}
|
|
@@ -3,19 +3,11 @@ import { DataSourceStream } from "../DataSourceStream";
|
|
|
3
3
|
export declare class ProtocolStrategy implements IProtocolStrategy {
|
|
4
4
|
protected connectionSettings: any;
|
|
5
5
|
private caller;
|
|
6
|
-
private storageEnabled;
|
|
7
|
-
private saveRawDataWhitelist;
|
|
8
|
-
private logger;
|
|
9
|
-
private rawDataStorage;
|
|
10
6
|
constructor(settings: any);
|
|
11
7
|
setCallerName: (caller?: string) => void;
|
|
12
8
|
setConnectionSettings(settings: any): void;
|
|
13
9
|
getConnectionSettings: () => ProtocolStrategySettings;
|
|
14
10
|
getData(...args: any[]): Promise<any | DataSourceStream>;
|
|
15
|
-
getDataWithMetadata(...args: unknown[]): Promise<{
|
|
16
|
-
data: unknown | DataSourceStream;
|
|
17
|
-
metadata: unknown;
|
|
18
|
-
}>;
|
|
19
11
|
getLastModified: () => Promise<string | null>;
|
|
20
12
|
protected getRawData(...args: any): Promise<any | DataSourceStream>;
|
|
21
13
|
private prepareRawData;
|
|
@@ -1,4 +1,27 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
2
25
|
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
26
|
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
27
|
return new (P || (P = Promise))(function (resolve, reject) {
|
|
@@ -8,15 +31,11 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
8
31
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
32
|
});
|
|
10
33
|
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
34
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
35
|
exports.ProtocolStrategy = void 0;
|
|
16
|
-
const
|
|
17
|
-
const
|
|
18
|
-
const
|
|
19
|
-
const Di_1 = require("../../ioc/Di");
|
|
36
|
+
const config_1 = require("../../config");
|
|
37
|
+
const helpers_1 = require("../../helpers");
|
|
38
|
+
const RawDataStore = __importStar(require("../../helpers/RawDataStore"));
|
|
20
39
|
const errors_1 = require("@golemio/errors");
|
|
21
40
|
const stream_1 = require("stream");
|
|
22
41
|
const DataSourceStream_1 = require("../DataSourceStream");
|
|
@@ -65,7 +84,7 @@ class ProtocolStrategy {
|
|
|
65
84
|
return outData;
|
|
66
85
|
});
|
|
67
86
|
this.pushRawDataToS3Stream = (data, S3Stream) => __awaiter(this, void 0, void 0, function* () {
|
|
68
|
-
if (
|
|
87
|
+
if (config_1.config.storage.enabled && config_1.config.saveRawDataWhitelist[this.caller]) {
|
|
69
88
|
const outData = yield this.prepareRawData(data);
|
|
70
89
|
if (outData && typeof outData === "string") {
|
|
71
90
|
S3Stream.push(outData);
|
|
@@ -74,30 +93,18 @@ class ProtocolStrategy {
|
|
|
74
93
|
}
|
|
75
94
|
}
|
|
76
95
|
else {
|
|
77
|
-
|
|
96
|
+
helpers_1.log.error(`Unable to save \`rawData\` in \`${this.caller}\`:
|
|
78
97
|
\`Buffer\`, \`Uint8Array\` or \`string\` is required, but got
|
|
79
98
|
\`${Object.prototype.toString.call(outData)}\``);
|
|
80
99
|
}
|
|
81
100
|
}
|
|
82
101
|
});
|
|
83
102
|
this.connectionSettings = settings;
|
|
84
|
-
const simpleConfig = Di_1.IntegrationEngineContainer.resolve(CoreToken_1.CoreToken.SimpleConfig);
|
|
85
|
-
this.storageEnabled = simpleConfig.getValue("env.STORAGE_ENABLED", "false") === "true";
|
|
86
|
-
const config = Di_1.IntegrationEngineContainer.resolve(ioc_1.ContainerToken.Config);
|
|
87
|
-
this.saveRawDataWhitelist = config.saveRawDataWhitelist;
|
|
88
|
-
this.logger = Di_1.IntegrationEngineContainer.resolve(CoreToken_1.CoreToken.Logger);
|
|
89
|
-
this.rawDataStorage = Di_1.IntegrationEngineContainer.resolve(ioc_1.ContainerToken.RawDataStorage);
|
|
90
103
|
}
|
|
91
104
|
setConnectionSettings(settings) {
|
|
92
105
|
this.connectionSettings = settings;
|
|
93
106
|
}
|
|
94
107
|
getData(...args) {
|
|
95
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
96
|
-
const { data } = yield this.getDataWithMetadata(...args);
|
|
97
|
-
return data;
|
|
98
|
-
});
|
|
99
|
-
}
|
|
100
|
-
getDataWithMetadata(...args) {
|
|
101
108
|
return __awaiter(this, void 0, void 0, function* () {
|
|
102
109
|
const result = yield this.getRawData(...args);
|
|
103
110
|
const rawData = result.data;
|
|
@@ -125,7 +132,7 @@ class ProtocolStrategy {
|
|
|
125
132
|
yield this.pushRawDataToS3Stream(data, S3Stream);
|
|
126
133
|
// because of headers - need to be sure that `response` event has been called before saving raw data
|
|
127
134
|
if (!dataSaveCalled) {
|
|
128
|
-
|
|
135
|
+
RawDataStore.save(S3Stream, meta, this.caller);
|
|
129
136
|
dataSaveCalled = true;
|
|
130
137
|
}
|
|
131
138
|
}));
|
|
@@ -145,17 +152,17 @@ class ProtocolStrategy {
|
|
|
145
152
|
return;
|
|
146
153
|
});
|
|
147
154
|
S3Stream.on("error", (err) => {
|
|
148
|
-
|
|
155
|
+
helpers_1.log.error(`Error on saving raw data for ${this.caller}`, err);
|
|
149
156
|
});
|
|
150
157
|
protocolDataStream.on("response", (res) => {
|
|
151
158
|
meta.statusCode = res.statusCode;
|
|
152
159
|
meta.headers = res.headers;
|
|
153
160
|
});
|
|
154
|
-
return
|
|
161
|
+
return dataStream;
|
|
155
162
|
}
|
|
156
163
|
else {
|
|
157
|
-
|
|
158
|
-
return
|
|
164
|
+
RawDataStore.save(yield this.prepareRawData(rawData), meta, this.caller);
|
|
165
|
+
return rawData;
|
|
159
166
|
}
|
|
160
167
|
});
|
|
161
168
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/ProtocolStrategy.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"ProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/ProtocolStrategy.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,yCAAoC;AAEpC,2CAAkC;AAClC,yEAAyD;AACzD,4CAAqE;AACrE,mCAAkC;AAClC,0DAAuD;AAEvD,MAAa,gBAAgB;IAKzB,YAAY,QAAa;QAFjB,WAAM,GAAG,SAAS,CAAC;QAMpB,kBAAa,GAAG,CAAC,SAAiB,oBAAoB,EAAE,EAAE;YAC7D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,iBAAiB;QAC3C,CAAC,CAAC;QAMK,0BAAqB,GAAG,GAA6B,EAAE;YAC1D,OAAO,IAAI,CAAC,kBAAkB,CAAC;QACnC,CAAC,CAAC;QAyEK,oBAAe,GAAG,GAAiC,EAAE;YACxD,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAChE,CAAC,CAAA,CAAC;QAMM,mBAAc,GAAG,CAAO,IAAS,EAA0B,EAAE;YACjE,IAAI,OAAO,GAAkB,IAAI,CAAC;YAElC,IAAI,IAAI,YAAY,IAAI,EAAE;gBACtB,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;aAC/B;iBAAM,IAAI,IAAI,YAAY,WAAW,EAAE;gBACpC,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;aAC5C;YAED,IAAI,IAAI,YAAY,UAAU,IAAI,IAAI,YAAY,MAAM,EAAE;gBACtD,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;aAC7B;YAED,IAAI,CAAC,OAAO,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;gBACtC,IAAI;oBACA,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;iBAC3C;gBAAC,WAAM;oBACJ,IAAI,CAAC;iBACR;aACJ;YAED,IAAI,CAAC,OAAO,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;gBACtC,IAAI;oBACA,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;iBACvD;gBAAC,WAAM;oBACJ,IAAI,CAAC;iBACR;aACJ;YAED,IAAI,CAAC,OAAO,IAAI,IAAI,IAAI,IAAI,CAAC,QAAQ,EAAE,EAAE;gBACrC,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;aAC7B;YAED,OAAO,OAAO,CAAC;QACnB,CAAC,CAAA,CAAC;QAEM,0BAAqB,GAAG,CAAO,IAAS,EAAE,QAAkB,EAAiB,EAAE;YACnF,IAAI,eAAM,CAAC,OAAO,CAAC,OAAO,IAAI,eAAM,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;gBACpE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;gBAEhD,IAAI,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE;oBACxC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBACvB,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE;wBAC7B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;qBACvB;iBACJ;qBAAM;oBACH,aAAG,CAAC,KAAK,CAAC,mCAAmC,IAAI,CAAC,MAAM;;qBAEnD,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;iBACrD;aACJ;QACL,CAAC,CAAA,CAAC;QAjJE,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;IACvC,CAAC;IAMM,qBAAqB,CAAC,QAAa;QACtC,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;IACvC,CAAC;IAMY,OAAO,CAAC,GAAG,IAAW;;YAC/B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC;YAC9C,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC;YAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YAC/B,IAAI,cAAc,GAAG,KAAK,CAAC;YAE3B,IAAI,kBAAuB,CAAC;YAE5B,uBAAuB;YACvB,IAAI,OAAO,IAAI,CAAC,OAAO,YAAY,iBAAQ,IAAI,OAAO,CAAC,WAAW,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,OAAO,CAAC,EAAE,EAAE;gBAClG,kBAAkB,GAAG,OAAO,CAAC;gBAE7B,MAAM,QAAQ,GAAG,IAAI,iBAAQ,CAAC;oBAC1B,UAAU,EAAE,KAAK;oBACjB,IAAI,EAAE,GAAG,EAAE;wBACP,OAAO;oBACX,CAAC;iBACJ,CAAC,CAAC;gBAEH,MAAM,UAAU,GAAG,IAAI,mCAAgB,CAAC;oBACpC,UAAU,EAAE,IAAI;oBAChB,IAAI,EAAE,GAAG,EAAE;wBACP,OAAO;oBACX,CAAC;iBACJ,CAAC,CAAC;gBAEH,sFAAsF;gBACtF,kBAAkB,CAAC,EAAE,CAAC,MAAM,EAAE,CAAO,IAAS,EAAE,EAAE;oBAC9C,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACtB,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;oBAEjD,oGAAoG;oBACpG,IAAI,CAAC,cAAc,EAAE;wBACjB,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;wBAC/C,cAAc,GAAG,IAAI,CAAC;qBACzB;gBACL,CAAC,CAAA,CAAC,CAAC;gBAEH,kBAAkB,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;oBAC1C,IAAI,GAAG,YAAY,6BAAoB,EAAE;wBACrC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;qBACjC;yBAAM;wBACH,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,2BAA2B,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;qBACvG;gBACL,CAAC,CAAC,CAAC;gBAEH,kBAAkB,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;oBAC9B,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACvB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxB,CAAC,CAAC,CAAC;gBAEH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;oBACpB,OAAO;gBACX,CAAC,CAAC,CAAC;gBAEH,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;oBACzB,aAAG,CAAC,KAAK,CAAC,gCAAgC,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;gBAClE,CAAC,CAAC,CAAC;gBAEH,kBAAkB,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,GAAQ,EAAE,EAAE;oBAC3C,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC;oBACjC,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;gBAC/B,CAAC,CAAC,CAAC;gBAEH,OAAO,UAAU,CAAC;aACrB;iBAAM;gBACH,YAAY,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBACzE,OAAO,OAAO,CAAC;aAClB;QACL,CAAC;KAAA;IAMe,UAAU,CAAC,GAAG,IAAS;;YACnC,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAC3D,CAAC;KAAA;CAsDJ;AAxJD,4CAwJC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.save = void 0;
|
|
16
|
+
const RawDataStorage_1 = __importDefault(require("../data-access/RawDataStorage"));
|
|
17
|
+
const ioc_1 = require("../ioc");
|
|
18
|
+
/**
|
|
19
|
+
* @deprecated For backwards compatibility. Use ioc container instead.
|
|
20
|
+
*/
|
|
21
|
+
const save = (data, meta, name = "Unknown") => __awaiter(void 0, void 0, void 0, function* () {
|
|
22
|
+
yield ioc_1.IntegrationEngineContainer.resolve(ioc_1.ContainerToken.RawDataStorage).save(data, meta, name);
|
|
23
|
+
});
|
|
24
|
+
exports.save = save;
|
|
25
|
+
//# sourceMappingURL=RawDataStore.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RawDataStore.js","sourceRoot":"","sources":["../../../src/integration-engine/helpers/RawDataStore.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,mFAA4D;AAC5D,gCAAqE;AAErE;;GAEG;AACI,MAAM,IAAI,GAAG,CAAO,IAAS,EAAE,IAAS,EAAE,IAAI,GAAG,SAAS,EAAE,EAAE;IACjE,MAAM,gCAA0B,CAAC,OAAO,CAAiB,oBAAc,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AACnH,CAAC,CAAA,CAAC;AAFW,QAAA,IAAI,QAEf"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@golemio/core",
|
|
3
|
-
"version": "1.16.0-
|
|
3
|
+
"version": "1.16.0-rc.1542535168",
|
|
4
4
|
"description": "Golemio Core Module",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -74,8 +74,8 @@
|
|
|
74
74
|
"@azure/data-tables": "^13.2.2",
|
|
75
75
|
"@azure/identity": "^4.0.1",
|
|
76
76
|
"@azure/storage-blob": "^12.11.0",
|
|
77
|
-
"@golemio/errors": "2.0.
|
|
78
|
-
"@golemio/validator": "0.3.
|
|
77
|
+
"@golemio/errors": "2.0.4",
|
|
78
|
+
"@golemio/validator": "0.3.5",
|
|
79
79
|
"@google-cloud/storage": "^7.11.0",
|
|
80
80
|
"@opentelemetry/api": "^1.0.4",
|
|
81
81
|
"@opentelemetry/exporter-jaeger": "^1.14.0",
|
|
@@ -131,7 +131,6 @@
|
|
|
131
131
|
},
|
|
132
132
|
"overrides": {
|
|
133
133
|
"dottie": "2.0.4",
|
|
134
|
-
"braces": "3.0.3"
|
|
135
|
-
"path-to-regexp@<1.0.0": "0.1.11"
|
|
134
|
+
"braces": "3.0.3"
|
|
136
135
|
}
|
|
137
136
|
}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.ResourceNotModified = void 0;
|
|
4
|
-
/** The requested resource has not been modified (received HTTP response 304) */
|
|
5
|
-
class ResourceNotModified {
|
|
6
|
-
}
|
|
7
|
-
exports.ResourceNotModified = ResourceNotModified;
|
|
8
|
-
//# sourceMappingURL=ResourceNotModified.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ResourceNotModified.js","sourceRoot":"","sources":["../../../src/integration-engine/helpers/ResourceNotModified.ts"],"names":[],"mappings":";;;AAAA,gFAAgF;AAChF,MAAa,mBAAmB;CAE/B;AAFD,kDAEC"}
|