@golemio/core 3.1.0 → 3.1.1-dev.2561427278

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/dist/helpers/data-access/azure/AzureBearerTokenProvider.d.ts +9 -0
  2. package/dist/helpers/data-access/azure/AzureBearerTokenProvider.js +27 -0
  3. package/dist/helpers/data-access/azure/AzureBearerTokenProvider.js.map +1 -0
  4. package/dist/helpers/data-access/azure/buildBearerTokenProvider.d.ts +12 -0
  5. package/dist/helpers/data-access/azure/buildBearerTokenProvider.js +30 -0
  6. package/dist/helpers/data-access/azure/buildBearerTokenProvider.js.map +1 -0
  7. package/dist/helpers/data-access/azure/index.d.ts +5 -0
  8. package/dist/helpers/data-access/azure/index.js +10 -0
  9. package/dist/helpers/data-access/azure/index.js.map +1 -0
  10. package/dist/helpers/data-access/opensearch/IOpenSearchConnector.d.ts +8 -0
  11. package/dist/helpers/data-access/opensearch/IOpenSearchConnector.js +3 -0
  12. package/dist/helpers/data-access/opensearch/IOpenSearchConnector.js.map +1 -0
  13. package/dist/helpers/data-access/opensearch/IOpenSearchInfo.d.ts +6 -0
  14. package/dist/helpers/data-access/opensearch/IOpenSearchInfo.js +3 -0
  15. package/dist/helpers/data-access/opensearch/IOpenSearchInfo.js.map +1 -0
  16. package/dist/helpers/data-access/opensearch/ISyncableIndex.d.ts +9 -0
  17. package/dist/helpers/data-access/opensearch/ISyncableIndex.js +3 -0
  18. package/dist/helpers/data-access/opensearch/ISyncableIndex.js.map +1 -0
  19. package/dist/helpers/data-access/opensearch/OpenSearchConnector.d.ts +21 -0
  20. package/dist/helpers/data-access/opensearch/OpenSearchConnector.js +113 -0
  21. package/dist/helpers/data-access/opensearch/OpenSearchConnector.js.map +1 -0
  22. package/dist/helpers/data-access/opensearch/OpenSearchService.d.ts +22 -0
  23. package/dist/helpers/data-access/opensearch/OpenSearchService.js +119 -0
  24. package/dist/helpers/data-access/opensearch/OpenSearchService.js.map +1 -0
  25. package/dist/helpers/data-access/opensearch/OpenSearchSyncOrchestrator.d.ts +14 -0
  26. package/dist/helpers/data-access/opensearch/OpenSearchSyncOrchestrator.js +78 -0
  27. package/dist/helpers/data-access/opensearch/OpenSearchSyncOrchestrator.js.map +1 -0
  28. package/dist/helpers/data-access/opensearch/czechAnalyzer.d.ts +6 -0
  29. package/dist/helpers/data-access/opensearch/czechAnalyzer.js +30 -0
  30. package/dist/helpers/data-access/opensearch/czechAnalyzer.js.map +1 -0
  31. package/dist/helpers/data-access/opensearch/index.d.ts +7 -0
  32. package/dist/helpers/data-access/opensearch/index.js +14 -0
  33. package/dist/helpers/data-access/opensearch/index.js.map +1 -0
  34. package/dist/helpers/routing/AbstractGeoJsonRouter.js.map +1 -1
  35. package/dist/index.d.ts +2 -0
  36. package/dist/index.js +3 -1
  37. package/dist/index.js.map +1 -1
  38. package/dist/input-gateway/connectors/AMQPConnector.d.ts +1 -1
  39. package/dist/input-gateway/helpers/SaveRawDataMiddleware.d.ts +1 -1
  40. package/dist/integration-engine/connectors/AMQPConnector.d.ts +1 -1
  41. package/dist/integration-engine/data-access/RawDataStorage.d.ts +1 -1
  42. package/dist/integration-engine/datasources/BatchWritable.d.ts +16 -0
  43. package/dist/integration-engine/datasources/BatchWritable.js +42 -0
  44. package/dist/integration-engine/datasources/BatchWritable.js.map +1 -0
  45. package/dist/integration-engine/datasources/DataSourceStreamedPipeline.d.ts +25 -0
  46. package/dist/integration-engine/datasources/DataSourceStreamedPipeline.js +72 -0
  47. package/dist/integration-engine/datasources/DataSourceStreamedPipeline.js.map +1 -0
  48. package/dist/integration-engine/datasources/datatype-strategy/JSONStreamTransformer.d.ts +5 -0
  49. package/dist/integration-engine/datasources/datatype-strategy/JSONStreamTransformer.js +12 -0
  50. package/dist/integration-engine/datasources/datatype-strategy/JSONStreamTransformer.js.map +1 -0
  51. package/dist/integration-engine/datasources/index.d.ts +5 -0
  52. package/dist/integration-engine/datasources/index.js +5 -0
  53. package/dist/integration-engine/datasources/index.js.map +1 -1
  54. package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.d.ts +5 -1
  55. package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.js +30 -1
  56. package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.js.map +1 -1
  57. package/dist/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.d.ts +5 -0
  58. package/dist/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.js +3 -0
  59. package/dist/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.js.map +1 -0
  60. package/dist/integration-engine/datasources/protocol-strategy/ProtocolStrategy.d.ts +3 -0
  61. package/dist/integration-engine/datasources/protocol-strategy/ProtocolStrategy.js +42 -3
  62. package/dist/integration-engine/datasources/protocol-strategy/ProtocolStrategy.js.map +1 -1
  63. package/dist/integration-engine/helpers/GeocodeApi.d.ts +1 -1
  64. package/dist/output-gateway/models/interfaces/IGeoJsonModel.d.ts +1 -1
  65. package/dist/output-gateway/routes/GeoJsonRouter.js.map +1 -1
  66. package/dist/shared/opensearch.d.ts +1 -0
  67. package/dist/shared/opensearch.js +18 -0
  68. package/dist/shared/opensearch.js.map +1 -0
  69. package/package.json +15 -13
@@ -0,0 +1,14 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.OpenSearchSyncOrchestrator = exports.OpenSearchService = exports.OpenSearchConnector = exports.czechText = exports.defaultIndexSettings = exports.czechAnalysis = void 0;
4
+ var czechAnalyzer_1 = require("./czechAnalyzer");
5
+ Object.defineProperty(exports, "czechAnalysis", { enumerable: true, get: function () { return czechAnalyzer_1.czechAnalysis; } });
6
+ Object.defineProperty(exports, "defaultIndexSettings", { enumerable: true, get: function () { return czechAnalyzer_1.defaultIndexSettings; } });
7
+ Object.defineProperty(exports, "czechText", { enumerable: true, get: function () { return czechAnalyzer_1.czechText; } });
8
+ var OpenSearchConnector_1 = require("./OpenSearchConnector");
9
+ Object.defineProperty(exports, "OpenSearchConnector", { enumerable: true, get: function () { return OpenSearchConnector_1.OpenSearchConnector; } });
10
+ var OpenSearchService_1 = require("./OpenSearchService");
11
+ Object.defineProperty(exports, "OpenSearchService", { enumerable: true, get: function () { return OpenSearchService_1.OpenSearchService; } });
12
+ var OpenSearchSyncOrchestrator_1 = require("./OpenSearchSyncOrchestrator");
13
+ Object.defineProperty(exports, "OpenSearchSyncOrchestrator", { enumerable: true, get: function () { return OpenSearchSyncOrchestrator_1.OpenSearchSyncOrchestrator; } });
14
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/helpers/data-access/opensearch/index.ts"],"names":[],"mappings":";;;AAAA,iDAAiF;AAAxE,8GAAA,aAAa,OAAA;AAAE,qHAAA,oBAAoB,OAAA;AAAE,0GAAA,SAAS,OAAA;AACvD,6DAA4D;AAAnD,0HAAA,mBAAmB,OAAA;AAE5B,yDAAwD;AAA/C,sHAAA,iBAAiB,OAAA;AAE1B,2EAA0E;AAAjE,wIAAA,0BAA0B,OAAA"}
@@ -1 +1 @@
1
- {"version":3,"file":"AbstractGeoJsonRouter.js","sourceRoot":"","sources":["../../../src/helpers/routing/AbstractGeoJsonRouter.ts"],"names":[],"mappings":";;;AAAA,qDAAiE;AAGjE,kDAA2C;AAC3C,4EAAwD;AACxD,oDAAoD;AACpD,wDAAiC;AAGjC,gEAAwE;AAExE,yDAA+E;AAE/E,4CAA+C;AAE/C;;;;;;GAMG;AACH,MAAsB,qBAAsB,SAAQ,+BAAc;IAwB9D,YAAsB,OAAe,EAAE,IAAY,EAAE,OAAsB;QACvE,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QArBf,mBAAc,GAAG;YACvB,IAAA,yBAAK,EAAC,cAAc,CAAC,CAAC,QAAQ,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE;YAC5D,IAAA,yBAAK,EAAC,WAAW,CAAC,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,IAAI,EAAE,CAAC;YACxE,IAAA,yBAAK,EAAC,KAAK,CAAC;iBACP,QAAQ,EAAE;iBACV,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;gBACd,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;oBACtB,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;wBAC7C,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;qBACtD;iBACJ;qBAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,EAAE;oBAClC,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;iBACrC;gBAED,OAAO,IAAI,CAAC;YAChB,CAAC,CAAC;YACN,IAAA,yBAAK,EAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE;YACjE,IAAA,yBAAK,EAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE;SACxD,CAAC;QAQF;;WAEG;QACI,mBAAc,GAAG,CAAC,SAAc,EAAE,EAAE;YACvC,OAAO,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAChE,CAAC,CAAC;QAEF;;;WAGG;QACO,eAAU,GAAG,KAAK,EAAE,SAAkC,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,oBAAoB,EAAE,CAAC,EAAE,EAAE,EAAE;YAC3G,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,8BAA8B,EAAE,CAAC;YAC5D,IAAI,CAAC,MAAM,CAAC,GAAG,CACX,GAAG,EACH,IAAI,CAAC,cAAc,EACnB,IAAA,sCAAyB,EAAC,eAAe,CAAC,EAC1C,wBAAW,EACX,IAAI,CAAC,qBAAqB,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,oBAAoB,CAAC,EACtF,IAAI,CAAC,MAAM,CACd,CAAC;YACF,IAAI,CAAC,MAAM,CAAC,GAAG,CACX,MAAM,EACN;gBACI,2EAA2E;gBAC3E,OAAO;aACV,EACD,wBAAW,EACX,IAAI,CAAC,qBAAqB,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,oBAAoB,CAAC,EACtF,IAAI,CAAC,MAAM,CACd,CAAC;QACN,CAAC,CAAC;QAEK,WAAM,GAAG,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,IAAkB,EAAE,EAAE;YACtE,MAAM,SAAS,GAAG,IAAA,+BAAW,EAAC,GAAG,CAAC,CAAC;YACnC,qBAAqB;YACrB,IAAI,GAAG,GAAG,SAAS,CAAC,GAAe,CAAC;YACpC,IAAI,SAAS,GAAG,SAAS,CAAC,SAAqB,CAAC;YAEhD,IAAI,SAAS,EAAE;gBACX,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;aAC9C;YACD,IAAI,GAAG,EAAE;gBACL,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;aAClC;YAED,IAAI;gBACA,MAAM,MAAM,GAAG,MAAM,IAAA,sBAAgB,EAAC,SAAS,CAAC,MAAgB,EAAE,SAAS,CAAC,KAAe,CAAC,CAAC;gBAC7F,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;oBACjC,SAAS;oBACT,GAAG,EAAE,GAAY;oBACjB,GAAG,EAAE,MAAM,CAAC,GAAG;oBACf,KAAK,EAAE,SAAS,CAAC,KAAK;oBACtB,GAAG,EAAE,MAAM,CAAC,GAAG;oBACf,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC;oBAChC,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,YAAY,EAAE,SAAS,CAAC,YAAsB;oBAC9C,iBAAiB,EAAE,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC;iBAC3B,CAAC,CAAC;gBAElC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aAC9B;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,CAAC,GAAG,CAAC,CAAC;aACb;QACL,CAAC,CAAC;QAEK,WAAM,GAAG,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,IAAkB,EAAE,EAAE;YACtE,MAAM,EAAE,GAAW,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAEjC,IAAI;gBACA,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAE3C,IAAI,MAAM,KAAK,SAAS,EAAE;oBACtB,MAAM,IAAI,qBAAY,CAAC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;iBAC7E;gBAED,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aAChC;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,CAAC,GAAG,CAAC,CAAC;aACb;QACL,CAAC,CAAC;QAEQ,mCAA8B,GAAG,KAAK,IAA8B,EAAE;YAC5E,IAAI,OAAwB,CAAC;YAE7B,wDAAwD;YACxD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,0BAA0B,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzE,IAAI,OAAO,GAAW,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,WAAW,GAAG,KAAK,GAAG,YAAY,CAAC;YACnG,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;YAEpE,IAAI,iBAAiB,EAAE;gBACnB,OAAO,IAAI,SAAS,CAAC;gBACrB,2DAA2D;gBAC3D,OAAO,GAAG,IAAA,yBAAK,EAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,CAAC;aAC9C;iBAAM;gBACH,OAAO,IAAI,SAAS,CAAC;gBACrB,2DAA2D;gBAC3D,OAAO,GAAG,IAAA,yBAAK,EAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC;aAC7C;YACD,YAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACnB,OAAO,OAAO,CAAC;QACnB,CAAC,CAAC;QAEK,yBAAoB,GAAG,CAAC,KAA2B,EAAE,EAAE;YAC1D,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC;YACtF,OAAO;gBACH,GAAG,IAAI;aACV,CAAC;QACN,CAAC,CAAC;QAhHE,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;QACrB,IAAI,CAAC,qBAAqB,GAAG,2BAAsB,CAAC,OAAO,CAAwB,+BAAc,CAAC,qBAAqB,CAAC,CAAC;IAC7H,CAAC;CA+GJ;AA3ID,sDA2IC"}
1
+ {"version":3,"file":"AbstractGeoJsonRouter.js","sourceRoot":"","sources":["../../../src/helpers/routing/AbstractGeoJsonRouter.ts"],"names":[],"mappings":";;;AAAA,qDAAiE;AAGjE,kDAA2C;AAC3C,4EAAwD;AACxD,oDAAoD;AACpD,wDAAiC;AAGjC,gEAAwE;AAExE,yDAA+E;AAE/E,4CAA+C;AAE/C;;;;;;GAMG;AACH,MAAsB,qBAAsB,SAAQ,+BAAc;IAwB9D,YAAsB,OAAe,EAAE,IAAY,EAAE,OAAsB;QACvE,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QArBf,mBAAc,GAAG;YACvB,IAAA,yBAAK,EAAC,cAAc,CAAC,CAAC,QAAQ,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE;YAC5D,IAAA,yBAAK,EAAC,WAAW,CAAC,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,IAAI,EAAE,CAAC;YACxE,IAAA,yBAAK,EAAC,KAAK,CAAC;iBACP,QAAQ,EAAE;iBACV,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;gBACd,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;oBACtB,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;wBAC7C,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;qBACtD;iBACJ;qBAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,EAAE;oBAClC,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;iBACrC;gBAED,OAAO,IAAI,CAAC;YAChB,CAAC,CAAC;YACN,IAAA,yBAAK,EAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE;YACjE,IAAA,yBAAK,EAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE;SACxD,CAAC;QAQF;;WAEG;QACI,mBAAc,GAAG,CAAC,SAAc,EAAE,EAAE;YACvC,OAAO,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAChE,CAAC,CAAC;QAEF;;;WAGG;QACO,eAAU,GAAG,KAAK,EAAE,SAAkC,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,oBAAoB,EAAE,CAAC,EAAE,EAAE,EAAE;YAC3G,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,8BAA8B,EAAE,CAAC;YAC5D,IAAI,CAAC,MAAM,CAAC,GAAG,CACX,GAAG,EACH,IAAI,CAAC,cAAc,EACnB,IAAA,sCAAyB,EAAC,eAAe,CAAC,EAC1C,wBAAW,EACX,IAAI,CAAC,qBAAqB,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,oBAAoB,CAAC,EACtF,IAAI,CAAC,MAAM,CACd,CAAC;YACF,IAAI,CAAC,MAAM,CAAC,GAAG,CACX,MAAM,EACN;gBACI,2EAA2E;gBAC3E,OAAO;aACV,EACD,wBAAW,EACX,IAAI,CAAC,qBAAqB,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,oBAAoB,CAAC,EACtF,IAAI,CAAC,MAAM,CACd,CAAC;QACN,CAAC,CAAC;QAEK,WAAM,GAAG,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,IAAkB,EAAE,EAAE;YACtE,MAAM,SAAS,GAAG,IAAA,+BAAW,EAAC,GAAG,CAAC,CAAC;YACnC,qBAAqB;YACrB,IAAI,GAAG,GAAG,SAAS,CAAC,GAAe,CAAC;YACpC,IAAI,SAAS,GAAG,SAAS,CAAC,SAAqB,CAAC;YAEhD,IAAI,SAAS,EAAE;gBACX,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;aAC9C;YACD,IAAI,GAAG,EAAE;gBACL,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;aAClC;YAED,IAAI;gBACA,MAAM,MAAM,GAAG,MAAM,IAAA,sBAAgB,EAAC,SAAS,CAAC,MAAgB,EAAE,SAAS,CAAC,KAAe,CAAC,CAAC;gBAC7F,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;oBACjC,SAAS;oBACT,GAAG,EAAE,GAAY;oBACjB,GAAG,EAAE,MAAM,CAAC,GAAG;oBACf,KAAK,EAAE,SAAS,CAAC,KAAK;oBACtB,GAAG,EAAE,MAAM,CAAC,GAAG;oBACf,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC;oBAChC,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,YAAY,EAAE,SAAS,CAAC,YAAsB;oBAC9C,iBAAiB,EAAE,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC;iBAC3B,CAAC,CAAC;gBAElC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aAC9B;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,CAAC,GAAG,CAAC,CAAC;aACb;QACL,CAAC,CAAC;QAEK,WAAM,GAAG,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,IAAkB,EAAE,EAAE;YACtE,MAAM,EAAE,GAAW,GAAG,CAAC,MAAM,CAAC,EAAY,CAAC;YAE3C,IAAI;gBACA,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAE3C,IAAI,MAAM,KAAK,SAAS,EAAE;oBACtB,MAAM,IAAI,qBAAY,CAAC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;iBAC7E;gBAED,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aAChC;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,CAAC,GAAG,CAAC,CAAC;aACb;QACL,CAAC,CAAC;QAEQ,mCAA8B,GAAG,KAAK,IAA8B,EAAE;YAC5E,IAAI,OAAwB,CAAC;YAE7B,wDAAwD;YACxD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,0BAA0B,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzE,IAAI,OAAO,GAAW,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,WAAW,GAAG,KAAK,GAAG,YAAY,CAAC;YACnG,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;YAEpE,IAAI,iBAAiB,EAAE;gBACnB,OAAO,IAAI,SAAS,CAAC;gBACrB,2DAA2D;gBAC3D,OAAO,GAAG,IAAA,yBAAK,EAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,CAAC;aAC9C;iBAAM;gBACH,OAAO,IAAI,SAAS,CAAC;gBACrB,2DAA2D;gBAC3D,OAAO,GAAG,IAAA,yBAAK,EAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC;aAC7C;YACD,YAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACnB,OAAO,OAAO,CAAC;QACnB,CAAC,CAAC;QAEK,yBAAoB,GAAG,CAAC,KAA2B,EAAE,EAAE;YAC1D,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC;YACtF,OAAO;gBACH,GAAG,IAAI;aACV,CAAC;QACN,CAAC,CAAC;QAhHE,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;QACrB,IAAI,CAAC,qBAAqB,GAAG,2BAAsB,CAAC,OAAO,CAAwB,+BAAc,CAAC,qBAAqB,CAAC,CAAC;IAC7H,CAAC;CA+GJ;AA3ID,sDA2IC"}
package/dist/index.d.ts CHANGED
@@ -2,3 +2,5 @@ export * as InputGateway from "./input-gateway/index";
2
2
  export * as IntegrationEngine from "./integration-engine/index";
3
3
  export * as OutputGateway from "./output-gateway/index";
4
4
  export * as SchemaDefinitions from "./schema-definitions/index";
5
+ export * as AzureHelpers from "./helpers/data-access/azure/index";
6
+ export * as OpenSearchHelpers from "./helpers/data-access/opensearch/index";
package/dist/index.js CHANGED
@@ -23,10 +23,12 @@ var __importStar = (this && this.__importStar) || function (mod) {
23
23
  return result;
24
24
  };
25
25
  Object.defineProperty(exports, "__esModule", { value: true });
26
- exports.SchemaDefinitions = exports.OutputGateway = exports.IntegrationEngine = exports.InputGateway = void 0;
26
+ exports.OpenSearchHelpers = exports.AzureHelpers = exports.SchemaDefinitions = exports.OutputGateway = exports.IntegrationEngine = exports.InputGateway = void 0;
27
27
  // Library exports
28
28
  exports.InputGateway = __importStar(require("./input-gateway/index"));
29
29
  exports.IntegrationEngine = __importStar(require("./integration-engine/index"));
30
30
  exports.OutputGateway = __importStar(require("./output-gateway/index"));
31
31
  exports.SchemaDefinitions = __importStar(require("./schema-definitions/index"));
32
+ exports.AzureHelpers = __importStar(require("./helpers/data-access/azure/index"));
33
+ exports.OpenSearchHelpers = __importStar(require("./helpers/data-access/opensearch/index"));
32
34
  //# sourceMappingURL=index.js.map
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kBAAkB;AAClB,sEAA0C;AAC1C,gFAA+C;AAC/C,wEAA2C;AAC3C,gFAAgD"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kBAAkB;AAClB,sEAA0C;AAC1C,gFAA+C;AAC/C,wEAA2C;AAC3C,gFAAgD;AAChD,kFAAiE;AACjE,4FAA2E"}
@@ -1,5 +1,5 @@
1
1
  import { ILogger } from "../../helpers";
2
- import { IConfiguration } from "../config";
2
+ import type { IConfiguration } from "../config";
3
3
  import amqplib from "amqplib";
4
4
  import { Disposable } from "tsyringe";
5
5
  import IQueueConnector from "../../helpers/connector/interfaces/IQueueConnector";
@@ -1,7 +1,7 @@
1
1
  import { ILogger } from "../../helpers";
2
2
  import { ISimpleConfig } from "../../helpers/configuration/ISimpleConfig";
3
3
  import { AbstractStorageService } from "../../helpers/data-access/storage";
4
- import { IConfiguration } from "../config";
4
+ import type { IConfiguration } from "../config";
5
5
  import { NextFunction, Request, Response } from "express";
6
6
  export declare class SaveRawDataMiddleware {
7
7
  private readonly log;
@@ -1,6 +1,6 @@
1
1
  import { ILogger } from "../../helpers";
2
2
  import IQueueConnector from "../../helpers/connector/interfaces/IQueueConnector";
3
- import { IConfiguration } from "../config";
3
+ import type { IConfiguration } from "../config";
4
4
  import amqplib from "amqplib";
5
5
  import { Disposable } from "tsyringe";
6
6
  export declare class AMQPConnector implements IQueueConnector, Disposable {
@@ -1,7 +1,7 @@
1
1
  import { ILogger } from "../../helpers/logger/LoggerProvider";
2
2
  import { ISimpleConfig } from "../../helpers/configuration/ISimpleConfig";
3
3
  import { AbstractStorageService } from "../../helpers/data-access/storage";
4
- import { IConfiguration } from "../config/abstract";
4
+ import type { IConfiguration } from "../config/abstract";
5
5
  export default class RawDataStorage {
6
6
  private storageService;
7
7
  private readonly log;
@@ -0,0 +1,16 @@
1
+ /// <reference types="node" />
2
+ /// <reference types="node" />
3
+ import { Writable } from "node:stream";
4
+ export interface BatchWritableOptions<T> {
5
+ batchSize: number;
6
+ onFlush: (batch: T[]) => Promise<void>;
7
+ }
8
+ export declare class BatchWritable<T> extends Writable {
9
+ private batch;
10
+ private readonly batchSize;
11
+ private readonly onFlush;
12
+ constructor(options: BatchWritableOptions<T>);
13
+ _write(item: T, _encoding: BufferEncoding, callback: (err?: Error | null) => void): void;
14
+ _final(callback: (err?: Error | null) => void): void;
15
+ private flush;
16
+ }
@@ -0,0 +1,42 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.BatchWritable = void 0;
4
+ const node_stream_1 = require("node:stream");
5
+ class BatchWritable extends node_stream_1.Writable {
6
+ constructor(options) {
7
+ super({
8
+ objectMode: true,
9
+ });
10
+ this.batch = [];
11
+ this.batchSize = options.batchSize;
12
+ this.onFlush = options.onFlush;
13
+ }
14
+ _write(item, _encoding, callback) {
15
+ this.batch.push(item);
16
+ if (this.batch.length < this.batchSize) {
17
+ callback();
18
+ return;
19
+ }
20
+ this.flush(callback);
21
+ }
22
+ _final(callback) {
23
+ if (this.batch.length === 0) {
24
+ callback();
25
+ return;
26
+ }
27
+ this.flush(callback);
28
+ }
29
+ async flush(callback) {
30
+ const fullBatch = this.batch;
31
+ this.batch = [];
32
+ try {
33
+ await this.onFlush(fullBatch);
34
+ callback();
35
+ }
36
+ catch (err) {
37
+ callback(err instanceof Error ? err : new Error(String(err)));
38
+ }
39
+ }
40
+ }
41
+ exports.BatchWritable = BatchWritable;
42
+ //# sourceMappingURL=BatchWritable.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"BatchWritable.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/BatchWritable.ts"],"names":[],"mappings":";;;AAAA,6CAAuC;AAOvC,MAAa,aAAiB,SAAQ,sBAAQ;IAK1C,YAAY,OAAgC;QACxC,KAAK,CAAC;YACF,UAAU,EAAE,IAAI;SACnB,CAAC,CAAC;QAPC,UAAK,GAAQ,EAAE,CAAC;QAQpB,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACnC,CAAC;IAEe,MAAM,CAAC,IAAO,EAAE,SAAyB,EAAE,QAAsC;QAC7F,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE;YACpC,QAAQ,EAAE,CAAC;YACX,OAAO;SACV;QACD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACzB,CAAC;IAEe,MAAM,CAAC,QAAsC;QACzD,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;YACzB,QAAQ,EAAE,CAAC;YACX,OAAO;SACV;QACD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACzB,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,QAAsC;QACtD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC;QAC7B,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAChB,IAAI;YACA,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAC9B,QAAQ,EAAE,CAAC;SACd;QAAC,OAAO,GAAG,EAAE;YACV,QAAQ,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;SACjE;IACL,CAAC;CACJ;AAxCD,sCAwCC"}
@@ -0,0 +1,25 @@
1
+ /// <reference types="node" />
2
+ import { IDataSource } from "./IDataSource";
3
+ import { IValidator } from "@golemio/validator";
4
+ import { Transform } from "node:stream";
5
+ import { DataSource } from "./DataSource";
6
+ import { IStreamingProtocolStrategy } from "./protocol-strategy/IStreamingProtocolStrategy";
7
+ export interface DataSourceStreamedPipelineOptions {
8
+ validator?: IValidator | null;
9
+ batchSize?: number;
10
+ timeoutMs?: number;
11
+ }
12
+ export declare class DataSourceStreamedPipeline<T extends object = any> extends DataSource implements IDataSource {
13
+ static readonly DEFAULT_BATCH_SIZE: number;
14
+ static readonly DEFAULT_PIPELINE_TIMEOUT_MS: number;
15
+ private readonly streamTransformer;
16
+ private readonly batchSize;
17
+ private readonly timeoutMs;
18
+ private dataProcessor;
19
+ constructor(name: string, streamingStrategy: IStreamingProtocolStrategy, streamTransformer: Transform, options?: DataSourceStreamedPipelineOptions);
20
+ private get streamingStrategy();
21
+ setDataProcessor(fn: (batch: T[]) => Promise<void>): this;
22
+ proceed(): Promise<void>;
23
+ private createBatchWriter;
24
+ private validateAndProcess;
25
+ }
@@ -0,0 +1,72 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.DataSourceStreamedPipeline = void 0;
4
+ const logger_1 = require("../../helpers/logger");
5
+ const errors_1 = require("@golemio/errors");
6
+ const promises_1 = require("node:stream/promises");
7
+ const BatchWritable_1 = require("./BatchWritable");
8
+ const DataSource_1 = require("./DataSource");
9
+ class DataSourceStreamedPipeline extends DataSource_1.DataSource {
10
+ constructor(name, streamingStrategy, streamTransformer, options = {}) {
11
+ super(name, streamingStrategy, null, options.validator ?? null);
12
+ this.dataProcessor = null;
13
+ this.streamTransformer = streamTransformer;
14
+ this.batchSize = options.batchSize ?? DataSourceStreamedPipeline.DEFAULT_BATCH_SIZE;
15
+ this.timeoutMs = options.timeoutMs ?? DataSourceStreamedPipeline.DEFAULT_PIPELINE_TIMEOUT_MS;
16
+ }
17
+ get streamingStrategy() {
18
+ return this.protocolStrategy;
19
+ }
20
+ setDataProcessor(fn) {
21
+ this.dataProcessor = fn;
22
+ return this;
23
+ }
24
+ async proceed() {
25
+ if (!this.dataProcessor) {
26
+ throw new errors_1.GeneralError("No data processor registered. Call setDataProcessor() before proceed().", this.name);
27
+ }
28
+ const bodyStream = await this.streamingStrategy.getBodyStream();
29
+ let totalRecords = 0;
30
+ const batchWriter = this.createBatchWriter((processedCount) => {
31
+ totalRecords += processedCount;
32
+ });
33
+ try {
34
+ await (0, promises_1.pipeline)(bodyStream, this.streamTransformer, batchWriter, {
35
+ signal: AbortSignal.timeout(this.timeoutMs),
36
+ });
37
+ }
38
+ catch (err) {
39
+ bodyStream.destroy();
40
+ if (err instanceof errors_1.GeneralError)
41
+ throw err;
42
+ throw new errors_1.GeneralError("DataSourceStreamedPipeline pipeline failed.", this.name, err);
43
+ }
44
+ this.loggerEmitter.emit(logger_1.LoggerEventType.NumberOfRecords, { name: this.name, numberOfRecords: totalRecords });
45
+ }
46
+ createBatchWriter(onBatchProcessed) {
47
+ const batchSize = this.batchSize;
48
+ return new BatchWritable_1.BatchWritable({
49
+ batchSize,
50
+ onFlush: async (batch) => {
51
+ const processedCount = batch.length;
52
+ await this.validateAndProcess(batch);
53
+ onBatchProcessed(processedCount);
54
+ },
55
+ });
56
+ }
57
+ async validateAndProcess(batch) {
58
+ if (this.validator) {
59
+ try {
60
+ await this.validator.Validate(batch);
61
+ }
62
+ catch (err) {
63
+ throw new errors_1.GeneralError("Error while validating source data.", this.name, err);
64
+ }
65
+ }
66
+ await this.dataProcessor(batch);
67
+ }
68
+ }
69
+ exports.DataSourceStreamedPipeline = DataSourceStreamedPipeline;
70
+ DataSourceStreamedPipeline.DEFAULT_BATCH_SIZE = process.env.DATA_BATCH_SIZE ? Number(process.env.DATA_BATCH_SIZE) : 1000;
71
+ DataSourceStreamedPipeline.DEFAULT_PIPELINE_TIMEOUT_MS = 30 * 60 * 1000;
72
+ //# sourceMappingURL=DataSourceStreamedPipeline.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"DataSourceStreamedPipeline.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/DataSourceStreamedPipeline.ts"],"names":[],"mappings":";;;AAAA,iDAAkD;AAElD,4CAA+C;AAG/C,mDAAgD;AAChD,mDAAgD;AAChD,6CAA0C;AAU1C,MAAa,0BAAmD,SAAQ,uBAAU;IAU9E,YACI,IAAY,EACZ,iBAA6C,EAC7C,iBAA4B,EAC5B,UAA6C,EAAE;QAE/C,KAAK,CAAC,IAAI,EAAE,iBAAiD,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC;QAR5F,kBAAa,GAA2C,IAAI,CAAC;QASjE,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,0BAA0B,CAAC,kBAAkB,CAAC;QACpF,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,0BAA0B,CAAC,2BAA2B,CAAC;IACjG,CAAC;IAED,IAAY,iBAAiB;QACzB,OAAO,IAAI,CAAC,gBAAyD,CAAC;IAC1E,CAAC;IAEM,gBAAgB,CAAC,EAAiC;QACrD,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,KAAK,CAAC,OAAO;QAChB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE;YACrB,MAAM,IAAI,qBAAY,CAAC,yEAAyE,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;SAChH;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE,CAAC;QAEhE,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,cAAc,EAAE,EAAE;YAC1D,YAAY,IAAI,cAAc,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,IAAI;YACA,MAAM,IAAA,mBAAQ,EAAC,UAAU,EAAE,IAAI,CAAC,iBAAiB,EAAE,WAAW,EAAE;gBAC5D,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;aAC9C,CAAC,CAAC;SACN;QAAC,OAAO,GAAG,EAAE;YACV,UAAU,CAAC,OAAO,EAAE,CAAC;YACrB,IAAI,GAAG,YAAY,qBAAY;gBAAE,MAAM,GAAG,CAAC;YAC3C,MAAM,IAAI,qBAAY,CAAC,6CAA6C,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;SACzF;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,wBAAe,CAAC,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,eAAe,EAAE,YAAY,EAAE,CAAC,CAAC;IACjH,CAAC;IAEO,iBAAiB,CAAC,gBAAyC;QAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QACjC,OAAO,IAAI,6BAAa,CAAI;YACxB,SAAS;YACT,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;gBACrB,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC;gBACpC,MAAM,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;gBACrC,gBAAgB,CAAC,cAAc,CAAC,CAAC;YACrC,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,KAAU;QACvC,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,IAAI;gBACA,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;aACxC;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,qBAAY,CAAC,qCAAqC,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aACjF;SACJ;QACD,MAAM,IAAI,CAAC,aAAc,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;;AA7EL,gEA8EC;AA7E0B,6CAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,AAA3E,CAA4E;AAC9F,sDAA2B,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,AAAjB,CAAkB"}
@@ -0,0 +1,5 @@
1
+ /// <reference types="node" />
2
+ import { Transform } from "node:stream";
3
+ export declare class JSONStreamTransformer {
4
+ static parse(path: string): Transform;
5
+ }
@@ -0,0 +1,12 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.JSONStreamTransformer = void 0;
4
+ // eslint-disable-next-line @typescript-eslint/no-var-requires
5
+ const JSONStream = require("JSONStream");
6
+ class JSONStreamTransformer {
7
+ static parse(path) {
8
+ return JSONStream.parse(path);
9
+ }
10
+ }
11
+ exports.JSONStreamTransformer = JSONStreamTransformer;
12
+ //# sourceMappingURL=JSONStreamTransformer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"JSONStreamTransformer.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/JSONStreamTransformer.ts"],"names":[],"mappings":";;;AAGA,8DAA8D;AAC9D,MAAM,UAAU,GAAG,OAAO,CAAC,YAAY,CAA4B,CAAC;AACpE,MAAa,qBAAqB;IAC9B,MAAM,CAAC,KAAK,CAAC,IAAY;QACrB,OAAO,UAAU,CAAC,KAAK,CAAC,IAAI,CAAyB,CAAC;IAC1D,CAAC;CACJ;AAJD,sDAIC"}
@@ -1,16 +1,21 @@
1
+ export * from "./BatchWritable";
1
2
  export * from "./DataSource";
2
3
  export * from "./DataSourceStream";
3
4
  export * from "./DataSourceStreamManager";
4
5
  export * from "./DataSourceStreamed";
6
+ export * from "./DataSourceStreamedPipeline";
5
7
  export * from "./IDataSource";
6
8
  export * from "./datatype-strategy/CSVDataTypeStrategy";
7
9
  export * from "./datatype-strategy/IDataTypeStrategy";
8
10
  export * from "./datatype-strategy/JSONDataTypeStrategy";
11
+ export * from "./datatype-strategy/JSONStreamTransformer";
9
12
  export * from "./datatype-strategy/XMLDataTypeStrategy";
10
13
  export * from "./protocol-strategy/FTPProtocolStrategy";
14
+ export * from "./protocol-strategy/HTTPRequestProtocolStrategyStreamed";
11
15
  export * from "./protocol-strategy/GoogleCloudStorageProtocolStrategy";
12
16
  export * from "./protocol-strategy/PaginatedHTTPProtocolStrategy";
13
17
  export * from "./protocol-strategy/IProtocolStrategy";
18
+ export * from "./protocol-strategy/IStreamingProtocolStrategy";
14
19
  export * from "./protocol-strategy/PostgresProtocolStrategy";
15
20
  export * from "./protocol-strategy/PostgresProtocolStrategyStreamed";
16
21
  export * from "./protocol-strategy/ProtocolStrategy";
@@ -15,19 +15,24 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
15
15
  };
16
16
  Object.defineProperty(exports, "__esModule", { value: true });
17
17
  /* ie/datasources/index.ts */
18
+ __exportStar(require("./BatchWritable"), exports);
18
19
  __exportStar(require("./DataSource"), exports);
19
20
  __exportStar(require("./DataSourceStream"), exports);
20
21
  __exportStar(require("./DataSourceStreamManager"), exports);
21
22
  __exportStar(require("./DataSourceStreamed"), exports);
23
+ __exportStar(require("./DataSourceStreamedPipeline"), exports);
22
24
  __exportStar(require("./IDataSource"), exports);
23
25
  __exportStar(require("./datatype-strategy/CSVDataTypeStrategy"), exports);
24
26
  __exportStar(require("./datatype-strategy/IDataTypeStrategy"), exports);
25
27
  __exportStar(require("./datatype-strategy/JSONDataTypeStrategy"), exports);
28
+ __exportStar(require("./datatype-strategy/JSONStreamTransformer"), exports);
26
29
  __exportStar(require("./datatype-strategy/XMLDataTypeStrategy"), exports);
27
30
  __exportStar(require("./protocol-strategy/FTPProtocolStrategy"), exports);
31
+ __exportStar(require("./protocol-strategy/HTTPRequestProtocolStrategyStreamed"), exports);
28
32
  __exportStar(require("./protocol-strategy/GoogleCloudStorageProtocolStrategy"), exports);
29
33
  __exportStar(require("./protocol-strategy/PaginatedHTTPProtocolStrategy"), exports);
30
34
  __exportStar(require("./protocol-strategy/IProtocolStrategy"), exports);
35
+ __exportStar(require("./protocol-strategy/IStreamingProtocolStrategy"), exports);
31
36
  __exportStar(require("./protocol-strategy/PostgresProtocolStrategy"), exports);
32
37
  __exportStar(require("./protocol-strategy/PostgresProtocolStrategyStreamed"), exports);
33
38
  __exportStar(require("./protocol-strategy/ProtocolStrategy"), exports);
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6BAA6B;AAC7B,+CAA6B;AAC7B,qDAAmC;AACnC,4DAA0C;AAC1C,uDAAqC;AACrC,gDAA8B;AAC9B,0EAAwD;AACxD,wEAAsD;AACtD,2EAAyD;AACzD,0EAAwD;AACxD,0EAAwD;AACxD,yFAAuE;AACvE,oFAAkE;AAClE,wEAAsD;AACtD,+EAA6D;AAC7D,uFAAqE;AACrE,uEAAqD;AACrD,2EAAyD"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6BAA6B;AAC7B,kDAAgC;AAChC,+CAA6B;AAC7B,qDAAmC;AACnC,4DAA0C;AAC1C,uDAAqC;AACrC,+DAA6C;AAC7C,gDAA8B;AAC9B,0EAAwD;AACxD,wEAAsD;AACtD,2EAAyD;AACzD,4EAA0D;AAC1D,0EAAwD;AACxD,0EAAwD;AACxD,0FAAwE;AACxE,yFAAuE;AACvE,oFAAkE;AAClE,wEAAsD;AACtD,iFAA+D;AAC/D,+EAA6D;AAC7D,uFAAqE;AACrE,uEAAqD;AACrD,2EAAyD"}
@@ -1,12 +1,16 @@
1
+ /// <reference types="node" />
1
2
  import { IProtocolStrategy } from "./IProtocolStrategy";
3
+ import { Readable } from "node:stream";
2
4
  import { DataSourceStream } from "../DataSourceStream";
3
5
  import { HTTPRequestProtocolStrategy } from "./HTTPRequestProtocolStrategy";
4
- export declare class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy implements IProtocolStrategy {
6
+ import { IStreamingProtocolStrategy } from "./IStreamingProtocolStrategy";
7
+ export declare class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy implements IProtocolStrategy, IStreamingProtocolStrategy {
5
8
  private streamTransform;
6
9
  /**
7
10
  * @param streamTransform data will be piped to this stream if provided
8
11
  */
9
12
  setStreamTransformer: (streamTransform: any) => HTTPRequestProtocolStrategyStreamed;
10
13
  getData(): Promise<DataSourceStream>;
14
+ getBodyStream(): Promise<Readable>;
11
15
  getRawData: () => Promise<any>;
12
16
  }
@@ -2,6 +2,7 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.HTTPRequestProtocolStrategyStreamed = void 0;
4
4
  const errors_1 = require("@golemio/errors");
5
+ const node_zlib_1 = require("node:zlib");
5
6
  const DataSourceStream_1 = require("../DataSourceStream");
6
7
  const HTTPRequestProtocolStrategy_1 = require("./HTTPRequestProtocolStrategy");
7
8
  class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy_1.HTTPRequestProtocolStrategy {
@@ -16,7 +17,8 @@ class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy_1.
16
17
  };
17
18
  this.getRawData = async () => {
18
19
  if (this.connectionSettings.compression) {
19
- throw new errors_1.GeneralError("compressed resources are not supported in HTTPRequestProtocolStrategyStreamed yet", this.constructor.name);
20
+ throw new errors_1.GeneralError("Compressed resources are not supported in the legacy DataSourceStream path;" +
21
+ "use DataSourceStreamedPipeline instead", this.constructor.name);
20
22
  }
21
23
  try {
22
24
  const result = await this.getResponseInternal();
@@ -97,6 +99,33 @@ class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy_1.
97
99
  }
98
100
  return outStream;
99
101
  }
102
+ async getBodyStream() {
103
+ try {
104
+ const result = await this.getResponseInternal();
105
+ if (result.statusCode < 200 || result.statusCode >= 300) {
106
+ result.body?.resume?.(); // drain to avoid connection leak (some statuses have no body)
107
+ throw new errors_1.GeneralError(`Unexpected HTTP status ${result.statusCode}`, this.constructor.name);
108
+ }
109
+ let body = result.body;
110
+ if (this.connectionSettings.compression === "gzip") {
111
+ const encoding = String(result.headers?.["content-encoding"] ?? "").toLowerCase();
112
+ if (encoding !== "gzip") {
113
+ body?.resume?.();
114
+ throw new errors_1.GeneralError(`Expected gzip-encoded response but server returned Content-Encoding: "${encoding || "<none>"}"`, this.constructor.name);
115
+ }
116
+ body = body.pipe((0, node_zlib_1.createGunzip)());
117
+ }
118
+ return this.teeRawDataToStorage(body, {
119
+ statusCode: result.statusCode,
120
+ headers: result.headers,
121
+ });
122
+ }
123
+ catch (err) {
124
+ if (err instanceof errors_1.GeneralError)
125
+ throw err;
126
+ throw new errors_1.GeneralError("Error while getting body stream from server.", this.constructor.name, err);
127
+ }
128
+ }
100
129
  }
101
130
  exports.HTTPRequestProtocolStrategyStreamed = HTTPRequestProtocolStrategyStreamed;
102
131
  //# sourceMappingURL=HTTPRequestProtocolStrategyStreamed.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"HTTPRequestProtocolStrategyStreamed.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.ts"],"names":[],"mappings":";;;AACA,4CAAqE;AACrE,0DAAuD;AACvD,+EAA4E;AAE5E,MAAa,mCAAoC,SAAQ,yDAA2B;IAApF;;QAGI;;WAEG;QACI,yBAAoB,GAAG,CAAC,eAAoB,EAAuC,EAAE;YACxF,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;YACvC,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC;QAsEK,eAAU,GAAG,KAAK,IAAkB,EAAE;YACzC,IAAI,IAAI,CAAC,kBAAkB,CAAC,WAAW,EAAE;gBACrC,MAAM,IAAI,qBAAY,CAClB,mFAAmF,EACnF,IAAI,CAAC,WAAW,CAAC,IAAI,CACxB,CAAC;aACL;YAED,IAAI;gBACA,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAEhD,OAAO;oBACH,IAAI,EAAE,MAAM,CAAC,IAAI;iBACpB,CAAC;aACL;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,qBAAY,CAAC,uCAAuC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aAC/F;QACL,CAAC,CAAC;IACN,CAAC;IAtFU,KAAK,CAAC,OAAO;QAChB,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QAEzC,MAAM,SAAS,GAAG,IAAI,mCAAgB,CAAC;YACnC,UAAU,EAAE,IAAI;YAChB,IAAI,EAAE,GAAG,EAAE;gBACP,OAAO;YACX,CAAC;SACJ,CAAC,CAAC;QAEH,IAAI,aAAa,GAAG,KAAK,CAAC;QAE1B,IAAI,IAAI,CAAC,eAAe,EAAE;YACtB,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;gBAChC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACrC,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACtB,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC;YAC/B,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;gBAC1C,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBAChC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBAC5C,IAAI,KAAK,YAAY,6BAAoB,EAAE;oBACvC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;iBAClC;qBAAM;oBACH,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,6BAA6B,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;iBAC1G;YACL,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBAClC,IAAI,KAAK,YAAY,6BAAoB,EAAE;oBACvC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;iBAC7C;qBAAM;oBACH,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;iBAC7G;YACL,CAAC,CAAC,CAAC;SACN;aAAM;YACH,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;gBAChC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACxB,qCAAqC;gBACrC,IAAI,CAAC,aAAa,EAAE;oBAChB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACrB,aAAa,GAAG,IAAI,CAAC;iBACxB;YACL,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACtB,uCAAuC;gBACvC,IAAI,CAAC,aAAa,EAAE;oBAChB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACrB,aAAa,GAAG,IAAI,CAAC;iBACxB;YACL,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBAClC,IAAI,KAAK,YAAY,6BAAoB,EAAE;oBACvC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;iBAClC;qBAAM;oBACH,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;iBAClG;YACL,CAAC,CAAC,CAAC;SACN;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;CAoBJ;AAjGD,kFAiGC"}
1
+ {"version":3,"file":"HTTPRequestProtocolStrategyStreamed.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.ts"],"names":[],"mappings":";;;AACA,4CAAqE;AAErE,yCAAyC;AACzC,0DAAuD;AACvD,+EAA4E;AAG5E,MAAa,mCACT,SAAQ,yDAA2B;IADvC;;QAMI;;WAEG;QACI,yBAAoB,GAAG,CAAC,eAAoB,EAAuC,EAAE;YACxF,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;YACvC,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC;QAuGK,eAAU,GAAG,KAAK,IAAkB,EAAE;YACzC,IAAI,IAAI,CAAC,kBAAkB,CAAC,WAAW,EAAE;gBACrC,MAAM,IAAI,qBAAY,CAClB,6EAA6E;oBACzE,wCAAwC,EAC5C,IAAI,CAAC,WAAW,CAAC,IAAI,CACxB,CAAC;aACL;YAED,IAAI;gBACA,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAEhD,OAAO;oBACH,IAAI,EAAE,MAAM,CAAC,IAAI;iBACpB,CAAC;aACL;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,qBAAY,CAAC,uCAAuC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aAC/F;QACL,CAAC,CAAC;IACN,CAAC;IAxHU,KAAK,CAAC,OAAO;QAChB,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QAEzC,MAAM,SAAS,GAAG,IAAI,mCAAgB,CAAC;YACnC,UAAU,EAAE,IAAI;YAChB,IAAI,EAAE,GAAG,EAAE;gBACP,OAAO;YACX,CAAC;SACJ,CAAC,CAAC;QAEH,IAAI,aAAa,GAAG,KAAK,CAAC;QAE1B,IAAI,IAAI,CAAC,eAAe,EAAE;YACtB,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;gBAChC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACrC,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACtB,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC;YAC/B,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;gBAC1C,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBAChC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBAC5C,IAAI,KAAK,YAAY,6BAAoB,EAAE;oBACvC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;iBAClC;qBAAM;oBACH,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,6BAA6B,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;iBAC1G;YACL,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBAClC,IAAI,KAAK,YAAY,6BAAoB,EAAE;oBACvC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;iBAC7C;qBAAM;oBACH,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;iBAC7G;YACL,CAAC,CAAC,CAAC;SACN;aAAM;YACH,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;gBAChC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACxB,qCAAqC;gBACrC,IAAI,CAAC,aAAa,EAAE;oBAChB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACrB,aAAa,GAAG,IAAI,CAAC;iBACxB;YACL,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACtB,uCAAuC;gBACvC,IAAI,CAAC,aAAa,EAAE;oBAChB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACrB,aAAa,GAAG,IAAI,CAAC;iBACxB;YACL,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBAClC,IAAI,KAAK,YAAY,6BAAoB,EAAE;oBACvC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;iBAClC;qBAAM;oBACH,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;iBAClG;YACL,CAAC,CAAC,CAAC;SACN;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAEM,KAAK,CAAC,aAAa;QACtB,IAAI;YACA,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAEhD,IAAI,MAAM,CAAC,UAAU,GAAG,GAAG,IAAI,MAAM,CAAC,UAAU,IAAI,GAAG,EAAE;gBACrD,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,8DAA8D;gBACvF,MAAM,IAAI,qBAAY,CAAC,0BAA0B,MAAM,CAAC,UAAU,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;aAChG;YAED,IAAI,IAAI,GAAG,MAAM,CAAC,IAA2B,CAAC;YAE9C,IAAI,IAAI,CAAC,kBAAkB,CAAC,WAAW,KAAK,MAAM,EAAE;gBAChD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;gBAClF,IAAI,QAAQ,KAAK,MAAM,EAAE;oBACrB,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC;oBACjB,MAAM,IAAI,qBAAY,CAClB,yEAAyE,QAAQ,IAAI,QAAQ,GAAG,EAChG,IAAI,CAAC,WAAW,CAAC,IAAI,CACxB,CAAC;iBACL;gBACD,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAY,GAAE,CAAC,CAAC;aACpC;YAED,OAAO,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE;gBAClC,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,OAAO,EAAE,MAAM,CAAC,OAAO;aAC1B,CAAC,CAAC;SACN;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,GAAG,YAAY,qBAAY;gBAAE,MAAM,GAAG,CAAC;YAC3C,MAAM,IAAI,qBAAY,CAAC,8CAA8C,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;SACtG;IACL,CAAC;CAqBJ;AAtID,kFAsIC"}
@@ -0,0 +1,5 @@
1
+ /// <reference types="node" />
2
+ import { Readable } from "stream";
3
+ export interface IStreamingProtocolStrategy {
4
+ getBodyStream(): Promise<Readable>;
5
+ }
@@ -0,0 +1,3 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ //# sourceMappingURL=IStreamingProtocolStrategy.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"IStreamingProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.ts"],"names":[],"mappings":""}
@@ -1,3 +1,5 @@
1
+ /// <reference types="node" />
2
+ import { Readable } from "node:stream";
1
3
  import { DataSourceStream } from "../DataSourceStream";
2
4
  import { IProtocolStrategy, ProtocolStrategySettings } from "./IProtocolStrategy";
3
5
  export declare class ProtocolStrategy implements IProtocolStrategy {
@@ -18,6 +20,7 @@ export declare class ProtocolStrategy implements IProtocolStrategy {
18
20
  }>;
19
21
  getLastModified: () => Promise<string | null>;
20
22
  protected getRawData(...args: any): Promise<any | DataSourceStream>;
23
+ protected teeRawDataToStorage(bodyStream: Readable, meta?: unknown): Readable;
21
24
  private prepareRawData;
22
25
  private pushRawDataToS3Stream;
23
26
  }
@@ -9,7 +9,8 @@ const RawDataStorage_1 = __importDefault(require("../../data-access/RawDataStora
9
9
  const ioc_1 = require("../../ioc");
10
10
  const Di_1 = require("../../ioc/Di");
11
11
  const errors_1 = require("@golemio/errors");
12
- const stream_1 = require("stream");
12
+ const node_stream_1 = require("node:stream");
13
+ const promises_1 = require("node:stream/promises");
13
14
  const DataSourceStream_1 = require("../DataSourceStream");
14
15
  class ProtocolStrategy {
15
16
  constructor(settings) {
@@ -93,9 +94,9 @@ class ProtocolStrategy {
93
94
  let dataSaveCalled = false;
94
95
  let protocolDataStream;
95
96
  // bit paranoid but ...
96
- if (rawData && (rawData instanceof stream_1.Readable || rawData.constructor.name === "Request") && rawData.on) {
97
+ if (rawData && (rawData instanceof node_stream_1.Readable || rawData.constructor.name === "Request") && rawData.on) {
97
98
  protocolDataStream = rawData;
98
- const S3Stream = new stream_1.Readable({
99
+ const S3Stream = new node_stream_1.Readable({
99
100
  objectMode: false,
100
101
  read: () => {
101
102
  return;
@@ -149,6 +150,44 @@ class ProtocolStrategy {
149
150
  async getRawData(...args) {
150
151
  throw new Error("getRawData: Method not implemented.");
151
152
  }
153
+ teeRawDataToStorage(bodyStream, meta = {}) {
154
+ if (!this.storageEnabled || !this.saveRawDataWhitelist[this.caller]) {
155
+ return bodyStream;
156
+ }
157
+ const S3Stream = new node_stream_1.Readable({
158
+ objectMode: false,
159
+ read: () => {
160
+ return;
161
+ },
162
+ });
163
+ const consumer = new node_stream_1.PassThrough();
164
+ let consumerErrorFired = false;
165
+ (0, promises_1.pipeline)(bodyStream, consumer).catch((err) => {
166
+ if (!consumerErrorFired) {
167
+ this.logger.error(`Body stream source failed for ${this.caller}`, err);
168
+ }
169
+ });
170
+ let saveCalled = false;
171
+ consumer.on("data", async (chunk) => {
172
+ await this.pushRawDataToS3Stream(chunk, S3Stream);
173
+ if (!saveCalled) {
174
+ this.rawDataStorage.save(S3Stream, meta, this.caller);
175
+ saveCalled = true;
176
+ }
177
+ });
178
+ consumer.on("end", () => {
179
+ S3Stream.push(null);
180
+ });
181
+ consumer.on("error", (err) => {
182
+ consumerErrorFired = true;
183
+ this.logger.warn(`Raw data save to storage interrupted for ${this.caller}`, err);
184
+ S3Stream.push(null);
185
+ });
186
+ S3Stream.on("error", (err) => {
187
+ this.logger.error(`Error on saving raw data for ${this.caller}`, err);
188
+ });
189
+ return consumer;
190
+ }
152
191
  }
153
192
  exports.ProtocolStrategy = ProtocolStrategy;
154
193
  //# sourceMappingURL=ProtocolStrategy.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"ProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/ProtocolStrategy.ts"],"names":[],"mappings":";;;;;;AACA,8DAAmD;AAGnD,sFAA4D;AAC5D,mCAAyC;AACzC,qCAAwD;AACxD,4CAAqE;AACrE,mCAAkC;AAClC,0DAAuD;AAGvD,MAAa,gBAAgB;IASzB,YAAY,QAAa;QANjB,WAAM,GAAG,SAAS,CAAC;QAgBpB,kBAAa,GAAG,CAAC,SAAiB,oBAAoB,EAAE,EAAE;YAC7D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,iBAAiB;QAC3C,CAAC,CAAC;QAMK,0BAAqB,GAAG,GAA6B,EAAE;YAC1D,OAAO,IAAI,CAAC,kBAAkB,CAAC;QACnC,CAAC,CAAC;QA8EK,oBAAe,GAAG,KAAK,IAA4B,EAAE;YACxD,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAChE,CAAC,CAAC;QAMM,mBAAc,GAAG,KAAK,EAAE,IAAS,EAA0B,EAAE;YACjE,IAAI,OAAO,GAAkB,IAAI,CAAC;YAElC,IAAI,IAAI,YAAY,IAAI,EAAE;gBACtB,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;aAC/B;iBAAM,IAAI,IAAI,YAAY,WAAW,EAAE;gBACpC,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;aAC5C;YAED,IAAI,IAAI,YAAY,UAAU,IAAI,IAAI,YAAY,MAAM,EAAE;gBACtD,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;aAC7B;YAED,IAAI,CAAC,OAAO,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;gBACtC,IAAI;oBACA,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;iBAC3C;gBAAC,MAAM;oBACJ,IAAI,CAAC;iBACR;aACJ;YAED,IAAI,CAAC,OAAO,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;gBACtC,IAAI;oBACA,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;iBACvD;gBAAC,MAAM;oBACJ,IAAI,CAAC;iBACR;aACJ;YAED,IAAI,CAAC,OAAO,IAAI,IAAI,IAAI,IAAI,CAAC,QAAQ,EAAE,EAAE;gBACrC,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;aAC7B;YAED,OAAO,OAAO,CAAC;QACnB,CAAC,CAAC;QAEM,0BAAqB,GAAG,KAAK,EAAE,IAAS,EAAE,QAAkB,EAAiB,EAAE;YACnF,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;gBAC/D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;gBAEhD,IAAI,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE;oBACxC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBACvB,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE;wBAC7B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;qBACvB;iBACJ;qBAAM;oBACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,IAAI,CAAC,MAAM;;qBAE3D,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;iBACrD;aACJ;QACL,CAAC,CAAC;QA5JE,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;QACnC,MAAM,YAAY,GAAG,+BAA0B,CAAC,OAAO,CAAgB,qBAAS,CAAC,YAAY,CAAC,CAAC;QAC/F,IAAI,CAAC,cAAc,GAAG,YAAY,CAAC,QAAQ,CAAC,qBAAqB,EAAE,OAAO,CAAC,KAAK,MAAM,CAAC;QACvF,MAAM,MAAM,GAAG,+BAA0B,CAAC,OAAO,CAAiB,oBAAc,CAAC,MAAM,CAAC,CAAC;QACzF,IAAI,CAAC,oBAAoB,GAAG,MAAM,CAAC,oBAAoB,CAAC;QACxD,IAAI,CAAC,MAAM,GAAG,+BAA0B,CAAC,OAAO,CAAU,qBAAS,CAAC,MAAM,CAAC,CAAC;QAC5E,IAAI,CAAC,cAAc,GAAG,+BAA0B,CAAC,OAAO,CAAiB,oBAAc,CAAC,cAAc,CAAC,CAAC;IAC5G,CAAC;IAMM,qBAAqB,CAAC,QAAa;QACtC,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;IACvC,CAAC;IAMM,KAAK,CAAC,OAAO,CAAC,GAAG,IAAW;QAC/B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,GAAG,IAAI,CAAC,CAAC;QACzD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,KAAK,CAAC,mBAAmB,CAAC,GAAG,IAAe;QAC/C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAI,cAAc,GAAG,KAAK,CAAC;QAE3B,IAAI,kBAAuB,CAAC;QAE5B,uBAAuB;QACvB,IAAI,OAAO,IAAI,CAAC,OAAO,YAAY,iBAAQ,IAAI,OAAO,CAAC,WAAW,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,OAAO,CAAC,EAAE,EAAE;YAClG,kBAAkB,GAAG,OAAO,CAAC;YAE7B,MAAM,QAAQ,GAAG,IAAI,iBAAQ,CAAC;gBAC1B,UAAU,EAAE,KAAK;gBACjB,IAAI,EAAE,GAAG,EAAE;oBACP,OAAO;gBACX,CAAC;aACJ,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,IAAI,mCAAgB,CAAC;gBACpC,UAAU,EAAE,IAAI;gBAChB,IAAI,EAAE,GAAG,EAAE;oBACP,OAAO;gBACX,CAAC;aACJ,CAAC,CAAC;YAEH,sFAAsF;YACtF,kBAAkB,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,IAAS,EAAE,EAAE;gBAC9C,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACtB,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;gBAEjD,oGAAoG;gBACpG,IAAI,CAAC,cAAc,EAAE;oBACjB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;oBACtD,cAAc,GAAG,IAAI,CAAC;iBACzB;YACL,CAAC,CAAC,CAAC;YAEH,kBAAkB,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;gBAC1C,IAAI,GAAG,YAAY,6BAAoB,EAAE;oBACrC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;iBACjC;qBAAM;oBACH,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,2BAA2B,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;iBACvG;YACL,CAAC,CAAC,CAAC;YAEH,kBAAkB,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBAC9B,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACvB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC,CAAC,CAAC;YAEH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACpB,OAAO;YACX,CAAC,CAAC,CAAC;YAEH,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;YAC1E,CAAC,CAAC,CAAC;YAEH,kBAAkB,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,GAAQ,EAAE,EAAE;gBAC3C,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC;gBACjC,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;YAC/B,CAAC,CAAC,CAAC;YAEH,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;SAC/C;aAAM;YACH,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAChF,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;SAC5C;IACL,CAAC;IAMS,KAAK,CAAC,UAAU,CAAC,GAAG,IAAS;QACnC,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAC3D,CAAC;CAsDJ;AAvKD,4CAuKC"}
1
+ {"version":3,"file":"ProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/ProtocolStrategy.ts"],"names":[],"mappings":";;;;;;AACA,8DAAmD;AAGnD,sFAA4D;AAC5D,mCAAyC;AACzC,qCAAwD;AACxD,4CAAqE;AACrE,6CAAoD;AACpD,mDAAgD;AAChD,0DAAuD;AAGvD,MAAa,gBAAgB;IASzB,YAAY,QAAa;QANjB,WAAM,GAAG,SAAS,CAAC;QAgBpB,kBAAa,GAAG,CAAC,SAAiB,oBAAoB,EAAE,EAAE;YAC7D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,iBAAiB;QAC3C,CAAC,CAAC;QAMK,0BAAqB,GAAG,GAA6B,EAAE;YAC1D,OAAO,IAAI,CAAC,kBAAkB,CAAC;QACnC,CAAC,CAAC;QA8EK,oBAAe,GAAG,KAAK,IAA4B,EAAE;YACxD,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAChE,CAAC,CAAC;QAqDM,mBAAc,GAAG,KAAK,EAAE,IAAS,EAA0B,EAAE;YACjE,IAAI,OAAO,GAAkB,IAAI,CAAC;YAElC,IAAI,IAAI,YAAY,IAAI,EAAE;gBACtB,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;aAC/B;iBAAM,IAAI,IAAI,YAAY,WAAW,EAAE;gBACpC,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;aAC5C;YAED,IAAI,IAAI,YAAY,UAAU,IAAI,IAAI,YAAY,MAAM,EAAE;gBACtD,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;aAC7B;YAED,IAAI,CAAC,OAAO,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;gBACtC,IAAI;oBACA,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;iBAC3C;gBAAC,MAAM;oBACJ,IAAI,CAAC;iBACR;aACJ;YAED,IAAI,CAAC,OAAO,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;gBACtC,IAAI;oBACA,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;iBACvD;gBAAC,MAAM;oBACJ,IAAI,CAAC;iBACR;aACJ;YAED,IAAI,CAAC,OAAO,IAAI,IAAI,IAAI,IAAI,CAAC,QAAQ,EAAE,EAAE;gBACrC,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;aAC7B;YAED,OAAO,OAAO,CAAC;QACnB,CAAC,CAAC;QAEM,0BAAqB,GAAG,KAAK,EAAE,IAAS,EAAE,QAAkB,EAAiB,EAAE;YACnF,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;gBAC/D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;gBAEhD,IAAI,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE;oBACxC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBACvB,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE;wBAC7B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;qBACvB;iBACJ;qBAAM;oBACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,IAAI,CAAC,MAAM;;qBAE3D,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;iBACrD;aACJ;QACL,CAAC,CAAC;QA3ME,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;QACnC,MAAM,YAAY,GAAG,+BAA0B,CAAC,OAAO,CAAgB,qBAAS,CAAC,YAAY,CAAC,CAAC;QAC/F,IAAI,CAAC,cAAc,GAAG,YAAY,CAAC,QAAQ,CAAC,qBAAqB,EAAE,OAAO,CAAC,KAAK,MAAM,CAAC;QACvF,MAAM,MAAM,GAAG,+BAA0B,CAAC,OAAO,CAAiB,oBAAc,CAAC,MAAM,CAAC,CAAC;QACzF,IAAI,CAAC,oBAAoB,GAAG,MAAM,CAAC,oBAAoB,CAAC;QACxD,IAAI,CAAC,MAAM,GAAG,+BAA0B,CAAC,OAAO,CAAU,qBAAS,CAAC,MAAM,CAAC,CAAC;QAC5E,IAAI,CAAC,cAAc,GAAG,+BAA0B,CAAC,OAAO,CAAiB,oBAAc,CAAC,cAAc,CAAC,CAAC;IAC5G,CAAC;IAMM,qBAAqB,CAAC,QAAa;QACtC,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC;IACvC,CAAC;IAMM,KAAK,CAAC,OAAO,CAAC,GAAG,IAAW;QAC/B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,GAAG,IAAI,CAAC,CAAC;QACzD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,KAAK,CAAC,mBAAmB,CAAC,GAAG,IAAe;QAC/C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAI,cAAc,GAAG,KAAK,CAAC;QAE3B,IAAI,kBAAuB,CAAC;QAE5B,uBAAuB;QACvB,IAAI,OAAO,IAAI,CAAC,OAAO,YAAY,sBAAQ,IAAI,OAAO,CAAC,WAAW,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,OAAO,CAAC,EAAE,EAAE;YAClG,kBAAkB,GAAG,OAAO,CAAC;YAE7B,MAAM,QAAQ,GAAG,IAAI,sBAAQ,CAAC;gBAC1B,UAAU,EAAE,KAAK;gBACjB,IAAI,EAAE,GAAG,EAAE;oBACP,OAAO;gBACX,CAAC;aACJ,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,IAAI,mCAAgB,CAAC;gBACpC,UAAU,EAAE,IAAI;gBAChB,IAAI,EAAE,GAAG,EAAE;oBACP,OAAO;gBACX,CAAC;aACJ,CAAC,CAAC;YAEH,sFAAsF;YACtF,kBAAkB,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,IAAS,EAAE,EAAE;gBAC9C,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACtB,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;gBAEjD,oGAAoG;gBACpG,IAAI,CAAC,cAAc,EAAE;oBACjB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;oBACtD,cAAc,GAAG,IAAI,CAAC;iBACzB;YACL,CAAC,CAAC,CAAC;YAEH,kBAAkB,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;gBAC1C,IAAI,GAAG,YAAY,6BAAoB,EAAE;oBACrC,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;iBACjC;qBAAM;oBACH,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,2BAA2B,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;iBACvG;YACL,CAAC,CAAC,CAAC;YAEH,kBAAkB,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBAC9B,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACvB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC,CAAC,CAAC;YAEH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACpB,OAAO;YACX,CAAC,CAAC,CAAC;YAEH,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;YAC1E,CAAC,CAAC,CAAC;YAEH,kBAAkB,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,GAAQ,EAAE,EAAE;gBAC3C,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC;gBACjC,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;YAC/B,CAAC,CAAC,CAAC;YAEH,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;SAC/C;aAAM;YACH,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAChF,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;SAC5C;IACL,CAAC;IAMS,KAAK,CAAC,UAAU,CAAC,GAAG,IAAS;QACnC,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAC3D,CAAC;IAES,mBAAmB,CAAC,UAAoB,EAAE,OAAgB,EAAE;QAClE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;YACjE,OAAO,UAAU,CAAC;SACrB;QAED,MAAM,QAAQ,GAAG,IAAI,sBAAQ,CAAC;YAC1B,UAAU,EAAE,KAAK;YACjB,IAAI,EAAE,GAAG,EAAE;gBACP,OAAO;YACX,CAAC;SACJ,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,IAAI,yBAAW,EAAE,CAAC;QAEnC,IAAI,kBAAkB,GAAG,KAAK,CAAC;QAC/B,IAAA,mBAAQ,EAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACzC,IAAI,CAAC,kBAAkB,EAAE;gBACrB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;aAC1E;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,KAAU,EAAE,EAAE;YACrC,MAAM,IAAI,CAAC,qBAAqB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAClD,IAAI,CAAC,UAAU,EAAE;gBACb,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBACtD,UAAU,GAAG,IAAI,CAAC;aACrB;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YACpB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACzB,kBAAkB,GAAG,IAAI,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4CAA4C,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;YACjF,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC;IACpB,CAAC;CAsDJ;AAtND,4CAsNC"}
@@ -1,5 +1,5 @@
1
1
  import { ILogger } from "../../helpers";
2
- import { IConfiguration } from "../config";
2
+ import type { IConfiguration } from "../config";
3
3
  export interface IPostalAddress {
4
4
  address_formatted: string;
5
5
  street_address?: string;
@@ -1,5 +1,5 @@
1
1
  import { IBaseModel } from "./IBaseModel";
2
- import { IPropertyResponseModel } from "../response/IPropertyResponseModel";
2
+ import type { IPropertyResponseModel } from "../response/IPropertyResponseModel";
3
3
  export interface IGeoJsonModel extends IBaseModel {
4
4
  GetAll(options?: any): Promise<any>;
5
5
  GetOne(id: string): Promise<any>;