@golemio/core 3.1.1 → 3.2.0-dev.2585439595

package/dist/helpers/data-access/opensearch/czechAnalyzer.js

@@ -0,0 +1,30 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.czechText = exports.defaultIndexSettings = exports.czechAnalysis = void 0;
+exports.czechAnalysis = {
+    analysis: {
+        filter: {
+            czech_stop: { type: "stop", stopwords: "_czech_" },
+            czech_stem: { type: "stemmer", language: "czech" },
+            czech_ascii: { type: "asciifolding", preserve_original: true },
+        },
+        analyzer: {
+            czech_analyzer: {
+                type: "custom",
+                tokenizer: "standard",
+                // pořadí záleží: stemming musí být před asciifolding
+                filter: ["lowercase", "czech_stop", "czech_stem", "czech_ascii"],
+            },
+        },
+    },
+};
+exports.defaultIndexSettings = {
+    number_of_shards: 1,
+    ...exports.czechAnalysis,
+};
+const czechText = () => ({
+    type: "text",
+    analyzer: "czech_analyzer",
+});
+exports.czechText = czechText;
+//# sourceMappingURL=czechAnalyzer.js.map

package/dist/helpers/data-access/opensearch/czechAnalyzer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"czechAnalyzer.js","sourceRoot":"","sources":["../../../../src/helpers/data-access/opensearch/czechAnalyzer.ts"],"names":[],"mappings":";;;AAEa,QAAA,aAAa,GAA6D;IACnF,QAAQ,EAAE;QACN,MAAM,EAAE;YACJ,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE;YAClD,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE;YAClD,WAAW,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,iBAAiB,EAAE,IAAI,EAAE;SACjE;QACD,QAAQ,EAAE;YACN,cAAc,EAAE;gBACZ,IAAI,EAAE,QAAQ;gBACd,SAAS,EAAE,UAAU;gBACrB,qDAAqD;gBACrD,MAAM,EAAE,CAAC,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,aAAa,CAAC;aACnE;SACJ;KACJ;CACJ,CAAC;AAEW,QAAA,oBAAoB,GAAuC;IACpE,gBAAgB,EAAE,CAAC;IACnB,GAAG,qBAAa;CACnB,CAAC;AAEK,MAAM,SAAS,GAAG,GAAsC,EAAE,CAAC,CAAC;IAC/D,IAAI,EAAE,MAAM;IACZ,QAAQ,EAAE,gBAAgB;CAC7B,CAAC,CAAC;AAHU,QAAA,SAAS,aAGnB"}

package/dist/helpers/data-access/opensearch/index.d.ts

@@ -0,0 +1,8 @@
+export { czechAnalysis, defaultIndexSettings, czechText } from "./czechAnalyzer";
+export { OpenSearchConnector } from "./OpenSearchConnector";
+export type { IOpenSearchConnector } from "./IOpenSearchConnector";
+export { OpenSearchService } from "./OpenSearchService";
+export type { IOpenSearchServiceConfig, IndexCount, HealthStatus } from "./OpenSearchService";
+export type { IOpenSearchInfo } from "./IOpenSearchInfo";
+export { OpenSearchSyncOrchestrator } from "./OpenSearchSyncOrchestrator";
+export type { ISyncableIndex, ISyncableDoc } from "./ISyncableIndex";

package/dist/helpers/data-access/opensearch/index.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenSearchSyncOrchestrator = exports.OpenSearchService = exports.OpenSearchConnector = exports.czechText = exports.defaultIndexSettings = exports.czechAnalysis = void 0;
+var czechAnalyzer_1 = require("./czechAnalyzer");
+Object.defineProperty(exports, "czechAnalysis", { enumerable: true, get: function () { return czechAnalyzer_1.czechAnalysis; } });
+Object.defineProperty(exports, "defaultIndexSettings", { enumerable: true, get: function () { return czechAnalyzer_1.defaultIndexSettings; } });
+Object.defineProperty(exports, "czechText", { enumerable: true, get: function () { return czechAnalyzer_1.czechText; } });
+var OpenSearchConnector_1 = require("./OpenSearchConnector");
+Object.defineProperty(exports, "OpenSearchConnector", { enumerable: true, get: function () { return OpenSearchConnector_1.OpenSearchConnector; } });
+var OpenSearchService_1 = require("./OpenSearchService");
+Object.defineProperty(exports, "OpenSearchService", { enumerable: true, get: function () { return OpenSearchService_1.OpenSearchService; } });
+var OpenSearchSyncOrchestrator_1 = require("./OpenSearchSyncOrchestrator");
+Object.defineProperty(exports, "OpenSearchSyncOrchestrator", { enumerable: true, get: function () { return OpenSearchSyncOrchestrator_1.OpenSearchSyncOrchestrator; } });
+//# sourceMappingURL=index.js.map

package/dist/helpers/data-access/opensearch/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/helpers/data-access/opensearch/index.ts"],"names":[],"mappings":";;;AAAA,iDAAiF;AAAxE,8GAAA,aAAa,OAAA;AAAE,qHAAA,oBAAoB,OAAA;AAAE,0GAAA,SAAS,OAAA;AACvD,6DAA4D;AAAnD,0HAAA,mBAAmB,OAAA;AAE5B,yDAAwD;AAA/C,sHAAA,iBAAiB,OAAA;AAG1B,2EAA0E;AAAjE,wIAAA,0BAA0B,OAAA"}

package/dist/helpers/data-access/postgres/DatabaseConnector.js

@@ -59,10 +59,17 @@ class DatabaseConnector {
         };
     }
     getOptions(additionalOptions) {
+        const dialectOptions = {
+            application_name: `${this.databaseInfo.applicationName} (${os_1.default.hostname()})`,
+        };
+        if (this.databaseInfo.sslCaBase64) {
+            dialectOptions.ssl = {
+                ca: Buffer.from(this.databaseInfo.sslCaBase64, "base64").toString("utf-8"),
+                rejectUnauthorized: true,
+            };
+        }
         return {
-            dialectOptions: {
-                application_name: `${this.databaseInfo.applicationName} (${os_1.default.hostname()})`,
-            },
+            dialectOptions,
             logging: this.log.silly,
             pool: {
                 acquire: 60000,

package/dist/helpers/data-access/postgres/DatabaseConnector.js.map

@@ -1 +1 @@
-{"version":3,"file":"DatabaseConnector.js","sourceRoot":"","sources":["../../../../src/helpers/data-access/postgres/DatabaseConnector.ts"],"names":[],"mappings":";;;;;;AACA,4CAA6C;AAC7C,4CAAoB;AACpB,yCAA+C;AAK/C,MAAa,iBAAiB;IAG1B,YACqB,YAA2B,EAC3B,GAAY,EACZ,gBAA0B;QAF1B,iBAAY,GAAZ,YAAY,CAAe;QAC3B,QAAG,GAAH,GAAG,CAAS;QACZ,qBAAgB,GAAhB,gBAAgB,CAAU;QAGxC,YAAO,GAAG,KAAK,EAAE,iBAA2B,EAAsB,EAAE;YACvE,IAAI;gBACA,IAAI,IAAI,CAAC,UAAU,EAAE;oBACjB,OAAO,IAAI,CAAC,UAAU,CAAC;iBAC1B;gBAED,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE;oBACrC,MAAM,IAAI,mBAAU,CAAC,qDAAqD,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;iBACtG;gBAED,IAAI,CAAC,UAAU,GAAG,IAAI,qBAAS,CAAC,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBAExG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;oBAClB,MAAM,IAAI,mBAAU,CAAC,0BAA0B,CAAC,CAAC;iBACpD;gBAED,MAAM,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC;gBACrC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;gBAC1C,OAAO,IAAI,CAAC,UAAU,CAAC;aAC1B;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,mBAAU,CAAC,uCAAuC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aAC7F;QACL,CAAC,CAAC;QAEK,kBAAa,GAAG,GAAc,EAAE;YACnC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;gBAClB,MAAM,IAAI,mBAAU,CAAC,mEAAmE,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;aACpH;YAED,OAAO,IAAI,CAAC,UAAU,CAAC;QAC3B,CAAC,CAAC;QAEK,gBAAW,GAAG,KAAK,IAAsB,EAAE;YAC9C,IAAI;gBACA,MAAM,IAAI,CAAC,UAAW,CAAC,YAAY,EAAE,CAAC;gBAEtC,OAAO,IAAI,CAAC;aACf;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,oDAAoD,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC;gBACnF,OAAO,KAAK,CAAC;aAChB;QACL,CAAC,CAAC;QAEK,eAAU,GAAG,KAAK,IAAmB,EAAE;YAC1C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;YAE9C,IAAI,IAAI,CAAC,UAAU,EAAE;gBACjB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;aACjC;QACL,CAAC,CAAC;QAEK,YAAO,GAAG,KAAK,IAAmB,EAAE;YACvC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC5B,CAAC,CAAC;IAvDC,CAAC;IAyDI,UAAU,CAAC,iBAA2B;QAC1C,OAAO;YACH,cAAc,EAAE;gBACZ,gBAAgB,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,eAAe,KAAK,YAAE,CAAC,QAAQ,EAAE,GAAG;aAC9E;YACD,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK;YACvB,IAAI,EAAE;gBACF,OAAO,EAAE,KAAK;gBACd,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC;gBAC/C,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC;gBACjD,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC;aACpD;YACD,KAAK,EAAE;gBACH,KAAK,EAAE;oBACH,0BAA0B;oBAC1B,iCAAiC;oBACjC,4BAA4B;oBAC5B,gCAAgC;oBAChC,iCAAiC;oBACjC,kCAAkC;oBAClC,cAAc;iBACjB;gBACD,GAAG,EAAE,CAAC;aACT;YACD,GAAG,IAAI,CAAC,gBAAgB;YACxB,GAAG,iBAAiB;SACvB,CAAC;IACN,CAAC;CACJ;AA5FD,8CA4FC"}
+{"version":3,"file":"DatabaseConnector.js","sourceRoot":"","sources":["../../../../src/helpers/data-access/postgres/DatabaseConnector.ts"],"names":[],"mappings":";;;;;;AACA,4CAA6C;AAC7C,4CAAoB;AACpB,yCAA+C;AAK/C,MAAa,iBAAiB;IAG1B,YACqB,YAA2B,EAC3B,GAAY,EACZ,gBAA0B;QAF1B,iBAAY,GAAZ,YAAY,CAAe;QAC3B,QAAG,GAAH,GAAG,CAAS;QACZ,qBAAgB,GAAhB,gBAAgB,CAAU;QAGxC,YAAO,GAAG,KAAK,EAAE,iBAA2B,EAAsB,EAAE;YACvE,IAAI;gBACA,IAAI,IAAI,CAAC,UAAU,EAAE;oBACjB,OAAO,IAAI,CAAC,UAAU,CAAC;iBAC1B;gBAED,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE;oBACrC,MAAM,IAAI,mBAAU,CAAC,qDAAqD,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;iBACtG;gBAED,IAAI,CAAC,UAAU,GAAG,IAAI,qBAAS,CAAC,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBAExG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;oBAClB,MAAM,IAAI,mBAAU,CAAC,0BAA0B,CAAC,CAAC;iBACpD;gBAED,MAAM,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC;gBACrC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;gBAC1C,OAAO,IAAI,CAAC,UAAU,CAAC;aAC1B;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,mBAAU,CAAC,uCAAuC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aAC7F;QACL,CAAC,CAAC;QAEK,kBAAa,GAAG,GAAc,EAAE;YACnC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;gBAClB,MAAM,IAAI,mBAAU,CAAC,mEAAmE,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;aACpH;YAED,OAAO,IAAI,CAAC,UAAU,CAAC;QAC3B,CAAC,CAAC;QAEK,gBAAW,GAAG,KAAK,IAAsB,EAAE;YAC9C,IAAI;gBACA,MAAM,IAAI,CAAC,UAAW,CAAC,YAAY,EAAE,CAAC;gBAEtC,OAAO,IAAI,CAAC;aACf;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,oDAAoD,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC;gBACnF,OAAO,KAAK,CAAC;aAChB;QACL,CAAC,CAAC;QAEK,eAAU,GAAG,KAAK,IAAmB,EAAE;YAC1C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;YAE9C,IAAI,IAAI,CAAC,UAAU,EAAE;gBACjB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;aACjC;QACL,CAAC,CAAC;QAEK,YAAO,GAAG,KAAK,IAAmB,EAAE;YACvC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC5B,CAAC,CAAC;IAvDC,CAAC;IAyDI,UAAU,CAAC,iBAA2B;QAC1C,MAAM,cAAc,GAA4B;YAC5C,gBAAgB,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,eAAe,KAAK,YAAE,CAAC,QAAQ,EAAE,GAAG;SAC9E,CAAC;QAEF,IAAI,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE;YAC/B,cAAc,CAAC,GAAG,GAAG;gBACjB,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAC1E,kBAAkB,EAAE,IAAI;aAC3B,CAAC;SACL;QAED,OAAO;YACH,cAAc;YACd,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK;YACvB,IAAI,EAAE;gBACF,OAAO,EAAE,KAAK;gBACd,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC;gBAC/C,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC;gBACjD,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC;aACpD;YACD,KAAK,EAAE;gBACH,KAAK,EAAE;oBACH,0BAA0B;oBAC1B,iCAAiC;oBACjC,4BAA4B;oBAC5B,gCAAgC;oBAChC,iCAAiC;oBACjC,kCAAkC;oBAClC,cAAc;iBACjB;gBACD,GAAG,EAAE,CAAC;aACT;YACD,GAAG,IAAI,CAAC,gBAAgB;YACxB,GAAG,iBAAiB;SACvB,CAAC;IACN,CAAC;CACJ;AArGD,8CAqGC"}

package/dist/helpers/data-access/postgres/IDatabaseInfo.d.ts

@@ -4,4 +4,5 @@ export interface IDatabaseInfo {
     poolMinConnections: number;
     poolMaxConnections: number;
     applicationName: string;
+    sslCaBase64?: string | null;
 }

package/dist/index.d.ts

@@ -2,3 +2,5 @@ export * as InputGateway from "./input-gateway/index";
 export * as IntegrationEngine from "./integration-engine/index";
 export * as OutputGateway from "./output-gateway/index";
 export * as SchemaDefinitions from "./schema-definitions/index";
+export * as AzureHelpers from "./helpers/data-access/azure/index";
+export * as OpenSearchHelpers from "./helpers/data-access/opensearch/index";

package/dist/index.js

@@ -23,10 +23,12 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SchemaDefinitions = exports.OutputGateway = exports.IntegrationEngine = exports.InputGateway = void 0;
+exports.OpenSearchHelpers = exports.AzureHelpers = exports.SchemaDefinitions = exports.OutputGateway = exports.IntegrationEngine = exports.InputGateway = void 0;
 // Library exports
 exports.InputGateway = __importStar(require("./input-gateway/index"));
 exports.IntegrationEngine = __importStar(require("./integration-engine/index"));
 exports.OutputGateway = __importStar(require("./output-gateway/index"));
 exports.SchemaDefinitions = __importStar(require("./schema-definitions/index"));
+exports.AzureHelpers = __importStar(require("./helpers/data-access/azure/index"));
+exports.OpenSearchHelpers = __importStar(require("./helpers/data-access/opensearch/index"));
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kBAAkB;AAClB,sEAA0C;AAC1C,gFAA+C;AAC/C,wEAA2C;AAC3C,gFAAgD"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kBAAkB;AAClB,sEAA0C;AAC1C,gFAA+C;AAC/C,wEAA2C;AAC3C,gFAAgD;AAChD,kFAAiE;AACjE,4FAA2E"}

package/dist/input-gateway/connectors/AMQPConnector.d.ts

@@ -1,5 +1,5 @@
 import { ILogger } from "../../helpers";
-import { IConfiguration } from "../config";
+import type { IConfiguration } from "../config";
 import amqplib from "amqplib";
 import { Disposable } from "tsyringe";
 import IQueueConnector from "../../helpers/connector/interfaces/IQueueConnector";

package/dist/input-gateway/helpers/SaveRawDataMiddleware.d.ts

@@ -1,7 +1,7 @@
 import { ILogger } from "../../helpers";
 import { ISimpleConfig } from "../../helpers/configuration/ISimpleConfig";
 import { AbstractStorageService } from "../../helpers/data-access/storage";
-import { IConfiguration } from "../config";
+import type { IConfiguration } from "../config";
 import { NextFunction, Request, Response } from "express";
 export declare class SaveRawDataMiddleware {
     private readonly log;

package/dist/integration-engine/connectors/AMQPConnector.d.ts

@@ -1,6 +1,6 @@
 import { ILogger } from "../../helpers";
 import IQueueConnector from "../../helpers/connector/interfaces/IQueueConnector";
-import { IConfiguration } from "../config";
+import type { IConfiguration } from "../config";
 import amqplib from "amqplib";
 import { Disposable } from "tsyringe";
 export declare class AMQPConnector implements IQueueConnector, Disposable {

package/dist/integration-engine/data-access/RawDataStorage.d.ts

@@ -1,7 +1,7 @@
 import { ILogger } from "../../helpers/logger/LoggerProvider";
 import { ISimpleConfig } from "../../helpers/configuration/ISimpleConfig";
 import { AbstractStorageService } from "../../helpers/data-access/storage";
-import { IConfiguration } from "../config/abstract";
+import type { IConfiguration } from "../config/abstract";
 export default class RawDataStorage {
     private storageService;
     private readonly log;

package/dist/integration-engine/datasources/BatchWritable.d.ts

@@ -0,0 +1,16 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { Writable } from "node:stream";
+export interface BatchWritableOptions<T> {
+    batchSize: number;
+    onFlush: (batch: T[]) => Promise<void>;
+}
+export declare class BatchWritable<T> extends Writable {
+    private batch;
+    private readonly batchSize;
+    private readonly onFlush;
+    constructor(options: BatchWritableOptions<T>);
+    _write(item: T, _encoding: BufferEncoding, callback: (err?: Error | null) => void): void;
+    _final(callback: (err?: Error | null) => void): void;
+    private flush;
+}

package/dist/integration-engine/datasources/BatchWritable.js

@@ -0,0 +1,42 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BatchWritable = void 0;
+const node_stream_1 = require("node:stream");
+class BatchWritable extends node_stream_1.Writable {
+    constructor(options) {
+        super({
+            objectMode: true,
+        });
+        this.batch = [];
+        this.batchSize = options.batchSize;
+        this.onFlush = options.onFlush;
+    }
+    _write(item, _encoding, callback) {
+        this.batch.push(item);
+        if (this.batch.length < this.batchSize) {
+            callback();
+            return;
+        }
+        this.flush(callback);
+    }
+    _final(callback) {
+        if (this.batch.length === 0) {
+            callback();
+            return;
+        }
+        this.flush(callback);
+    }
+    async flush(callback) {
+        const fullBatch = this.batch;
+        this.batch = [];
+        try {
+            await this.onFlush(fullBatch);
+            callback();
+        }
+        catch (err) {
+            callback(err instanceof Error ? err : new Error(String(err)));
+        }
+    }
+}
+exports.BatchWritable = BatchWritable;
+//# sourceMappingURL=BatchWritable.js.map

package/dist/integration-engine/datasources/BatchWritable.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"BatchWritable.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/BatchWritable.ts"],"names":[],"mappings":";;;AAAA,6CAAuC;AAOvC,MAAa,aAAiB,SAAQ,sBAAQ;IAK1C,YAAY,OAAgC;QACxC,KAAK,CAAC;YACF,UAAU,EAAE,IAAI;SACnB,CAAC,CAAC;QAPC,UAAK,GAAQ,EAAE,CAAC;QAQpB,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACnC,CAAC;IAEe,MAAM,CAAC,IAAO,EAAE,SAAyB,EAAE,QAAsC;QAC7F,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE;YACpC,QAAQ,EAAE,CAAC;YACX,OAAO;SACV;QACD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACzB,CAAC;IAEe,MAAM,CAAC,QAAsC;QACzD,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;YACzB,QAAQ,EAAE,CAAC;YACX,OAAO;SACV;QACD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACzB,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,QAAsC;QACtD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC;QAC7B,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAChB,IAAI;YACA,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAC9B,QAAQ,EAAE,CAAC;SACd;QAAC,OAAO,GAAG,EAAE;YACV,QAAQ,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;SACjE;IACL,CAAC;CACJ;AAxCD,sCAwCC"}

package/dist/integration-engine/datasources/DataSourceStreamedPipeline.d.ts

@@ -0,0 +1,25 @@
+/// <reference types="node" />
+import { IDataSource } from "./IDataSource";
+import { IValidator } from "@golemio/validator";
+import { Transform } from "node:stream";
+import { DataSource } from "./DataSource";
+import { IStreamingProtocolStrategy } from "./protocol-strategy/IStreamingProtocolStrategy";
+export interface DataSourceStreamedPipelineOptions {
+    validator?: IValidator | null;
+    batchSize?: number;
+    timeoutMs?: number;
+}
+export declare class DataSourceStreamedPipeline<T extends object = any> extends DataSource implements IDataSource {
+    static readonly DEFAULT_BATCH_SIZE: number;
+    static readonly DEFAULT_PIPELINE_TIMEOUT_MS: number;
+    private readonly streamTransformer;
+    private readonly batchSize;
+    private readonly timeoutMs;
+    private dataProcessor;
+    constructor(name: string, streamingStrategy: IStreamingProtocolStrategy, streamTransformer: Transform, options?: DataSourceStreamedPipelineOptions);
+    private get streamingStrategy();
+    setDataProcessor(fn: (batch: T[]) => Promise<void>): this;
+    proceed(): Promise<void>;
+    private createBatchWriter;
+    private validateAndProcess;
+}

package/dist/integration-engine/datasources/DataSourceStreamedPipeline.js

@@ -0,0 +1,72 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DataSourceStreamedPipeline = void 0;
+const logger_1 = require("../../helpers/logger");
+const errors_1 = require("@golemio/errors");
+const promises_1 = require("node:stream/promises");
+const BatchWritable_1 = require("./BatchWritable");
+const DataSource_1 = require("./DataSource");
+class DataSourceStreamedPipeline extends DataSource_1.DataSource {
+    constructor(name, streamingStrategy, streamTransformer, options = {}) {
+        super(name, streamingStrategy, null, options.validator ?? null);
+        this.dataProcessor = null;
+        this.streamTransformer = streamTransformer;
+        this.batchSize = options.batchSize ?? DataSourceStreamedPipeline.DEFAULT_BATCH_SIZE;
+        this.timeoutMs = options.timeoutMs ?? DataSourceStreamedPipeline.DEFAULT_PIPELINE_TIMEOUT_MS;
+    }
+    get streamingStrategy() {
+        return this.protocolStrategy;
+    }
+    setDataProcessor(fn) {
+        this.dataProcessor = fn;
+        return this;
+    }
+    async proceed() {
+        if (!this.dataProcessor) {
+            throw new errors_1.GeneralError("No data processor registered. Call setDataProcessor() before proceed().", this.name);
+        }
+        const bodyStream = await this.streamingStrategy.getBodyStream();
+        let totalRecords = 0;
+        const batchWriter = this.createBatchWriter((processedCount) => {
+            totalRecords += processedCount;
+        });
+        try {
+            await (0, promises_1.pipeline)(bodyStream, this.streamTransformer, batchWriter, {
+                signal: AbortSignal.timeout(this.timeoutMs),
+            });
+        }
+        catch (err) {
+            bodyStream.destroy();
+            if (err instanceof errors_1.GeneralError)
+                throw err;
+            throw new errors_1.GeneralError("DataSourceStreamedPipeline pipeline failed.", this.name, err);
+        }
+        this.loggerEmitter.emit(logger_1.LoggerEventType.NumberOfRecords, { name: this.name, numberOfRecords: totalRecords });
+    }
+    createBatchWriter(onBatchProcessed) {
+        const batchSize = this.batchSize;
+        return new BatchWritable_1.BatchWritable({
+            batchSize,
+            onFlush: async (batch) => {
+                const processedCount = batch.length;
+                await this.validateAndProcess(batch);
+                onBatchProcessed(processedCount);
+            },
+        });
+    }
+    async validateAndProcess(batch) {
+        if (this.validator) {
+            try {
+                await this.validator.Validate(batch);
+            }
+            catch (err) {
+                throw new errors_1.GeneralError("Error while validating source data.", this.name, err);
+            }
+        }
+        await this.dataProcessor(batch);
+    }
+}
+exports.DataSourceStreamedPipeline = DataSourceStreamedPipeline;
+DataSourceStreamedPipeline.DEFAULT_BATCH_SIZE = process.env.DATA_BATCH_SIZE ? Number(process.env.DATA_BATCH_SIZE) : 1000;
+DataSourceStreamedPipeline.DEFAULT_PIPELINE_TIMEOUT_MS = 30 * 60 * 1000;
+//# sourceMappingURL=DataSourceStreamedPipeline.js.map

package/dist/integration-engine/datasources/DataSourceStreamedPipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"DataSourceStreamedPipeline.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/DataSourceStreamedPipeline.ts"],"names":[],"mappings":";;;AAAA,iDAAkD;AAElD,4CAA+C;AAG/C,mDAAgD;AAChD,mDAAgD;AAChD,6CAA0C;AAU1C,MAAa,0BAAmD,SAAQ,uBAAU;IAU9E,YACI,IAAY,EACZ,iBAA6C,EAC7C,iBAA4B,EAC5B,UAA6C,EAAE;QAE/C,KAAK,CAAC,IAAI,EAAE,iBAAiD,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC;QAR5F,kBAAa,GAA2C,IAAI,CAAC;QASjE,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,0BAA0B,CAAC,kBAAkB,CAAC;QACpF,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,0BAA0B,CAAC,2BAA2B,CAAC;IACjG,CAAC;IAED,IAAY,iBAAiB;QACzB,OAAO,IAAI,CAAC,gBAAyD,CAAC;IAC1E,CAAC;IAEM,gBAAgB,CAAC,EAAiC;QACrD,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,KAAK,CAAC,OAAO;QAChB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE;YACrB,MAAM,IAAI,qBAAY,CAAC,yEAAyE,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;SAChH;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE,CAAC;QAEhE,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,cAAc,EAAE,EAAE;YAC1D,YAAY,IAAI,cAAc,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,IAAI;YACA,MAAM,IAAA,mBAAQ,EAAC,UAAU,EAAE,IAAI,CAAC,iBAAiB,EAAE,WAAW,EAAE;gBAC5D,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;aAC9C,CAAC,CAAC;SACN;QAAC,OAAO,GAAG,EAAE;YACV,UAAU,CAAC,OAAO,EAAE,CAAC;YACrB,IAAI,GAAG,YAAY,qBAAY;gBAAE,MAAM,GAAG,CAAC;YAC3C,MAAM,IAAI,qBAAY,CAAC,6CAA6C,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;SACzF;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,wBAAe,CAAC,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,eAAe,EAAE,YAAY,EAAE,CAAC,CAAC;IACjH,CAAC;IAEO,iBAAiB,CAAC,gBAAyC;QAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QACjC,OAAO,IAAI,6BAAa,CAAI;YACxB,SAAS;YACT,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;gBACrB,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC;gBACpC,MAAM,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;gBACrC,gBAAgB,CAAC,cAAc,CAAC,CAAC;YACrC,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,KAAU;QACvC,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,IAAI;gBACA,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;aACxC;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,qBAAY,CAAC,qCAAqC,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aACjF;SACJ;QACD,MAAM,IAAI,CAAC,aAAc,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;;AA7EL,gEA8EC;AA7E0B,6CAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,AAA3E,CAA4E;AAC9F,sDAA2B,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,AAAjB,CAAkB"}

package/dist/integration-engine/datasources/datatype-strategy/JSONStreamTransformer.d.ts

@@ -0,0 +1,5 @@
+/// <reference types="node" />
+import { Transform } from "node:stream";
+export declare class JSONStreamTransformer {
+    static parse(path: string): Transform;
+}

package/dist/integration-engine/datasources/datatype-strategy/JSONStreamTransformer.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JSONStreamTransformer = void 0;
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const JSONStream = require("JSONStream");
+class JSONStreamTransformer {
+    static parse(path) {
+        return JSONStream.parse(path);
+    }
+}
+exports.JSONStreamTransformer = JSONStreamTransformer;
+//# sourceMappingURL=JSONStreamTransformer.js.map

package/dist/integration-engine/datasources/datatype-strategy/JSONStreamTransformer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"JSONStreamTransformer.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/JSONStreamTransformer.ts"],"names":[],"mappings":";;;AAGA,8DAA8D;AAC9D,MAAM,UAAU,GAAG,OAAO,CAAC,YAAY,CAA4B,CAAC;AACpE,MAAa,qBAAqB;IAC9B,MAAM,CAAC,KAAK,CAAC,IAAY;QACrB,OAAO,UAAU,CAAC,KAAK,CAAC,IAAI,CAAyB,CAAC;IAC1D,CAAC;CACJ;AAJD,sDAIC"}

package/dist/integration-engine/datasources/index.d.ts

@@ -1,16 +1,21 @@
+export * from "./BatchWritable";
 export * from "./DataSource";
 export * from "./DataSourceStream";
 export * from "./DataSourceStreamManager";
 export * from "./DataSourceStreamed";
+export * from "./DataSourceStreamedPipeline";
 export * from "./IDataSource";
 export * from "./datatype-strategy/CSVDataTypeStrategy";
 export * from "./datatype-strategy/IDataTypeStrategy";
 export * from "./datatype-strategy/JSONDataTypeStrategy";
+export * from "./datatype-strategy/JSONStreamTransformer";
 export * from "./datatype-strategy/XMLDataTypeStrategy";
 export * from "./protocol-strategy/FTPProtocolStrategy";
+export * from "./protocol-strategy/HTTPRequestProtocolStrategyStreamed";
 export * from "./protocol-strategy/GoogleCloudStorageProtocolStrategy";
 export * from "./protocol-strategy/PaginatedHTTPProtocolStrategy";
 export * from "./protocol-strategy/IProtocolStrategy";
+export * from "./protocol-strategy/IStreamingProtocolStrategy";
 export * from "./protocol-strategy/PostgresProtocolStrategy";
 export * from "./protocol-strategy/PostgresProtocolStrategyStreamed";
 export * from "./protocol-strategy/ProtocolStrategy";

package/dist/integration-engine/datasources/index.js

@@ -15,19 +15,24 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 /* ie/datasources/index.ts */
+__exportStar(require("./BatchWritable"), exports);
 __exportStar(require("./DataSource"), exports);
 __exportStar(require("./DataSourceStream"), exports);
 __exportStar(require("./DataSourceStreamManager"), exports);
 __exportStar(require("./DataSourceStreamed"), exports);
+__exportStar(require("./DataSourceStreamedPipeline"), exports);
 __exportStar(require("./IDataSource"), exports);
 __exportStar(require("./datatype-strategy/CSVDataTypeStrategy"), exports);
 __exportStar(require("./datatype-strategy/IDataTypeStrategy"), exports);
 __exportStar(require("./datatype-strategy/JSONDataTypeStrategy"), exports);
+__exportStar(require("./datatype-strategy/JSONStreamTransformer"), exports);
 __exportStar(require("./datatype-strategy/XMLDataTypeStrategy"), exports);
 __exportStar(require("./protocol-strategy/FTPProtocolStrategy"), exports);
+__exportStar(require("./protocol-strategy/HTTPRequestProtocolStrategyStreamed"), exports);
 __exportStar(require("./protocol-strategy/GoogleCloudStorageProtocolStrategy"), exports);
 __exportStar(require("./protocol-strategy/PaginatedHTTPProtocolStrategy"), exports);
 __exportStar(require("./protocol-strategy/IProtocolStrategy"), exports);
+__exportStar(require("./protocol-strategy/IStreamingProtocolStrategy"), exports);
 __exportStar(require("./protocol-strategy/PostgresProtocolStrategy"), exports);
 __exportStar(require("./protocol-strategy/PostgresProtocolStrategyStreamed"), exports);
 __exportStar(require("./protocol-strategy/ProtocolStrategy"), exports);

package/dist/integration-engine/datasources/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6BAA6B;AAC7B,+CAA6B;AAC7B,qDAAmC;AACnC,4DAA0C;AAC1C,uDAAqC;AACrC,gDAA8B;AAC9B,0EAAwD;AACxD,wEAAsD;AACtD,2EAAyD;AACzD,0EAAwD;AACxD,0EAAwD;AACxD,yFAAuE;AACvE,oFAAkE;AAClE,wEAAsD;AACtD,+EAA6D;AAC7D,uFAAqE;AACrE,uEAAqD;AACrD,2EAAyD"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6BAA6B;AAC7B,kDAAgC;AAChC,+CAA6B;AAC7B,qDAAmC;AACnC,4DAA0C;AAC1C,uDAAqC;AACrC,+DAA6C;AAC7C,gDAA8B;AAC9B,0EAAwD;AACxD,wEAAsD;AACtD,2EAAyD;AACzD,4EAA0D;AAC1D,0EAAwD;AACxD,0EAAwD;AACxD,0FAAwE;AACxE,yFAAuE;AACvE,oFAAkE;AAClE,wEAAsD;AACtD,iFAA+D;AAC/D,+EAA6D;AAC7D,uFAAqE;AACrE,uEAAqD;AACrD,2EAAyD"}

package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.d.ts

@@ -1,12 +1,16 @@
+/// <reference types="node" />
 import { IProtocolStrategy } from "./IProtocolStrategy";
+import { Readable } from "node:stream";
 import { DataSourceStream } from "../DataSourceStream";
 import { HTTPRequestProtocolStrategy } from "./HTTPRequestProtocolStrategy";
-export declare class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy implements IProtocolStrategy {
+import { IStreamingProtocolStrategy } from "./IStreamingProtocolStrategy";
+export declare class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy implements IProtocolStrategy, IStreamingProtocolStrategy {
     private streamTransform;
     /**
      * @param streamTransform data will be piped  to this stream if provided
      */
     setStreamTransformer: (streamTransform: any) => HTTPRequestProtocolStrategyStreamed;
     getData(): Promise<DataSourceStream>;
+    getBodyStream(): Promise<Readable>;
     getRawData: () => Promise<any>;
 }

package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.HTTPRequestProtocolStrategyStreamed = void 0;
 const errors_1 = require("@golemio/errors");
+const node_zlib_1 = require("node:zlib");
 const DataSourceStream_1 = require("../DataSourceStream");
 const HTTPRequestProtocolStrategy_1 = require("./HTTPRequestProtocolStrategy");
 class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy_1.HTTPRequestProtocolStrategy {
@@ -16,7 +17,8 @@ class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy_1.
         };
         this.getRawData = async () => {
             if (this.connectionSettings.compression) {
-                throw new errors_1.GeneralError("compressed resources are not supported in HTTPRequestProtocolStrategyStreamed yet", this.constructor.name);
+                throw new errors_1.GeneralError("Compressed resources are not supported in the legacy DataSourceStream path;" +
+                    "use DataSourceStreamedPipeline instead", this.constructor.name);
             }
             try {
                 const result = await this.getResponseInternal();
@@ -97,6 +99,33 @@ class HTTPRequestProtocolStrategyStreamed extends HTTPRequestProtocolStrategy_1.
         }
         return outStream;
     }
+    async getBodyStream() {
+        try {
+            const result = await this.getResponseInternal();
+            if (result.statusCode < 200 || result.statusCode >= 300) {
+                result.body?.resume?.(); // drain to avoid connection leak (some statuses have no body)
+                throw new errors_1.GeneralError(`Unexpected HTTP status ${result.statusCode}`, this.constructor.name);
+            }
+            let body = result.body;
+            if (this.connectionSettings.compression === "gzip") {
+                const encoding = String(result.headers?.["content-encoding"] ?? "").toLowerCase();
+                if (encoding !== "gzip") {
+                    body?.resume?.();
+                    throw new errors_1.GeneralError(`Expected gzip-encoded response but server returned Content-Encoding: "${encoding || "<none>"}"`, this.constructor.name);
+                }
+                body = body.pipe((0, node_zlib_1.createGunzip)());
+            }
+            return this.teeRawDataToStorage(body, {
+                statusCode: result.statusCode,
+                headers: result.headers,
+            });
+        }
+        catch (err) {
+            if (err instanceof errors_1.GeneralError)
+                throw err;
+            throw new errors_1.GeneralError("Error while getting body stream from server.", this.constructor.name, err);
+        }
+    }
 }
 exports.HTTPRequestProtocolStrategyStreamed = HTTPRequestProtocolStrategyStreamed;
 //# sourceMappingURL=HTTPRequestProtocolStrategyStreamed.js.map

package/dist/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.js.map

@@ -1 +1 @@
-{"version":3,"file":"HTTPRequestProtocolStrategyStreamed.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.ts"],"names":[],"mappings":";;;AACA,4CAAqE;AACrE,0DAAuD;AACvD,+EAA4E;AAE5E,MAAa,mCAAoC,SAAQ,yDAA2B;IAApF;;QAGI;;WAEG;QACI,yBAAoB,GAAG,CAAC,eAAoB,EAAuC,EAAE;YACxF,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;YACvC,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC;QAsEK,eAAU,GAAG,KAAK,IAAkB,EAAE;YACzC,IAAI,IAAI,CAAC,kBAAkB,CAAC,WAAW,EAAE;gBACrC,MAAM,IAAI,qBAAY,CAClB,mFAAmF,EACnF,IAAI,CAAC,WAAW,CAAC,IAAI,CACxB,CAAC;aACL;YAED,IAAI;gBACA,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAEhD,OAAO;oBACH,IAAI,EAAE,MAAM,CAAC,IAAI;iBACpB,CAAC;aACL;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,qBAAY,CAAC,uCAAuC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aAC/F;QACL,CAAC,CAAC;IACN,CAAC;IAtFU,KAAK,CAAC,OAAO;QAChB,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QAEzC,MAAM,SAAS,GAAG,IAAI,mCAAgB,CAAC;YACnC,UAAU,EAAE,IAAI;YAChB,IAAI,EAAE,GAAG,EAAE;gBACP,OAAO;YACX,CAAC;SACJ,CAAC,CAAC;QAEH,IAAI,aAAa,GAAG,KAAK,CAAC;QAE1B,IAAI,IAAI,CAAC,eAAe,EAAE;YACtB,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;gBAChC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACrC,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACtB,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC;YAC/B,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;gBAC1C,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBAChC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBAC5C,IAAI,KAAK,YAAY,6BAAoB,EAAE;oBACvC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;iBAClC;qBAAM;oBACH,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,6BAA6B,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;iBAC1G;YACL,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBAClC,IAAI,KAAK,YAAY,6BAAoB,EAAE;oBACvC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;iBAC7C;qBAAM;oBACH,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;iBAC7G;YACL,CAAC,CAAC,CAAC;SACN;aAAM;YACH,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;gBAChC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACxB,qCAAqC;gBACrC,IAAI,CAAC,aAAa,EAAE;oBAChB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACrB,aAAa,GAAG,IAAI,CAAC;iBACxB;YACL,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACtB,uCAAuC;gBACvC,IAAI,CAAC,aAAa,EAAE;oBAChB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACrB,aAAa,GAAG,IAAI,CAAC;iBACxB;YACL,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBAClC,IAAI,KAAK,YAAY,6BAAoB,EAAE;oBACvC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;iBAClC;qBAAM;oBACH,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;iBAClG;YACL,CAAC,CAAC,CAAC;SACN;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;CAoBJ;AAjGD,kFAiGC"}
+{"version":3,"file":"HTTPRequestProtocolStrategyStreamed.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/HTTPRequestProtocolStrategyStreamed.ts"],"names":[],"mappings":";;;AACA,4CAAqE;AAErE,yCAAyC;AACzC,0DAAuD;AACvD,+EAA4E;AAG5E,MAAa,mCACT,SAAQ,yDAA2B;IADvC;;QAMI;;WAEG;QACI,yBAAoB,GAAG,CAAC,eAAoB,EAAuC,EAAE;YACxF,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;YACvC,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC;QAuGK,eAAU,GAAG,KAAK,IAAkB,EAAE;YACzC,IAAI,IAAI,CAAC,kBAAkB,CAAC,WAAW,EAAE;gBACrC,MAAM,IAAI,qBAAY,CAClB,6EAA6E;oBACzE,wCAAwC,EAC5C,IAAI,CAAC,WAAW,CAAC,IAAI,CACxB,CAAC;aACL;YAED,IAAI;gBACA,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAEhD,OAAO;oBACH,IAAI,EAAE,MAAM,CAAC,IAAI;iBACpB,CAAC;aACL;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,qBAAY,CAAC,uCAAuC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aAC/F;QACL,CAAC,CAAC;IACN,CAAC;IAxHU,KAAK,CAAC,OAAO;QAChB,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QAEzC,MAAM,SAAS,GAAG,IAAI,mCAAgB,CAAC;YACnC,UAAU,EAAE,IAAI;YAChB,IAAI,EAAE,GAAG,EAAE;gBACP,OAAO;YACX,CAAC;SACJ,CAAC,CAAC;QAEH,IAAI,aAAa,GAAG,KAAK,CAAC;QAE1B,IAAI,IAAI,CAAC,eAAe,EAAE;YACtB,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;gBAChC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACrC,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACtB,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,CAAC;YAC/B,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;gBAC1C,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBAChC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBAC5C,IAAI,KAAK,YAAY,6BAAoB,EAAE;oBACvC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;iBAClC;qBAAM;oBACH,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,6BAA6B,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;iBAC1G;YACL,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBAClC,IAAI,KAAK,YAAY,6BAAoB,EAAE;oBACvC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;iBAC7C;qBAAM;oBACH,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;iBAC7G;YACL,CAAC,CAAC,CAAC;SACN;aAAM;YACH,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;gBAChC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACxB,qCAAqC;gBACrC,IAAI,CAAC,aAAa,EAAE;oBAChB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACrB,aAAa,GAAG,IAAI,CAAC;iBACxB;YACL,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACtB,uCAAuC;gBACvC,IAAI,CAAC,aAAa,EAAE;oBAChB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACrB,aAAa,GAAG,IAAI,CAAC;iBACxB;YACL,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;gBAClC,IAAI,KAAK,YAAY,6BAAoB,EAAE;oBACvC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;iBAClC;qBAAM;oBACH,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,qBAAY,CAAC,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;iBAClG;YACL,CAAC,CAAC,CAAC;SACN;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAEM,KAAK,CAAC,aAAa;QACtB,IAAI;YACA,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAEhD,IAAI,MAAM,CAAC,UAAU,GAAG,GAAG,IAAI,MAAM,CAAC,UAAU,IAAI,GAAG,EAAE;gBACrD,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,8DAA8D;gBACvF,MAAM,IAAI,qBAAY,CAAC,0BAA0B,MAAM,CAAC,UAAU,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;aAChG;YAED,IAAI,IAAI,GAAG,MAAM,CAAC,IAA2B,CAAC;YAE9C,IAAI,IAAI,CAAC,kBAAkB,CAAC,WAAW,KAAK,MAAM,EAAE;gBAChD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;gBAClF,IAAI,QAAQ,KAAK,MAAM,EAAE;oBACrB,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC;oBACjB,MAAM,IAAI,qBAAY,CAClB,yEAAyE,QAAQ,IAAI,QAAQ,GAAG,EAChG,IAAI,CAAC,WAAW,CAAC,IAAI,CACxB,CAAC;iBACL;gBACD,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAA,wBAAY,GAAE,CAAC,CAAC;aACpC;YAED,OAAO,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE;gBAClC,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,OAAO,EAAE,MAAM,CAAC,OAAO;aAC1B,CAAC,CAAC;SACN;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,GAAG,YAAY,qBAAY;gBAAE,MAAM,GAAG,CAAC;YAC3C,MAAM,IAAI,qBAAY,CAAC,8CAA8C,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;SACtG;IACL,CAAC;CAqBJ;AAtID,kFAsIC"}

package/dist/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.d.ts

@@ -0,0 +1,5 @@
+/// <reference types="node" />
+import { Readable } from "stream";
+export interface IStreamingProtocolStrategy {
+    getBodyStream(): Promise<Readable>;
+}

package/dist/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=IStreamingProtocolStrategy.js.map

package/dist/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IStreamingProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.ts"],"names":[],"mappings":""}

package/dist/integration-engine/datasources/protocol-strategy/ProtocolStrategy.d.ts

@@ -1,3 +1,5 @@
+/// <reference types="node" />
+import { Readable } from "node:stream";
 import { DataSourceStream } from "../DataSourceStream";
 import { IProtocolStrategy, ProtocolStrategySettings } from "./IProtocolStrategy";
 export declare class ProtocolStrategy implements IProtocolStrategy {
@@ -18,6 +20,7 @@ export declare class ProtocolStrategy implements IProtocolStrategy {
     }>;
     getLastModified: () => Promise<string | null>;
     protected getRawData(...args: any): Promise<any | DataSourceStream>;
+    protected teeRawDataToStorage(bodyStream: Readable, meta?: unknown): Readable;
     private prepareRawData;
     private pushRawDataToS3Stream;
 }

package/dist/integration-engine/datasources/protocol-strategy/ProtocolStrategy.js

@@ -9,7 +9,8 @@ const RawDataStorage_1 = __importDefault(require("../../data-access/RawDataStora
 const ioc_1 = require("../../ioc");
 const Di_1 = require("../../ioc/Di");
 const errors_1 = require("@golemio/errors");
-const stream_1 = require("stream");
+const node_stream_1 = require("node:stream");
+const promises_1 = require("node:stream/promises");
 const DataSourceStream_1 = require("../DataSourceStream");
 class ProtocolStrategy {
     constructor(settings) {
@@ -93,9 +94,9 @@ class ProtocolStrategy {
         let dataSaveCalled = false;
         let protocolDataStream;
         // bit paranoid but ...
-        if (rawData && (rawData instanceof stream_1.Readable || rawData.constructor.name === "Request") && rawData.on) {
+        if (rawData && (rawData instanceof node_stream_1.Readable || rawData.constructor.name === "Request") && rawData.on) {
             protocolDataStream = rawData;
-            const S3Stream = new stream_1.Readable({
+            const S3Stream = new node_stream_1.Readable({
                 objectMode: false,
                 read: () => {
                     return;
@@ -149,6 +150,44 @@ class ProtocolStrategy {
     async getRawData(...args) {
         throw new Error("getRawData: Method not implemented.");
     }
+    teeRawDataToStorage(bodyStream, meta = {}) {
+        if (!this.storageEnabled || !this.saveRawDataWhitelist[this.caller]) {
+            return bodyStream;
+        }
+        const S3Stream = new node_stream_1.Readable({
+            objectMode: false,
+            read: () => {
+                return;
+            },
+        });
+        const consumer = new node_stream_1.PassThrough();
+        let consumerErrorFired = false;
+        (0, promises_1.pipeline)(bodyStream, consumer).catch((err) => {
+            if (!consumerErrorFired) {
+                this.logger.error(`Body stream source failed for ${this.caller}`, err);
+            }
+        });
+        let saveCalled = false;
+        consumer.on("data", async (chunk) => {
+            await this.pushRawDataToS3Stream(chunk, S3Stream);
+            if (!saveCalled) {
+                this.rawDataStorage.save(S3Stream, meta, this.caller);
+                saveCalled = true;
+            }
+        });
+        consumer.on("end", () => {
+            S3Stream.push(null);
+        });
+        consumer.on("error", (err) => {
+            consumerErrorFired = true;
+            this.logger.warn(`Raw data save to storage interrupted for ${this.caller}`, err);
+            S3Stream.push(null);
+        });
+        S3Stream.on("error", (err) => {
+            this.logger.error(`Error on saving raw data for ${this.caller}`, err);
+        });
+        return consumer;
+    }
 }
 exports.ProtocolStrategy = ProtocolStrategy;
 //# sourceMappingURL=ProtocolStrategy.js.map