@golemio/core 3.3.0-dev.2644036130 → 3.3.0-rc.2646964518

package/dist/integration-engine/datasources/DataSourceStreamedPipeline.js

@@ -1,72 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DataSourceStreamedPipeline = void 0;
-const logger_1 = require("../../helpers/logger");
-const errors_1 = require("@golemio/errors");
-const promises_1 = require("node:stream/promises");
-const BatchWritable_1 = require("./BatchWritable");
-const DataSource_1 = require("./DataSource");
-class DataSourceStreamedPipeline extends DataSource_1.DataSource {
-    constructor(name, streamingStrategy, streamTransformer, options = {}) {
-        super(name, streamingStrategy, null, options.validator ?? null);
-        this.dataProcessor = null;
-        this.streamTransformer = streamTransformer;
-        this.batchSize = options.batchSize ?? DataSourceStreamedPipeline.DEFAULT_BATCH_SIZE;
-        this.timeoutMs = options.timeoutMs ?? DataSourceStreamedPipeline.DEFAULT_PIPELINE_TIMEOUT_MS;
-    }
-    get streamingStrategy() {
-        return this.protocolStrategy;
-    }
-    setDataProcessor(fn) {
-        this.dataProcessor = fn;
-        return this;
-    }
-    async proceed() {
-        if (!this.dataProcessor) {
-            throw new errors_1.GeneralError("No data processor registered. Call setDataProcessor() before proceed().", this.name);
-        }
-        const bodyStream = await this.streamingStrategy.getBodyStream();
-        let totalRecords = 0;
-        const batchWriter = this.createBatchWriter((processedCount) => {
-            totalRecords += processedCount;
-        });
-        try {
-            await (0, promises_1.pipeline)(bodyStream, this.streamTransformer, batchWriter, {
-                signal: AbortSignal.timeout(this.timeoutMs),
-            });
-        }
-        catch (err) {
-            bodyStream.destroy();
-            if (err instanceof errors_1.GeneralError)
-                throw err;
-            throw new errors_1.GeneralError("DataSourceStreamedPipeline pipeline failed.", this.name, err);
-        }
-        this.loggerEmitter.emit(logger_1.LoggerEventType.NumberOfRecords, { name: this.name, numberOfRecords: totalRecords });
-    }
-    createBatchWriter(onBatchProcessed) {
-        const batchSize = this.batchSize;
-        return new BatchWritable_1.BatchWritable({
-            batchSize,
-            onFlush: async (batch) => {
-                const processedCount = batch.length;
-                await this.validateAndProcess(batch);
-                onBatchProcessed(processedCount);
-            },
-        });
-    }
-    async validateAndProcess(batch) {
-        if (this.validator) {
-            try {
-                await this.validator.Validate(batch);
-            }
-            catch (err) {
-                throw new errors_1.GeneralError("Error while validating source data.", this.name, err);
-            }
-        }
-        await this.dataProcessor(batch);
-    }
-}
-exports.DataSourceStreamedPipeline = DataSourceStreamedPipeline;
-DataSourceStreamedPipeline.DEFAULT_BATCH_SIZE = process.env.DATA_BATCH_SIZE ? Number(process.env.DATA_BATCH_SIZE) : 1000;
-DataSourceStreamedPipeline.DEFAULT_PIPELINE_TIMEOUT_MS = 30 * 60 * 1000;
-//# sourceMappingURL=DataSourceStreamedPipeline.js.map

package/dist/integration-engine/datasources/DataSourceStreamedPipeline.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"DataSourceStreamedPipeline.js","sourceRoot":"","sources":["../../../src/integration-engine/datasources/DataSourceStreamedPipeline.ts"],"names":[],"mappings":";;;AAAA,iDAAkD;AAElD,4CAA+C;AAG/C,mDAAgD;AAChD,mDAAgD;AAChD,6CAA0C;AAU1C,MAAa,0BAAmD,SAAQ,uBAAU;IAU9E,YACI,IAAY,EACZ,iBAA6C,EAC7C,iBAA4B,EAC5B,UAA6C,EAAE;QAE/C,KAAK,CAAC,IAAI,EAAE,iBAAiD,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC;QAR5F,kBAAa,GAA2C,IAAI,CAAC;QASjE,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,0BAA0B,CAAC,kBAAkB,CAAC;QACpF,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,0BAA0B,CAAC,2BAA2B,CAAC;IACjG,CAAC;IAED,IAAY,iBAAiB;QACzB,OAAO,IAAI,CAAC,gBAAyD,CAAC;IAC1E,CAAC;IAEM,gBAAgB,CAAC,EAAiC;QACrD,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,KAAK,CAAC,OAAO;QAChB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,MAAM,IAAI,qBAAY,CAAC,yEAAyE,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACjH,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE,CAAC;QAEhE,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAC,cAAc,EAAE,EAAE;YAC1D,YAAY,IAAI,cAAc,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACD,MAAM,IAAA,mBAAQ,EAAC,UAAU,EAAE,IAAI,CAAC,iBAAiB,EAAE,WAAW,EAAE;gBAC5D,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;aAC9C,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,UAAU,CAAC,OAAO,EAAE,CAAC;YACrB,IAAI,GAAG,YAAY,qBAAY;gBAAE,MAAM,GAAG,CAAC;YAC3C,MAAM,IAAI,qBAAY,CAAC,6CAA6C,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC1F,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,wBAAe,CAAC,eAAe,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,eAAe,EAAE,YAAY,EAAE,CAAC,CAAC;IACjH,CAAC;IAEO,iBAAiB,CAAC,gBAAyC;QAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QACjC,OAAO,IAAI,6BAAa,CAAI;YACxB,SAAS;YACT,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;gBACrB,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC;gBACpC,MAAM,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;gBACrC,gBAAgB,CAAC,cAAc,CAAC,CAAC;YACrC,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,KAAU;QACvC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACzC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,qBAAY,CAAC,qCAAqC,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAClF,CAAC;QACL,CAAC;QACD,MAAM,IAAI,CAAC,aAAc,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;;AA7EL,gEA8EC;AA7E0B,6CAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,AAA3E,CAA4E;AAC9F,sDAA2B,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,AAAjB,CAAkB"}

package/dist/integration-engine/datasources/datatype-strategy/JSONStreamTransformer.d.ts

@@ -1,4 +0,0 @@
-import { Transform } from "node:stream";
-export declare class JSONStreamTransformer {
-    static parse(path: string): Transform;
-}

package/dist/integration-engine/datasources/datatype-strategy/JSONStreamTransformer.js

@@ -1,12 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.JSONStreamTransformer = void 0;
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const JSONStream = require("JSONStream");
-class JSONStreamTransformer {
-    static parse(path) {
-        return JSONStream.parse(path);
-    }
-}
-exports.JSONStreamTransformer = JSONStreamTransformer;
-//# sourceMappingURL=JSONStreamTransformer.js.map

package/dist/integration-engine/datasources/datatype-strategy/JSONStreamTransformer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"JSONStreamTransformer.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/datatype-strategy/JSONStreamTransformer.ts"],"names":[],"mappings":";;;AAGA,8DAA8D;AAC9D,MAAM,UAAU,GAAG,OAAO,CAAC,YAAY,CAA4B,CAAC;AACpE,MAAa,qBAAqB;IAC9B,MAAM,CAAC,KAAK,CAAC,IAAY;QACrB,OAAO,UAAU,CAAC,KAAK,CAAC,IAAI,CAAyB,CAAC;IAC1D,CAAC;CACJ;AAJD,sDAIC"}

package/dist/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.d.ts

@@ -1,4 +0,0 @@
-import { Readable } from "stream";
-export interface IStreamingProtocolStrategy {
-    getBodyStream(): Promise<Readable>;
-}

package/dist/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.js

@@ -1,3 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-//# sourceMappingURL=IStreamingProtocolStrategy.js.map

package/dist/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"IStreamingProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/IStreamingProtocolStrategy.ts"],"names":[],"mappings":""}