npm - @golemio/core - Versions diffs - 1.2.24 → 1.2.25-dev.722047718 - Mend

@golemio/core 1.2.24 → 1.2.25-dev.722047718

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (46) hide show

package/dist/integration-engine/data-access/pubsub/AbstractPubSubChannel.d.ts ADDED Viewed

@@ -0,0 +1,5 @@
+import { Subscriber } from "./subscribers/AbstractSubscriber";
+export declare abstract class PubSubChannel {
+    abstract publishMessage(message: string): Promise<void>;
+    abstract createSubscriber(): Subscriber;
+}

package/dist/integration-engine/data-access/pubsub/AbstractPubSubChannel.js ADDED Viewed

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PubSubChannel = void 0;
+class PubSubChannel {
+}
+exports.PubSubChannel = PubSubChannel;
+//# sourceMappingURL=AbstractPubSubChannel.js.map

package/dist/integration-engine/data-access/pubsub/AbstractPubSubChannel.js.map ADDED Viewed

	@@ -0,0 +1 @@
1	+ {"version":3,"file":"AbstractPubSubChannel.js","sourceRoot":"","sources":["../../../../src/integration-engine/data-access/pubsub/AbstractPubSubChannel.ts"],"names":[],"mappings":";;;AAEA,MAAsB,aAAa;CAGlC;AAHD,sCAGC"}

package/dist/integration-engine/data-access/pubsub/RedisPubSubChannel.d.ts ADDED Viewed

@@ -0,0 +1,13 @@
+import { PubSubChannel } from "./AbstractPubSubChannel";
+import { RedisSubscriber } from "./subscribers/RedisSubscriber";
+interface IPublishMessageOptions {
+    channelSuffix?: string;
+}
+export declare class RedisPubSubChannel extends PubSubChannel {
+    private readonly channel;
+    private readonly connection;
+    constructor(channel: string);
+    publishMessage(message: string, options?: IPublishMessageOptions): Promise<void>;
+    createSubscriber(): RedisSubscriber;
+}
+export {};

package/dist/integration-engine/data-access/pubsub/RedisPubSubChannel.js ADDED Viewed

@@ -0,0 +1,34 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisPubSubChannel = void 0;
+const RedisConnector_1 = require("../../connectors/RedisConnector");
+const AbstractPubSubChannel_1 = require("./AbstractPubSubChannel");
+const getRedisChannel_1 = require("./helpers/getRedisChannel");
+const RedisSubscriber_1 = require("./subscribers/RedisSubscriber");
+class RedisPubSubChannel extends AbstractPubSubChannel_1.PubSubChannel {
+    constructor(channel) {
+        super();
+        this.channel = channel;
+        this.connection = RedisConnector_1.RedisConnector.getConnection();
+    }
+    publishMessage(message, options = {}) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const redisChannel = (0, getRedisChannel_1.getRedisChannel)(this.channel, options.channelSuffix);
+            yield this.connection.publish(redisChannel, message);
+        });
+    }
+    createSubscriber() {
+        return new RedisSubscriber_1.RedisSubscriber(this.channel);
+    }
+}
+exports.RedisPubSubChannel = RedisPubSubChannel;
+//# sourceMappingURL=RedisPubSubChannel.js.map

package/dist/integration-engine/data-access/pubsub/RedisPubSubChannel.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"RedisPubSubChannel.js","sourceRoot":"","sources":["../../../../src/integration-engine/data-access/pubsub/RedisPubSubChannel.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,oEAA+D;AAE/D,mEAAwD;AACxD,+DAA4D;AAC5D,mEAAgE;AAMhE,MAAa,kBAAmB,SAAQ,qCAAa;IAGjD,YAA6B,OAAe;QACxC,KAAK,EAAE,CAAC;QADiB,YAAO,GAAP,OAAO,CAAQ;QAExC,IAAI,CAAC,UAAU,GAAG,+BAAc,CAAC,aAAa,EAAE,CAAC;IACrD,CAAC;IAEY,cAAc,CAAC,OAAe,EAAE,UAAkC,EAAE;;YAC7E,MAAM,YAAY,GAAG,IAAA,iCAAe,EAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;YAC1E,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACzD,CAAC;KAAA;IAEM,gBAAgB;QACnB,OAAO,IAAI,iCAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC7C,CAAC;CACJ;AAhBD,gDAgBC"}

package/dist/integration-engine/data-access/pubsub/helpers/getRedisChannel.d.ts ADDED Viewed

	@@ -0,0 +1 @@
1	+ export declare const getRedisChannel: (channel: string, channelSuffix: string \| undefined) => string;

package/dist/integration-engine/data-access/pubsub/helpers/getRedisChannel.js ADDED Viewed

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getRedisChannel = void 0;
+const getRedisChannel = (channel, channelSuffix) => {
+    return channelSuffix ? `${channel}:${channelSuffix}` : channel;
+};
+exports.getRedisChannel = getRedisChannel;
+//# sourceMappingURL=getRedisChannel.js.map

package/dist/integration-engine/data-access/pubsub/helpers/getRedisChannel.js.map ADDED Viewed

	@@ -0,0 +1 @@
1	+ {"version":3,"file":"getRedisChannel.js","sourceRoot":"","sources":["../../../../../src/integration-engine/data-access/pubsub/helpers/getRedisChannel.ts"],"names":[],"mappings":";;;AAAO,MAAM,eAAe,GAAG,CAAC,OAAe,EAAE,aAAiC,EAAU,EAAE;IAC1F,OAAO,aAAa,CAAC,CAAC,CAAC,GAAG,OAAO,IAAI,aAAa,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;AACnE,CAAC,CAAC;AAFW,QAAA,eAAe,mBAE1B"}

package/dist/integration-engine/data-access/pubsub/index.d.ts ADDED Viewed

	@@ -0,0 +1 @@
1	+ export * from "./RedisPubSubChannel";

package/dist/integration-engine/data-access/pubsub/index.js ADDED Viewed

@@ -0,0 +1,18 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./RedisPubSubChannel"), exports);
+//# sourceMappingURL=index.js.map

package/dist/integration-engine/data-access/pubsub/index.js.map ADDED Viewed

	@@ -0,0 +1 @@
1	+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/integration-engine/data-access/pubsub/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,uDAAqC"}

package/dist/integration-engine/data-access/pubsub/subscribers/AbstractSubscriber.d.ts ADDED Viewed

@@ -0,0 +1,5 @@
+import { ISubscribeMessage } from "./interfaces/ISubscribeMessage";
+export declare abstract class Subscriber {
+    abstract subscribe(): Promise<AsyncGenerator<ISubscribeMessage>>;
+    abstract unsubscribeAll(): Promise<void>;
+}

package/dist/integration-engine/data-access/pubsub/subscribers/AbstractSubscriber.js ADDED Viewed

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Subscriber = void 0;
+class Subscriber {
+}
+exports.Subscriber = Subscriber;
+//# sourceMappingURL=AbstractSubscriber.js.map

package/dist/integration-engine/data-access/pubsub/subscribers/AbstractSubscriber.js.map ADDED Viewed

	@@ -0,0 +1 @@
1	+ {"version":3,"file":"AbstractSubscriber.js","sourceRoot":"","sources":["../../../../../src/integration-engine/data-access/pubsub/subscribers/AbstractSubscriber.ts"],"names":[],"mappings":";;;AAEA,MAAsB,UAAU;CAG/B;AAHD,gCAGC"}

package/dist/integration-engine/data-access/pubsub/subscribers/RedisSubscriber.d.ts ADDED Viewed

@@ -0,0 +1,17 @@
+import { Subscriber } from "./AbstractSubscriber";
+import { ISubscribeMessage } from "./interfaces/ISubscribeMessage";
+interface ISubscribeOptions {
+    channelSuffix?: string;
+    maxMessages?: number;
+}
+export declare class RedisSubscriber extends Subscriber {
+    private readonly channel;
+    private static DEFAULT_MAX_MESSAGES;
+    private readonly connection;
+    private activeChannels;
+    constructor(channel: string);
+    subscribe(options?: ISubscribeOptions): Promise<AsyncGenerator<ISubscribeMessage>>;
+    unsubscribeAll(): Promise<void>;
+    private listenForMessages;
+}
+export {};

package/dist/integration-engine/data-access/pubsub/subscribers/RedisSubscriber.js ADDED Viewed

@@ -0,0 +1,91 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); }
+var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var g = generator.apply(thisArg, _arguments || []), i, q = [];
+    return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i;
+    function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; }
+    function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
+    function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
+    function fulfill(value) { resume("next", value); }
+    function reject(value) { resume("throw", value); }
+    function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisSubscriber = void 0;
+const config_1 = require("../../../config");
+const helpers_1 = require("../../../helpers");
+const errors_1 = require("@golemio/errors");
+const ioredis_1 = __importDefault(require("ioredis"));
+const getRedisChannel_1 = require("../helpers/getRedisChannel");
+const AbstractSubscriber_1 = require("./AbstractSubscriber");
+class RedisSubscriber extends AbstractSubscriber_1.Subscriber {
+    constructor(channel) {
+        super();
+        this.channel = channel;
+        this.activeChannels = new Set();
+        this.connection = new ioredis_1.default(config_1.config.REDIS_CONN, {
+            connectionName: `Subscriber for ${channel}`,
+            lazyConnect: true,
+            autoResubscribe: true,
+        });
+        this.connection.on("error", (err) => {
+            throw new errors_1.CustomError("Error while connecting to Redis", true, this.constructor.name, undefined, err);
+        });
+    }
+    subscribe(options = {}) {
+        var _a;
+        return __awaiter(this, void 0, void 0, function* () {
+            const maxMessages = (_a = options.maxMessages) !== null && _a !== void 0 ? _a : RedisSubscriber.DEFAULT_MAX_MESSAGES;
+            const redisChannel = (0, getRedisChannel_1.getRedisChannel)(this.channel, options.channelSuffix);
+            if (this.connection.status !== "ready") {
+                yield this.connection.connect();
+            }
+            if (this.activeChannels.has(redisChannel)) {
+                helpers_1.log.warn(`Already subscribed to Redis channel ${redisChannel}`);
+            }
+            else {
+                this.activeChannels.add(redisChannel);
+                yield this.connection.subscribe(redisChannel);
+            }
+            return this.listenForMessages(redisChannel, maxMessages);
+        });
+    }
+    unsubscribeAll() {
+        return __awaiter(this, void 0, void 0, function* () {
+            for (const channel of this.activeChannels) {
+                this.activeChannels.delete(channel);
+                yield this.connection.unsubscribe(channel);
+            }
+            this.connection.disconnect();
+        });
+    }
+    listenForMessages(redisChannel, maxMessages) {
+        return __asyncGenerator(this, arguments, function* listenForMessages_1() {
+            for (let i = 0; i < maxMessages; i++) {
+                yield yield __await(yield __await(new Promise((resolve) => {
+                    this.connection.once("message", (channel, message) => {
+                        if (channel === redisChannel) {
+                            resolve({ message, messageCount: i + 1 });
+                        }
+                    });
+                })));
+            }
+        });
+    }
+}
+exports.RedisSubscriber = RedisSubscriber;
+RedisSubscriber.DEFAULT_MAX_MESSAGES = 1;
+//# sourceMappingURL=RedisSubscriber.js.map

package/dist/integration-engine/data-access/pubsub/subscribers/RedisSubscriber.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"RedisSubscriber.js","sourceRoot":"","sources":["../../../../../src/integration-engine/data-access/pubsub/subscribers/RedisSubscriber.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,4CAAoC;AACpC,8CAAkC;AAClC,4CAA8C;AAC9C,sDAAwD;AACxD,gEAA6D;AAC7D,6DAAkD;AAQlD,MAAa,eAAgB,SAAQ,+BAAU;IAK3C,YAA6B,OAAe;QACxC,KAAK,EAAE,CAAC;QADiB,YAAO,GAAP,OAAO,CAAQ;QAFpC,mBAAc,GAAgB,IAAI,GAAG,EAAE,CAAC;QAI5C,IAAI,CAAC,UAAU,GAAG,IAAI,iBAAK,CAAC,eAAM,CAAC,UAAU,EAAE;YAC3C,cAAc,EAAE,kBAAkB,OAAO,EAAE;YAC3C,WAAW,EAAE,IAAI;YACjB,eAAe,EAAE,IAAI;SACxB,CAAC,CAAC;QAEH,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YAChC,MAAM,IAAI,oBAAW,CAAC,iCAAiC,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;QAC1G,CAAC,CAAC,CAAC;IACP,CAAC;IAEY,SAAS,CAAC,UAA6B,EAAE;;;YAClD,MAAM,WAAW,GAAG,MAAA,OAAO,CAAC,WAAW,mCAAI,eAAe,CAAC,oBAAoB,CAAC;YAChF,MAAM,YAAY,GAAG,IAAA,iCAAe,EAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;YAE1E,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,OAAO,EAAE;gBACpC,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;aACnC;YAED,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE;gBACvC,aAAG,CAAC,IAAI,CAAC,uCAAuC,YAAY,EAAE,CAAC,CAAC;aACnE;iBAAM;gBACH,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;gBACtC,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;aACjD;YAED,OAAO,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;;KAC5D;IAEY,cAAc;;YACvB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,cAAc,EAAE;gBACvC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBACpC,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;aAC9C;YAED,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;QACjC,CAAC;KAAA;IAEc,iBAAiB,CAAC,YAAoB,EAAE,WAAmB;;YACtE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE;gBAClC,oBAAM,cAAM,IAAI,OAAO,CAAoB,CAAC,OAAO,EAAE,EAAE;oBACnD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE;wBACjD,IAAI,OAAO,KAAK,YAAY,EAAE;4BAC1B,OAAO,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;yBAC7C;oBACL,CAAC,CAAC,CAAC;gBACP,CAAC,CAAC,CAAA,CAAA,CAAC;aACN;QACL,CAAC;KAAA;;AAvDL,0CAwDC;AAvDkB,oCAAoB,GAAG,CAAC,CAAC"}

package/dist/integration-engine/data-access/pubsub/subscribers/interfaces/ISubscribeMessage.d.ts ADDED Viewed

@@ -0,0 +1,4 @@
+export interface ISubscribeMessage {
+    message: string;
+    messageCount: number;
+}

package/dist/integration-engine/data-access/pubsub/subscribers/interfaces/ISubscribeMessage.js ADDED Viewed

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=ISubscribeMessage.js.map

package/dist/integration-engine/data-access/pubsub/subscribers/interfaces/ISubscribeMessage.js.map ADDED Viewed

	@@ -0,0 +1 @@
1	+ {"version":3,"file":"ISubscribeMessage.js","sourceRoot":"","sources":["../../../../../../src/integration-engine/data-access/pubsub/subscribers/interfaces/ISubscribeMessage.ts"],"names":[],"mappings":""}

package/dist/integration-engine/datasources/protocol-strategy/FTPProtocolStrategy.d.ts CHANGED Viewed

@@ -1,15 +1,16 @@
-import { ProtocolStrategy } from "./ProtocolStrategy";
 import { IFile, IFTPSettings } from "./IProtocolStrategy";
+import { ProtocolStrategy } from "./ProtocolStrategy";
+interface IRawDataOutput {
+    data: IFile | IFile[];
+}
 export declare class FTPProtocolStrategy extends ProtocolStrategy {
     protected connectionSettings: IFTPSettings;
     private ftpClient;
+    private fileProcessor;
+    private downloadPath;
     constructor(settings: IFTPSettings);
     setConnectionSettings: (settings: IFTPSettings) => void;
-    getRawData: () => Promise<{
-        data: IFile | IFile[];
-    }>;
+    getRawData: () => Promise<IRawDataOutput>;
     getLastModified: () => Promise<string | null>;
-    private encode;
-    private readFile;
-    private readDir;
 }
+export {};

package/dist/integration-engine/datasources/protocol-strategy/FTPProtocolStrategy.js CHANGED Viewed

@@ -13,84 +13,70 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.FTPProtocolStrategy = void 0;
+const IProtocolStrategy_1 = require("./IProtocolStrategy");
+const ProtocolStrategy_1 = require("./ProtocolStrategy");
+const helpers_1 = require("../../helpers");
 const errors_1 = require("@golemio/errors");
+const basic_ftp_1 = require("basic-ftp");
+const fs_1 = __importDefault(require("fs"));
 const moment_1 = __importDefault(require("moment"));
+const os_1 = __importDefault(require("os"));
 const path_1 = __importDefault(require("path"));
-const fs_1 = __importDefault(require("fs"));
-const basic_ftp_1 = require("basic-ftp");
-const decompress_1 = __importDefault(require("decompress"));
-const helpers_1 = require("../../helpers");
-const ProtocolStrategy_1 = require("./ProtocolStrategy");
-const IProtocolStrategy_1 = require("./IProtocolStrategy");
+const FTPFileProcessor_1 = require("./helpers/FTPFileProcessor");
 const defaultSettings = {
-    tmpDir: "/tmp/",
+    tmpDir: os_1.default.tmpdir(),
     targetType: IProtocolStrategy_1.FTPTargetType.FILE,
+    returnType: IProtocolStrategy_1.FTPReturnType.File,
 };
 class FTPProtocolStrategy extends ProtocolStrategy_1.ProtocolStrategy {
     constructor(settings) {
         super(Object.assign(Object.assign({}, defaultSettings), settings));
         this.setConnectionSettings = (settings) => {
             this.connectionSettings = Object.assign(Object.assign({}, defaultSettings), settings);
+            this.downloadPath = this.connectionSettings.tmpDir;
+            this.fileProcessor = new FTPFileProcessor_1.FTPFileProcessor(this.connectionSettings, this.downloadPath);
         };
         this.getRawData = () => __awaiter(this, void 0, void 0, function* () {
-            const tmpDir = this.connectionSettings.tmpDir;
+            try {
+                const dirHandle = yield fs_1.default.promises.open(this.downloadPath, fs_1.default.constants.O_DIRECTORY);
+                yield dirHandle.close();
+            }
+            catch (err) {
+                throw new errors_1.CustomError("FTP download path is not a directory", true, this.constructor.name, undefined, err);
+            }
             try {
                 yield this.ftpClient.access(this.connectionSettings.url);
                 yield this.ftpClient.cd(this.connectionSettings.path);
-                yield this.ftpClient.downloadTo(fs_1.default.createWriteStream(path_1.default.join(tmpDir, this.connectionSettings.filename)), this.connectionSettings.filename);
-                yield this.ftpClient.close();
-                const prefix = path_1.default.parse(this.connectionSettings.filename).name + "/";
+                yield this.ftpClient.downloadTo(fs_1.default.createWriteStream(path_1.default.join(this.downloadPath, this.connectionSettings.filename)), this.connectionSettings.filename);
+                this.ftpClient.close();
+            }
+            catch (err) {
+                throw new errors_1.CustomError("Error while downloading FTP files", true, this.constructor.name, undefined, err);
+            }
+            const prefix = path_1.default.parse(this.connectionSettings.filename).name + "/";
+            let result;
+            try {
                 if (this.connectionSettings.targetType === IProtocolStrategy_1.FTPTargetType.COMPRESSED) {
-                    const files = yield (0, decompress_1.default)(path_1.default.join(tmpDir, this.connectionSettings.filename), {
-                        filter: this.connectionSettings.whitelistedFiles && this.connectionSettings.whitelistedFiles.length
-                            ? (file) => { var _a; return ((_a = this.connectionSettings.whitelistedFiles) === null || _a === void 0 ? void 0 : _a.indexOf(file.path)) !== -1; }
-                            : (file) => file,
-                    });
-                    const result = files.map((file) => {
-                        return {
-                            data: this.encode(file.data),
-                            filepath: prefix + file.path,
-                            name: path_1.default.parse(file.path).name,
-                            mtime: file.mtime,
-                            type: file.type,
-                        };
-                    });
-                    return { data: result };
+                    result = yield this.fileProcessor.processTargetCompressed(prefix);
+                }
+                else if (this.connectionSettings.targetType === IProtocolStrategy_1.FTPTargetType.FOLDER) {
+                    result = yield this.fileProcessor.processTargetFolder(prefix);
                 }
-                if (this.connectionSettings.targetType === IProtocolStrategy_1.FTPTargetType.FOLDER) {
-                    let files = yield this.readDir(path_1.default.join(tmpDir, this.connectionSettings.filename));
-                    if (this.connectionSettings.whitelistedFiles && this.connectionSettings.whitelistedFiles.length) {
-                        files = files.filter((file) => { var _a; return ((_a = this.connectionSettings.whitelistedFiles) === null || _a === void 0 ? void 0 : _a.indexOf(file)) !== -1; });
-                    }
-                    const result = yield Promise.all(files.map((file) => __awaiter(this, void 0, void 0, function* () {
-                        const buffer = yield this.readFile(path_1.default.join(tmpDir, prefix, file));
-                        return {
-                            data: this.encode(buffer),
-                            filepath: prefix + file,
-                            name: path_1.default.parse(file).name,
-                        };
-                    })));
-                    return { data: result };
+                else {
+                    result = yield this.fileProcessor.processTargetFile();
                 }
-                const buffer = yield this.readFile(path_1.default.join(tmpDir, this.connectionSettings.filename));
-                return {
-                    data: {
-                        data: this.encode(buffer),
-                        filepath: this.connectionSettings.filename,
-                        name: path_1.default.parse(this.connectionSettings.filename).name,
-                    },
-                };
             }
             catch (err) {
-                throw new errors_1.CustomError("Error while getting data from server.", true, this.constructor.name, 2002, err);
+                throw new errors_1.CustomError("Error while processing downloaded FTP files", true, this.constructor.name, undefined, err);
             }
+            return { data: result };
         });
         this.getLastModified = () => __awaiter(this, void 0, void 0, function* () {
             try {
                 yield this.ftpClient.access(this.connectionSettings.url);
                 yield this.ftpClient.cd(this.connectionSettings.path);
                 const lastModified = yield this.ftpClient.lastMod(this.connectionSettings.filename);
-                yield this.ftpClient.close();
+                this.ftpClient.close();
                 return lastModified ? (0, moment_1.default)(lastModified, "MM-DD-YYYY hh:mmA").toISOString() : null;
             }
             catch (err) {
@@ -98,37 +84,11 @@ class FTPProtocolStrategy extends ProtocolStrategy_1.ProtocolStrategy {
                 return null;
             }
         });
-        this.encode = (data) => {
-            return this.connectionSettings.encoding ? data.toString(this.connectionSettings.encoding) : data;
-        };
-        this.readFile = (file) => {
-            return new Promise((resolve, reject) => {
-                const stream = fs_1.default.createReadStream(file);
-                const chunks = [];
-                stream.on("error", (err) => {
-                    reject(err);
-                });
-                stream.on("data", (data) => {
-                    chunks.push(data);
-                });
-                stream.on("close", () => {
-                    resolve(Buffer.concat(chunks));
-                });
-            });
-        };
-        this.readDir = (dirPath) => {
-            return new Promise((resolve, reject) => {
-                fs_1.default.readdir(dirPath, (err, files) => {
-                    if (err) {
-                        return reject(err);
-                    }
-                    return resolve(files);
-                });
-            });
-        };
         this.ftpClient = new basic_ftp_1.Client();
         this.ftpClient.ftp.log = helpers_1.log.silly;
         this.ftpClient.ftp.verbose = true;
+        this.downloadPath = this.connectionSettings.tmpDir;
+        this.fileProcessor = new FTPFileProcessor_1.FTPFileProcessor(this.connectionSettings, this.downloadPath);
     }
 }
 exports.FTPProtocolStrategy = FTPProtocolStrategy;

package/dist/integration-engine/datasources/protocol-strategy/FTPProtocolStrategy.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"FTPProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/FTPProtocolStrategy.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,~~4CAA8C~~;~~AAC9C~~,~~oDAA4B~~;~~AAC5B~~,~~gDAAwB~~;~~AACxB~~,~~4CAAoB~~;~~AACpB~~,yCAAmC;AACnC,~~4DAAoC~~;~~AACpC~~,~~2CAAkC~~;~~AAClC~~,~~yDAAsF~~;~~AACtF~~,~~2DAAyG~~;~~AAEzG~~,MAAM,eAAe,~~GAAgD~~;~~IACjE~~,MAAM,EAAE,~~OAAO~~;~~IACf~~,UAAU,EAAE,iCAAa,CAAC,IAAI;CACjC,CAAC;AAEF,MAAa,mBAAoB,SAAQ,mCAAgB;~~IAIrD~~,YAAY,QAAsB;QAC9B,KAAK,iCAAM,eAAe,GAAK,QAAQ,EAAG,CAAC;~~QAMxC~~,0BAAqB,GAAG,CAAC,QAAsB,EAAQ,EAAE;YAC5D,IAAI,CAAC,kBAAkB,mCAAQ,eAAe,GAAK,QAAQ,CAAE,CAAC;~~QAClE~~,~~CAAC~~,CAAC~~;QAEK~~,~~eAAU~~,GAAG,~~GAA6C,EAAE;YAC/D,MAAM,MAAM,GAAG,~~IAAI,CAAC,kBAAkB,CAAC,MAAO,CAAC;~~YAE/C~~,IAAI~~;gBACA~~,~~MAAM~~,~~IAAI~~,~~CAAC~~,~~SAAS~~,~~CAAC~~,~~MAAM,~~CAAC,IAAI,CAAC,kBAAkB,CAAC,~~GAAG~~,CAAC,CAAC;~~gBACzD~~,~~MAAM,IAAI,~~CAAC,~~SAAS,~~CAAC,~~EAAE~~,~~CAAC~~,~~IAAI~~,~~CAAC~~,~~kBAAkB,CAAC,~~IAAI~~,CAAC,CAAC~~;~~gBACtD~~,MAAM,~~IAAI,CAAC,~~SAAS,~~CAAC~~,~~UAAU~~,~~CAC3B,~~YAAE,CAAC,~~iBAAiB~~,CAAC,~~cAAI~~,CAAC,IAAI,CAAC,~~MAAM~~,EAAE,~~IAAI~~,CAAC,~~kBAAkB~~,CAAC,~~QAAQ~~,CAAC,CAAC~~,EACzE,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CACnC,CAAC~~;~~gBACF~~,MAAM,~~IAAI,CAAC,~~SAAS,CAAC,KAAK,EAAE,CAAC;~~gBAE7B~~,~~MAAM~~,~~MAAM,~~GAAG,~~cAAI,CAAC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC;gBAEvE,IAAI,IAAI,CAAC,kBAAkB,CAAC,UAAU,KAAK,iCAAa,CAAC,UAAU,~~EAAE;~~oBACjE~~,MAAM,~~KAAK~~,~~GAAG~~,~~MAAM,IAAA,oBAAU,EAAC,cAAI,~~CAAC,~~IAAI~~,~~CAAC,MAAM,~~EAAE,IAAI,~~CAAC,kBAAkB,CAAC,QAAQ,CAAC,~~EAAE~~;wBAChF~~,~~MAAM,EACF,~~IAAI,CAAC,~~kBAAkB~~,CAAC,~~gBAAgB,~~IAAI,~~IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,MAAM;4BACvF,CAAC,CAAC,CAAC,IAAS,~~EAAE,~~EAAE~~,~~WAAC,OAAA,CAAA,MAAA,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,0CAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAK,CAAC,CAAC,CAAA,EAAA;4BACpF,CAAC,CAAC,CAAC,IAAS,~~EAAE,~~EAAE,CAAC,IAAI;qBAChC,CAAC,CAAC;oBAEH,MAAM,MAAM,GAAY,KAAK,CAAC,~~GAAG,CAAC,CAAC~~,IAAqB,EAAE,EAAE~~;~~wBACxD,OAAO~~;~~4BACH~~,IAAI,~~EAAE,IAAI,CAAC,~~MAAM,~~CAAC,~~IAAI,CAAC,~~IAAI~~,CAAC~~;4BAC5B~~,~~QAAQ,EAAE,~~MAAM,~~GAAG,IAAI,~~CAAC,IAAI~~;4BAC5B~~,~~IAAI,EAAE,cAAI,~~CAAC,~~KAAK~~,CAAC,~~IAAI~~,CAAC,~~IAAI,~~CAAC~~,CAAC,IAAI~~;~~4BAChC~~,~~KAAK~~,~~EAAE,~~IAAI,CAAC,~~KAAK;4BACjB~~,~~IAAI~~,EAAE,~~IAAI,~~CAAC,IAAI~~;yBAClB~~,CAAC~~;oBACN~~,~~CAAC~~,CAAC,~~CAAC;oBACH,OAAO,EAAE,~~IAAI,~~EAAE~~,~~MAAM,EAAE,~~CAAC;~~iBAC3B;gBAED~~,~~IAAI~~,IAAI,CAAC,~~kBAAkB~~,CAAC,UAAU,~~KAAK~~,~~iCAAa~~,CAAC,~~MAAM~~,~~EAAE;oBAC7D,IAAI,KAAK,GAAG,MAAM,IAAI,~~CAAC,~~OAAO~~,CAAC,~~cAAI~~,CAAC,IAAI,CAAC,~~MAAM~~,EAAE,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,~~CAAC;oBACpF~~,IAAI,~~IAAI,~~CAAC,kBAAkB,CAAC,~~gBAAgB~~,~~IAAI~~,~~IAAI,~~CAAC,~~kBAAkB~~,CAAC,~~gBAAgB~~,CAAC,~~MAAM~~,EAAE;~~wBAC7F~~,~~KAAK~~,GAAG,~~KAAK~~,~~CAAC,~~MAAM,~~CAAC~~,CAAC,~~IAAS~~,EAAE,EAAE,~~WAAC,OAAA,CAAA,MAAA,~~IAAI,CAAC,~~kBAAkB~~,CAAC,~~gBAAgB,0CAAE,OAAO,CAAC,~~IAAI,~~CAAC~~,~~MAAK~~,~~CAAC~~,~~CAAC~~,~~CAAA,EAAA,~~CAAC,CAAC;~~qBACvG~~;~~oBAED~~,MAAM,MAAM,~~GAAY~~,~~MAAM~~,~~OAAO,~~CAAC,~~GAAG,CACrC,~~KAAK,CAAC,~~GAAG~~,CAAC,~~CAAO~~,~~IAAY,EAAE,EAAE;wBAC7B,MAAM,MAAM,GAAG,MAAM,IAAI,~~CAAC,QAAQ,CAAC,~~cAAI,~~CAAC,IAAI,~~CAAC~~,~~MAAM~~,~~EAAE~~,~~MAAM,EAAE,~~IAAI,~~CAAC~~,CAAC~~,CAAC~~;~~wBACpE~~,~~OAAO~~;~~4BACH~~,IAAI,~~EAAE,~~IAAI,CAAC,~~MAAM~~,CAAC,~~MAAM~~,CAAC~~;4BACzB~~,~~QAAQ~~,EAAE,MAAM,GAAG,~~IAAI;4BACvB~~,IAAI,~~EAAE~~,~~cAAI~~,CAAC,~~KAAK~~,CAAC,~~IAAI~~,CAAC,CAAC~~,IAAI~~;~~yBAC9B,CAAC~~;~~oBACN~~,~~CAAC~~,~~CAAA~~,CAAC,~~CACL~~,CAAC~~;oBACF~~,~~OAAO~~,~~EAAE~~,~~IAAI~~,~~EAAE~~,MAAM,EAAE~~,CAAC~~;~~iBAC3B;gBAED~~,MAAM,~~MAAM,~~GAAG,MAAM,IAAI,CAAC,~~QAAQ~~,CAAC,~~cAAI~~,CAAC,~~IAAI,CAAC,~~MAAM,~~EAAE,IAAI,~~CAAC,~~kBAAkB,~~CAAC~~,QAAQ,CAAC,CAAC,CAAC~~;~~gBACxF,OAAO~~;oBACH,~~IAAI,EAAE;wBACF,IAAI,EAAE,IAAI,CAAC,~~MAAM,~~CAAC~~,MAAM,~~CAAC;wBACzB,QAAQ,EAAE,~~IAAI,CAAC,~~kBAAkB~~,CAAC,~~QAAQ;wBAC1C~~,~~IAAI,~~EAAE,~~cAAI,~~CAAC~~,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,IAAI~~;~~qBAC1D~~;~~iBACJ,CAAC~~;~~aACL;~~YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,oBAAW,CAAC,~~uCAAuC~~,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,~~IAAI~~,EAAE,GAAG,CAAC,CAAC;~~aAC1G~~;~~QACL~~,CAAC,CAAA,CAAC;QAEK,oBAAe,GAAG,GAAiC,EAAE;YACxD,IAAI;gBACA,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;gBACzD,MAAM,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;gBACtD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;gBACpF,~~MAAM,~~IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;~~gBAC7B~~,OAAO,YAAY,CAAC,CAAC,CAAC,IAAA,gBAAM,EAAC,YAAY,EAAE,mBAAmB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aACxF;YAAC,OAAO,GAAG,EAAE;gBACV,aAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACf,OAAO,IAAI,CAAC;aACf;QACL,CAAC,CAAA,CAAC;~~QAEM,WAAM,GAAG,CAAC,IAAY,EAAmB,EAAE;YAC~~/C,~~OAAO,~~IAAI,CAAC,~~kBAAkB~~,~~CAAC~~,~~QAAQ,CAAC,CAAC,CAAC,~~IAAI,~~CAAC~~,~~QAAQ~~,CAAC,IAAI,CAAC,~~kBAAkB~~,CAAC,~~QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACrG,CAAC,CAAC;QAEM,aAAQ,~~GAAG,CAAC,~~IAAY,EAAmB,EAAE;YACjD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACnC,MAAM,MAAM,~~GAAG,~~YAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;gBACzC,MAAM,MAAM,GAAU,EAAE,CAAC;gBAEzB,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;oBAC9B,MAAM,CAAC,~~GAAG,~~CAAC~~,CAAC~~;gBAChB~~,~~CAAC~~,CAAC~~,CAAC~~;~~gBACH~~,~~MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAS,EAAE,EAAE;oBAC5B,MAAM,CAAC,~~IAAI,CAAC,~~IAAI~~,CAAC,~~CAAC;gBACtB~~,CAAC,~~CAAC,CAAC;gBACH,MAAM,CAAC,EAAE,CAAC,~~OAAO,~~EAAE,~~GAAG,~~EAAE;oBACpB~~,~~OAAO,~~CAAC~~,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC~~;~~gBACnC~~,~~CAAC~~,CAAC,~~CAAC;YACP~~,~~CAAC,CAAC,CAAC;QACP,CAAC,CAAC;QAEM,YAAO,~~GAAG,~~CAAC,OAAe,EAAqB,EAAE;YACrD,OAAO,~~IAAI,~~OAAO,~~CAAC,~~CAAC~~,~~OAAO,EAAE,MAAM,EAAE,EAAE;gBACnC,YAAE,~~CAAC,~~OAAO~~,CAAC~~,OAAO,EAAE,CAAC,GAAiB,EAAE,KAAU,EAAE,EAAE~~;~~oBAClD~~,IAAI,~~GAAG,EAAE;wBACL,OAAO,MAAM,~~CAAC,~~GAAG~~,~~CAAC,CAAC;qBACtB;oBACD,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC;gBAC1B,CAAC,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;QACP,CAAC,CAAC;QAtHE,IAAI,CAAC,SAAS,~~GAAG,IAAI,~~kBAAM~~,~~EAAE,~~CAAC~~;QAC9B~~,IAAI,CAAC,~~SAAS~~,~~CAAC~~,~~GAAG,CAAC,GAAG,GAAG,aAAG,CAAC,KAAK,CAAC;QACnC,~~IAAI,CAAC,~~SAAS~~,CAAC,~~GAAG,~~CAAC~~,OAAO,GAAG,IAAI,CAAC~~;~~IACtC~~,CAAC;~~CAoHJ~~;~~AA7HD~~,~~kDA6HC~~"}
1	+ {"version":3,"file":"FTPProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/FTPProtocolStrategy.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2DAAwH;AACxH,yDAAsF;AACtF,2CAAkC;AAClC,4CAA8C;AAC9C,yCAAmC;AACnC,4CAAoB;AACpB,oDAA4B;AAC5B,4CAAoB;AACpB,gDAAwB;AACxB,iEAA8D;AAM9D,MAAM,eAAe,GAA0B;IAC3C,MAAM,EAAE,YAAE,CAAC,MAAM,EAAE;IACnB,UAAU,EAAE,iCAAa,CAAC,IAAI;IAC9B,UAAU,EAAE,iCAAa,CAAC,IAAI;CACjC,CAAC;AAEF,MAAa,mBAAoB,SAAQ,mCAAgB;IAMrD,YAAY,QAAsB;QAC9B,KAAK,iCAAM,eAAe,GAAK,QAAQ,EAAG,CAAC;QAQxC,0BAAqB,GAAG,CAAC,QAAsB,EAAQ,EAAE;YAC5D,IAAI,CAAC,kBAAkB,mCAAQ,eAAe,GAAK,QAAQ,CAAE,CAAC;YAC9D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAO,CAAC;YACpD,IAAI,CAAC,aAAa,GAAG,IAAI,mCAAgB,CAAC,IAAI,CAAC,kBAAkB,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAC1F,CAAC,CAAC;QAEK,eAAU,GAAG,GAAkC,EAAE;YACpD,IAAI;gBACA,MAAM,SAAS,GAAG,MAAM,YAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,YAAE,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;gBACtF,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;aAC3B;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,oBAAW,CAAC,sCAAsC,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;aAC9G;YAED,IAAI;gBACA,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;gBACzD,MAAM,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;gBACtD,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAC3B,YAAE,CAAC,iBAAiB,CAAC,cAAI,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,EACpF,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CACnC,CAAC;gBAEF,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;aAC1B;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,oBAAW,CAAC,mCAAmC,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;aAC3G;YAED,MAAM,MAAM,GAAG,cAAI,CAAC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC;YACvE,IAAI,MAAuB,CAAC;YAE5B,IAAI;gBACA,IAAI,IAAI,CAAC,kBAAkB,CAAC,UAAU,KAAK,iCAAa,CAAC,UAAU,EAAE;oBACjE,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC;iBACrE;qBAAM,IAAI,IAAI,CAAC,kBAAkB,CAAC,UAAU,KAAK,iCAAa,CAAC,MAAM,EAAE;oBACpE,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;iBACjE;qBAAM;oBACH,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,iBAAiB,EAAE,CAAC;iBACzD;aACJ;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,oBAAW,CAAC,6CAA6C,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;aACrH;YAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAC5B,CAAC,CAAA,CAAC;QAEK,oBAAe,GAAG,GAAiC,EAAE;YACxD,IAAI;gBACA,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;gBACzD,MAAM,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;gBACtD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;gBACpF,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;gBACvB,OAAO,YAAY,CAAC,CAAC,CAAC,IAAA,gBAAM,EAAC,YAAY,EAAE,mBAAmB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aACxF;YAAC,OAAO,GAAG,EAAE;gBACV,aAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACf,OAAO,IAAI,CAAC;aACf;QACL,CAAC,CAAA,CAAC;QA/DE,IAAI,CAAC,SAAS,GAAG,IAAI,kBAAM,EAAE,CAAC;QAC9B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,GAAG,aAAG,CAAC,KAAK,CAAC;QACnC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC;QAClC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAO,CAAC;QACpD,IAAI,CAAC,aAAa,GAAG,IAAI,mCAAgB,CAAC,IAAI,CAAC,kBAAkB,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAC1F,CAAC;CA2DJ;AAxED,kDAwEC"}

package/dist/integration-engine/datasources/protocol-strategy/HTTPProtocolStrategy.d.ts CHANGED Viewed

@@ -2,6 +2,7 @@ import { IHTTPSettings } from "./IProtocolStrategy";
 import { ProtocolStrategy } from "./ProtocolStrategy";
 export declare class HTTPProtocolStrategy extends ProtocolStrategy {
     protected connectionSettings: IHTTPSettings;
+    private fileCompressor;
     constructor(settings: IHTTPSettings);
     setConnectionSettings: (settings: IHTTPSettings) => void;
     getRawData: () => Promise<any>;

package/dist/integration-engine/datasources/protocol-strategy/HTTPProtocolStrategy.js CHANGED Viewed

@@ -13,22 +13,28 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.HTTPProtocolStrategy = void 0;
+const ProtocolStrategy_1 = require("./ProtocolStrategy");
+const helpers_1 = require("../../helpers");
+const FileCompressor_1 = require("../../helpers/FileCompressor");
+const models_1 = require("../../models");
 const errors_1 = require("@golemio/errors");
+const axios_1 = __importDefault(require("axios"));
+const promises_1 = __importDefault(require("fs/promises"));
 const moment_1 = __importDefault(require("moment"));
+const os_1 = __importDefault(require("os"));
 const path_1 = __importDefault(require("path"));
-const decompress_1 = __importDefault(require("decompress"));
-const zlib_1 = require("zlib");
 const util_1 = require("util");
-const helpers_1 = require("../../helpers");
-const models_1 = require("../../models");
-const ProtocolStrategy_1 = require("./ProtocolStrategy");
-const axios_1 = __importDefault(require("axios"));
+const zlib_1 = require("zlib");
 const gunzip = (0, util_1.promisify)(zlib_1.gunzip);
+const defaultSettings = {
+    tmpDir: os_1.default.tmpdir(),
+};
 class HTTPProtocolStrategy extends ProtocolStrategy_1.ProtocolStrategy {
     constructor(settings) {
-        super(settings);
+        super(Object.assign(Object.assign({}, defaultSettings), settings));
         this.setConnectionSettings = (settings) => {
-            this.connectionSettings = settings;
+            this.connectionSettings = Object.assign(Object.assign({}, defaultSettings), settings);
+            this.fileCompressor = new FileCompressor_1.FileCompressor(this.connectionSettings.tmpDir);
         };
         this.getRawData = () => __awaiter(this, void 0, void 0, function* () {
             var _a;
@@ -50,10 +56,11 @@ class HTTPProtocolStrategy extends ProtocolStrategy_1.ProtocolStrategy {
                 }
                 if (this.connectionSettings.isCompressed) {
                     const prefix = path_1.default.parse(this.connectionSettings.url).name + "/";
-                    const files = yield (0, decompress_1.default)(body, {
-                        filter: this.connectionSettings.whitelistedFiles && this.connectionSettings.whitelistedFiles.length
-                            ? (file) => { var _a; return ((_a = this.connectionSettings.whitelistedFiles) === null || _a === void 0 ? void 0 : _a.indexOf(file.path)) !== -1; }
-                            : (file) => file,
+                    const { extractionPath, files } = yield this.fileCompressor.inflateBuffer(body, (filename) => {
+                        if (this.connectionSettings.whitelistedFiles) {
+                            return this.connectionSettings.whitelistedFiles.indexOf(filename) >= 0;
+                        }
+                        return !!filename;
                     });
                     const redisModel = new models_1.RedisModel("HTTPProtocolStrategy" + "Model", {
                         isKeyConstructedFromData: false,
@@ -61,14 +68,17 @@ class HTTPProtocolStrategy extends ProtocolStrategy_1.ProtocolStrategy {
                     }, null);
                     const encoding = (_a = this.connectionSettings.encoding) !== null && _a !== void 0 ? _a : "utf8";
                     body = yield Promise.all(files.map((file) => __awaiter(this, void 0, void 0, function* () {
-                        yield redisModel.hset(prefix + file.path, file.data.toString(encoding));
+                        const data = yield promises_1.default.readFile(file.path, encoding);
+                        const relativePath = path_1.default.join(prefix, file.name);
+                        yield redisModel.hset(relativePath, data.toString(encoding));
                         return {
-                            filepath: prefix + file.path,
+                            filepath: relativePath,
                             mtime: file.mtime,
-                            name: path_1.default.parse(file.path).name,
+                            name: path_1.default.parse(relativePath).name,
                             path: file.path,
                         };
                     })));
+                    yield promises_1.default.rm(extractionPath, { recursive: true, force: true });
                 }
                 return {
                     data: body,
@@ -98,6 +108,7 @@ class HTTPProtocolStrategy extends ProtocolStrategy_1.ProtocolStrategy {
                 return null;
             }
         });
+        this.fileCompressor = new FileCompressor_1.FileCompressor(this.connectionSettings.tmpDir);
     }
 }
 exports.HTTPProtocolStrategy = HTTPProtocolStrategy;

package/dist/integration-engine/datasources/protocol-strategy/HTTPProtocolStrategy.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"HTTPProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/HTTPProtocolStrategy.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;~~AAAA~~,~~4CAA8C~~;~~AAC9C~~,~~oDAA4B~~;~~AAC5B~~,~~gDAAwB~~;~~AACxB~~,~~4DAAoC~~;~~AACpC,+BAAoC~~;~~AACpC,+BAAiC~~;~~AACjC~~,~~2CAAkC~~;~~AAClC~~,~~yCAAwC~~;~~AAExC~~,~~yDAAsF~~;~~AACtF~~,~~kDAAkD~~;~~AAClD~~,MAAM,MAAM,GAAG,IAAA,gBAAS,EAAC,aAAE,CAAC,CAAC;AAE7B,MAAa,oBAAqB,SAAQ,mCAAgB;~~IAGtD~~,YAAY,QAAuB;QAC/B,KAAK,~~CAAC~~,QAAQ,~~CAAC~~,CAAC;~~QAGb~~,0BAAqB,GAAG,CAAC,QAAuB,EAAQ,EAAE;YAC7D,IAAI,CAAC,kBAAkB,~~GAAG~~,QAAQ,CAAC;~~QACvC~~,CAAC,CAAC;QAEK,eAAU,GAAG,GAAuB,EAAE;;YACzC,IAAI;gBACA,IAAI,CAAC,kBAAkB,CAAC,uBAAuB,GAAG,IAAI,CAAC;gBAEvD,IAAI,0BAA0B,GAAG,IAAI,CAAC,kBAAwC,CAAC;gBAC/E,IAAI,IAAI,CAAC,kBAAkB,CAAC,YAAY;oBAAE,0BAA0B,CAAC,YAAY,GAAG,aAAa,CAAC;gBAClG,IAAI,IAAI,CAAC,kBAAkB,CAAC,WAAW;oBAAE,0BAA0B,CAAC,YAAY,GAAG,aAAa,CAAC;gBACjG,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI;oBAAE,0BAA0B,CAAC,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;gBAEjG,MAAM,MAAM,GAAG,MAAM,IAAA,eAAK,EAAC,0BAA0B,CAAC,CAAC;gBAEvD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;gBAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;gBACjC,IAAI,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;gBAEvB,IAAI,IAAI,CAAC,kBAAkB,CAAC,WAAW,EAAE;oBACrC,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;iBAC7B;gBAED,IAAI,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE;oBACtC,MAAM,MAAM,GAAG,cAAI,CAAC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC;oBAClE,MAAM,~~KAAK~~,~~GAAG~~,~~MAAM~~,~~IAAA~~,~~oBAAU~~,~~EAAC~~,~~IAAI,EAAE;wBACjC,~~MAAM,~~EACF,~~IAAI,CAAC,~~kBAAkB~~,CAAC,~~gBAAgB~~,~~IAAI~~,IAAI,~~CAAC~~,~~kBAAkB,~~CAAC,~~gBAAgB~~,~~CAAC~~,~~MAAM~~;~~4BACvF~~,~~CAAC~~,CAAC,CAAC,~~IAAS~~,EAAE,~~EAAE~~,~~WAAC,OAAA,CAAA,MAAA,~~IAAI,CAAC,kBAAkB,CAAC,gBAAgB,~~0CAAE~~,OAAO,CAAC,~~IAAI~~,CAAC,IAAI,CAAC,~~MAAK,~~CAAC~~,CAAC,CAAA,EAAA~~;~~4BACpF~~,~~CAAC~~,CAAC,CAAC,~~IAAS~~,~~EAAE,EAAE,~~CAAC~~,IAAI~~;~~qBAChC~~,CAAC,CAAC;~~oBACH~~,MAAM,UAAU,GAAG,IAAI,mBAAU,CAC7B,sBAAsB,GAAG,OAAO,EAChC;wBACI,wBAAwB,EAAE,KAAK;wBAC/B,MAAM,EAAE,OAAO;qBAClB,EACD,IAAI,CACP,CAAC;oBAEF,MAAM,QAAQ,GAAG,MAAA,IAAI,CAAC,kBAAkB,CAAC,QAAQ,mCAAI,MAAM,CAAC;oBAE5D,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CACpB,KAAK,CAAC,GAAG,CAAC,CAAO,IAAI,EAAE,EAAE;wBACrB,MAAM,~~UAAU~~,CAAC,IAAI,CAAC,MAAM,GAAG,~~IAAI~~,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;~~wBACxE~~,OAAO;4BACH,QAAQ,EAAE,~~MAAM,GAAG,IAAI,CAAC,IAAI~~;~~4BAC5B~~,KAAK,EAAE,IAAI,CAAC,KAAK;4BACjB,IAAI,EAAE,cAAI,CAAC,KAAK,CAAC,~~IAAI~~,CAAC,~~IAAI,~~CAAC,~~CAAC,~~IAAI;~~4BAChC~~,IAAI,EAAE,IAAI,CAAC,IAAI;yBAClB,CAAC;oBACN,CAAC,CAAA,CAAC,CACL,CAAC;~~iBACL~~;gBACD,OAAO;oBACH,IAAI,EAAE,IAAI;oBACV,IAAI,EAAE;wBACF,OAAO;wBACP,UAAU;qBACb;iBACJ,CAAC;aACL;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,oBAAW,CAAC,uCAAuC,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;aAC1G;QACL,CAAC,CAAA,CAAC;QAEK,oBAAe,GAAG,GAAiC,EAAE;YACxD,IAAI;gBACA,MAAM,CAAC,GAAG,IAAI,CAAC,kBAAwC,CAAC;gBACxD,IAAI,IAAI,CAAC,kBAAkB,CAAC,YAAY;oBAAE,CAAC,CAAC,YAAY,GAAG,aAAa,CAAC;gBACzE,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI;oBAAE,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;gBACxE,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC;gBAClB,MAAM,GAAG,GAAG,MAAM,IAAA,eAAK,EAAC,CAAC,CAAC,CAAC;gBAC3B,OAAO,GAAG,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aACnG;YAAC,OAAO,GAAG,EAAE;gBACV,aAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACf,OAAO,IAAI,CAAC;aACf;QACL,CAAC,CAAA,CAAC;~~IAhFF~~,CAAC;~~CAiFJ~~;~~AAtFD~~,~~oDAsFC~~"}
1	+ {"version":3,"file":"HTTPProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/HTTPProtocolStrategy.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AACA,yDAAsF;AACtF,2CAAkC;AAClC,iEAA4D;AAC5D,yCAAwC;AACxC,4CAA8C;AAC9C,kDAAkD;AAClD,2DAA6B;AAC7B,oDAA4B;AAC5B,4CAAoB;AACpB,gDAAwB;AACxB,+BAAiC;AACjC,+BAAoC;AAEpC,MAAM,MAAM,GAAG,IAAA,gBAAS,EAAC,aAAE,CAAC,CAAC;AAE7B,MAAM,eAAe,GAA2B;IAC5C,MAAM,EAAE,YAAE,CAAC,MAAM,EAAE;CACtB,CAAC;AAEF,MAAa,oBAAqB,SAAQ,mCAAgB;IAItD,YAAY,QAAuB;QAC/B,KAAK,iCAAM,eAAe,GAAK,QAAQ,EAAG,CAAC;QAIxC,0BAAqB,GAAG,CAAC,QAAuB,EAAQ,EAAE;YAC7D,IAAI,CAAC,kBAAkB,mCAAQ,eAAe,GAAK,QAAQ,CAAE,CAAC;YAC9D,IAAI,CAAC,cAAc,GAAG,IAAI,+BAAc,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAO,CAAC,CAAC;QAC9E,CAAC,CAAC;QAEK,eAAU,GAAG,GAAuB,EAAE;;YACzC,IAAI;gBACA,IAAI,CAAC,kBAAkB,CAAC,uBAAuB,GAAG,IAAI,CAAC;gBAEvD,IAAI,0BAA0B,GAAG,IAAI,CAAC,kBAAwC,CAAC;gBAC/E,IAAI,IAAI,CAAC,kBAAkB,CAAC,YAAY;oBAAE,0BAA0B,CAAC,YAAY,GAAG,aAAa,CAAC;gBAClG,IAAI,IAAI,CAAC,kBAAkB,CAAC,WAAW;oBAAE,0BAA0B,CAAC,YAAY,GAAG,aAAa,CAAC;gBACjG,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI;oBAAE,0BAA0B,CAAC,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;gBAEjG,MAAM,MAAM,GAAG,MAAM,IAAA,eAAK,EAAC,0BAA0B,CAAC,CAAC;gBAEvD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;gBAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;gBACjC,IAAI,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;gBAEvB,IAAI,IAAI,CAAC,kBAAkB,CAAC,WAAW,EAAE;oBACrC,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;iBAC7B;gBAED,IAAI,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE;oBACtC,MAAM,MAAM,GAAG,cAAI,CAAC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC;oBAClE,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,QAAQ,EAAE,EAAE;wBACzF,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;4BAC1C,OAAO,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;yBAC1E;wBAED,OAAO,CAAC,CAAC,QAAQ,CAAC;oBACtB,CAAC,CAAC,CAAC;oBAEH,MAAM,UAAU,GAAG,IAAI,mBAAU,CAC7B,sBAAsB,GAAG,OAAO,EAChC;wBACI,wBAAwB,EAAE,KAAK;wBAC/B,MAAM,EAAE,OAAO;qBAClB,EACD,IAAI,CACP,CAAC;oBAEF,MAAM,QAAQ,GAAG,MAAA,IAAI,CAAC,kBAAkB,CAAC,QAAQ,mCAAI,MAAM,CAAC;oBAE5D,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CACpB,KAAK,CAAC,GAAG,CAAC,CAAO,IAAI,EAAE,EAAE;wBACrB,MAAM,IAAI,GAAG,MAAM,kBAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;wBACpD,MAAM,YAAY,GAAG,cAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;wBAElD,MAAM,UAAU,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;wBAC7D,OAAO;4BACH,QAAQ,EAAE,YAAY;4BACtB,KAAK,EAAE,IAAI,CAAC,KAAK;4BACjB,IAAI,EAAE,cAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI;4BACnC,IAAI,EAAE,IAAI,CAAC,IAAI;yBAClB,CAAC;oBACN,CAAC,CAAA,CAAC,CACL,CAAC;oBAEF,MAAM,kBAAE,CAAC,EAAE,CAAC,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;iBACjE;gBACD,OAAO;oBACH,IAAI,EAAE,IAAI;oBACV,IAAI,EAAE;wBACF,OAAO;wBACP,UAAU;qBACb;iBACJ,CAAC;aACL;YAAC,OAAO,GAAG,EAAE;gBACV,MAAM,IAAI,oBAAW,CAAC,uCAAuC,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;aAC1G;QACL,CAAC,CAAA,CAAC;QAEK,oBAAe,GAAG,GAAiC,EAAE;YACxD,IAAI;gBACA,MAAM,CAAC,GAAG,IAAI,CAAC,kBAAwC,CAAC;gBACxD,IAAI,IAAI,CAAC,kBAAkB,CAAC,YAAY;oBAAE,CAAC,CAAC,YAAY,GAAG,aAAa,CAAC;gBACzE,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI;oBAAE,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;gBACxE,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC;gBAClB,MAAM,GAAG,GAAG,MAAM,IAAA,eAAK,EAAC,CAAC,CAAC,CAAC;gBAC3B,OAAO,GAAG,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aACnG;YAAC,OAAO,GAAG,EAAE;gBACV,aAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACf,OAAO,IAAI,CAAC;aACf;QACL,CAAC,CAAA,CAAC;QAzFE,IAAI,CAAC,cAAc,GAAG,IAAI,+BAAc,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAO,CAAC,CAAC;IAC9E,CAAC;CAyFJ;AAhGD,oDAgGC"}

package/dist/integration-engine/datasources/protocol-strategy/IProtocolStrategy.d.ts CHANGED Viewed

@@ -1,7 +1,9 @@
 /// <reference types="node" />
-import { ModelAttributes } from "sequelize";
+/// <reference types="node" />
 import { DataSourceStream } from "./..";
 import { AccessOptions } from "basic-ftp";
+import { ReadStream } from "fs";
+import { ModelAttributes } from "sequelize";
 export interface IHTTPSettings {
     /** (optional) Data to send with request, e.g. credentials */
     body?: any;
@@ -13,6 +15,8 @@ export interface IHTTPSettings {
     method: string;
     /** Url of the data source. */
     url: string;
+    /** Tmp directory, default: /tmp */
+    tmpDir?: string;
     httpsAgent?: any;
     strictSSL?: boolean;
     encoding?: any;
@@ -28,8 +32,12 @@ export declare enum FTPTargetType {
     FOLDER = "FOLDER",
     COMPRESSED = "COMPRESSED"
 }
+export declare enum FTPReturnType {
+    File = 0,
+    Stream = 1
+}
 export interface IFile {
-    data: Buffer | Record<string, any> | string;
+    data: ReadStream | Buffer | Record<string, any> | string;
     filepath: string;
     name: string;
     mtime?: string;
@@ -47,6 +55,8 @@ export interface IFTPSettings {
     tmpDir?: string;
     /** Type of downloaded file, default: FTPTargetType.FILE */
     targetType?: FTPTargetType;
+    /** Return a file/stream or a readable stream/collection of readable streams */
+    returnType?: FTPReturnType;
     /** Whitelist of files for FOLDER and COMPRESSED (archive) type **/
     whitelistedFiles?: string[];
     /** Encoding for file content, i.e. 'utf8' **/

package/dist/integration-engine/datasources/protocol-strategy/IProtocolStrategy.js CHANGED Viewed

@@ -1,12 +1,17 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isFileObject = exports.FTPTargetType = void 0;
+exports.isFileObject = exports.FTPReturnType = exports.FTPTargetType = void 0;
 var FTPTargetType;
 (function (FTPTargetType) {
     FTPTargetType["FILE"] = "FILE";
     FTPTargetType["FOLDER"] = "FOLDER";
     FTPTargetType["COMPRESSED"] = "COMPRESSED";
 })(FTPTargetType = exports.FTPTargetType || (exports.FTPTargetType = {}));
+var FTPReturnType;
+(function (FTPReturnType) {
+    FTPReturnType[FTPReturnType["File"] = 0] = "File";
+    FTPReturnType[FTPReturnType["Stream"] = 1] = "Stream";
+})(FTPReturnType = exports.FTPReturnType || (exports.FTPReturnType = {}));
 const isFileObject = (obj) => {
     return obj instanceof Object && obj.data !== undefined && obj.filepath !== undefined && obj.name !== undefined;
 };

package/dist/integration-engine/datasources/protocol-strategy/IProtocolStrategy.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"IProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/IProtocolStrategy.ts"],"names":[],"mappings":";;;~~AA8BA~~,IAAY,aAIX;AAJD,WAAY,aAAa;IACrB,8BAAa,CAAA;IACb,kCAAiB,CAAA;IACjB,0CAAyB,CAAA;AAC7B,CAAC,EAJW,aAAa,GAAb,qBAAa,KAAb,qBAAa,QAIxB;AAUM,MAAM,YAAY,GAAG,CAAC,GAAQ,EAAgB,EAAE;IACnD,OAAO,GAAG,YAAY,MAAM,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC;AACnH,CAAC,CAAC;AAFW,QAAA,YAAY,gBAEvB"}
1	+ {"version":3,"file":"IProtocolStrategy.js","sourceRoot":"","sources":["../../../../src/integration-engine/datasources/protocol-strategy/IProtocolStrategy.ts"],"names":[],"mappings":";;;AAkCA,IAAY,aAIX;AAJD,WAAY,aAAa;IACrB,8BAAa,CAAA;IACb,kCAAiB,CAAA;IACjB,0CAAyB,CAAA;AAC7B,CAAC,EAJW,aAAa,GAAb,qBAAa,KAAb,qBAAa,QAIxB;AAED,IAAY,aAGX;AAHD,WAAY,aAAa;IACrB,iDAAI,CAAA;IACJ,qDAAM,CAAA;AACV,CAAC,EAHW,aAAa,GAAb,qBAAa,KAAb,qBAAa,QAGxB;AAUM,MAAM,YAAY,GAAG,CAAC,GAAQ,EAAgB,EAAE;IACnD,OAAO,GAAG,YAAY,MAAM,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC;AACnH,CAAC,CAAC;AAFW,QAAA,YAAY,gBAEvB"}

package/dist/integration-engine/datasources/protocol-strategy/helpers/FTPFileProcessor.d.ts ADDED Viewed

@@ -0,0 +1,13 @@
+import { IFile, IFTPSettings } from "../IProtocolStrategy";
+export declare class FTPFileProcessor {
+    private readonly connectionSettings;
+    private readonly downloadPath;
+    private readonly fileCompressor;
+    constructor(connectionSettings: IFTPSettings, downloadPath: string);
+    processTargetFile(): Promise<IFile>;
+    processTargetFolder(prefix: string): Promise<IFile[]>;
+    processTargetCompressed(prefix: string): Promise<IFile[]>;
+    private prepareFileData;
+    private encode;
+    private createWhitelistPredicate;
+}

package/dist/integration-engine/datasources/protocol-strategy/helpers/FTPFileProcessor.js ADDED Viewed

@@ -0,0 +1,87 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FTPFileProcessor = void 0;
+const IProtocolStrategy_1 = require("../IProtocolStrategy");
+const FileCompressor_1 = require("../../../helpers/FileCompressor");
+const fs_1 = require("fs");
+const promises_1 = __importDefault(require("fs/promises"));
+const path_1 = __importDefault(require("path"));
+class FTPFileProcessor {
+    constructor(connectionSettings, downloadPath) {
+        this.connectionSettings = connectionSettings;
+        this.downloadPath = downloadPath;
+        this.fileCompressor = new FileCompressor_1.FileCompressor(this.downloadPath);
+    }
+    processTargetFile() {
+        return this.prepareFileData(path_1.default.join(this.downloadPath, this.connectionSettings.filename), this.connectionSettings.filename);
+    }
+    processTargetFolder(prefix) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const whitelistPredicate = this.createWhitelistPredicate();
+            const files = (yield promises_1.default.readdir(path_1.default.join(this.downloadPath, this.connectionSettings.filename))).filter(whitelistPredicate);
+            const result = [];
+            for (const file of files) {
+                result.push(yield this.prepareFileData(path_1.default.join(this.downloadPath, prefix, file), path_1.default.join(prefix, file)));
+            }
+            return result;
+        });
+    }
+    processTargetCompressed(prefix) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const whitelistPredicate = this.createWhitelistPredicate();
+            const { files } = yield this.fileCompressor.inflateFile(path_1.default.join(this.downloadPath, this.connectionSettings.filename), whitelistPredicate);
+            const result = [];
+            for (const file of files) {
+                const fileResult = yield this.prepareFileData(file.path, path_1.default.join(prefix, file.name));
+                result.push(Object.assign(Object.assign({}, fileResult), { mtime: file.mtime.toISOString(), type: file.type }));
+                if (fileResult.data instanceof fs_1.ReadStream) {
+                    fileResult.data.once("end", () => __awaiter(this, void 0, void 0, function* () {
+                        yield promises_1.default.rm(file.path, { force: true });
+                    }));
+                }
+                else {
+                    yield promises_1.default.rm(file.path, { force: true });
+                }
+            }
+            return result;
+        });
+    }
+    prepareFileData(absoluteFilePath, relativeFilePath) {
+        var _a;
+        return __awaiter(this, void 0, void 0, function* () {
+            const data = this.connectionSettings.returnType === IProtocolStrategy_1.FTPReturnType.Stream
+                ? (0, fs_1.createReadStream)(absoluteFilePath, (_a = this.connectionSettings.encoding) !== null && _a !== void 0 ? _a : undefined)
+                : this.encode(yield promises_1.default.readFile(absoluteFilePath));
+            return {
+                data,
+                filepath: relativeFilePath,
+                name: path_1.default.parse(relativeFilePath).name,
+            };
+        });
+    }
+    encode(data) {
+        return this.connectionSettings.encoding ? data.toString(this.connectionSettings.encoding) : data;
+    }
+    createWhitelistPredicate() {
+        return (filename) => {
+            if (this.connectionSettings.whitelistedFiles) {
+                return this.connectionSettings.whitelistedFiles.indexOf(filename) >= 0;
+            }
+            return !!filename;
+        };
+    }
+}
+exports.FTPFileProcessor = FTPFileProcessor;
+//# sourceMappingURL=FTPFileProcessor.js.map

package/dist/integration-engine/datasources/protocol-strategy/helpers/FTPFileProcessor.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"FTPFileProcessor.js","sourceRoot":"","sources":["../../../../../src/integration-engine/datasources/protocol-strategy/helpers/FTPFileProcessor.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,4DAAyG;AACzG,oEAA4D;AAE5D,2BAAkD;AAClD,2DAA6B;AAC7B,gDAAwB;AAExB,MAAa,gBAAgB;IAGzB,YAA6B,kBAAgC,EAAmB,YAAoB;QAAvE,uBAAkB,GAAlB,kBAAkB,CAAc;QAAmB,iBAAY,GAAZ,YAAY,CAAQ;QAChG,IAAI,CAAC,cAAc,GAAG,IAAI,+BAAc,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAChE,CAAC;IAEM,iBAAiB;QACpB,OAAO,IAAI,CAAC,eAAe,CACvB,cAAI,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAC9D,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CACnC,CAAC;IACN,CAAC;IAEY,mBAAmB,CAAC,MAAc;;YAC3C,MAAM,kBAAkB,GAAG,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAC3D,MAAM,KAAK,GAAG,CAAC,MAAM,kBAAE,CAAC,OAAO,CAAC,cAAI,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,CACnG,kBAAkB,CACrB,CAAC;YAEF,MAAM,MAAM,GAAY,EAAE,CAAC;YAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;gBACtB,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,cAAI,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,cAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;aAChH;YAED,OAAO,MAAM,CAAC;QAClB,CAAC;KAAA;IAEY,uBAAuB,CAAC,MAAc;;YAC/C,MAAM,kBAAkB,GAAG,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAC3D,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,WAAW,CACnD,cAAI,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAC9D,kBAAkB,CACrB,CAAC;YAEF,MAAM,MAAM,GAAY,EAAE,CAAC;YAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;gBACtB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,EAAE,cAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;gBAEvF,MAAM,CAAC,IAAI,iCACJ,UAAU,KACb,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,EAC/B,IAAI,EAAE,IAAI,CAAC,IAAI,IACjB,CAAC;gBAEH,IAAI,UAAU,CAAC,IAAI,YAAY,eAAU,EAAE;oBACvC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,GAAS,EAAE;wBACnC,MAAM,kBAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC5C,CAAC,CAAA,CAAC,CAAC;iBACN;qBAAM;oBACH,MAAM,kBAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;iBAC3C;aACJ;YAED,OAAO,MAAM,CAAC;QAClB,CAAC;KAAA;IAEa,eAAe,CAAC,gBAAwB,EAAE,gBAAwB;;;YAC5E,MAAM,IAAI,GACN,IAAI,CAAC,kBAAkB,CAAC,UAAU,KAAK,iCAAa,CAAC,MAAM;gBACvD,CAAC,CAAC,IAAA,qBAAgB,EAAC,gBAAgB,EAAE,MAAA,IAAI,CAAC,kBAAkB,CAAC,QAAQ,mCAAI,SAAS,CAAC;gBACnF,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,kBAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC;YAE3D,OAAO;gBACH,IAAI;gBACJ,QAAQ,EAAE,gBAAgB;gBAC1B,IAAI,EAAE,cAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,IAAI;aAC1C,CAAC;;KACL;IAEO,MAAM,CAAC,IAAY;QACvB,OAAO,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACrG,CAAC;IAEO,wBAAwB;QAC5B,OAAO,CAAC,QAAQ,EAAE,EAAE;YAChB,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;gBAC1C,OAAO,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;aAC1E;YAED,OAAO,CAAC,CAAC,QAAQ,CAAC;QACtB,CAAC,CAAC;IACN,CAAC;CACJ;AAnFD,4CAmFC"}

package/dist/integration-engine/helpers/FileCompressor.d.ts ADDED Viewed

@@ -0,0 +1,17 @@
+/// <reference types="node" />
+import { IInflatedFileMetadata } from "./interfaces/IInflatedFileMetadata";
+import { Predicate } from "./interfaces/Predicate";
+interface IInflateOutput {
+    extractionPath: string;
+    files: IInflatedFileMetadata[];
+}
+export declare class FileCompressor {
+    private readonly basePath;
+    private static LIB_OPTIONS;
+    constructor(basePath: string);
+    inflateBuffer(buffer: Buffer, extractFilePredicate: Predicate<[string]>): Promise<IInflateOutput>;
+    inflateFile(archivePath: string, extractFilePredicate: Predicate<[string]>): Promise<IInflateOutput>;
+    private processZipFile;
+    private getFileMetadata;
+}
+export {};

package/dist/integration-engine/helpers/FileCompressor.js ADDED Viewed

@@ -0,0 +1,113 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FileCompressor = void 0;
+const crypto_1 = __importDefault(require("crypto"));
+const fs_1 = require("fs");
+const promises_1 = __importDefault(require("fs/promises"));
+const path_1 = __importDefault(require("path"));
+const yauzl_1 = __importDefault(require("yauzl"));
+const Logger_1 = require("./Logger");
+class FileCompressor {
+    constructor(basePath) {
+        this.basePath = basePath;
+    }
+    inflateBuffer(buffer, extractFilePredicate) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const archiveName = crypto_1.default.randomBytes(5).toString("hex");
+            return new Promise((resolve, reject) => {
+                yauzl_1.default.fromBuffer(buffer, FileCompressor.LIB_OPTIONS, (err, zipFile) => __awaiter(this, void 0, void 0, function* () {
+                    if (err) {
+                        reject(err);
+                    }
+                    try {
+                        const result = yield this.processZipFile(archiveName, zipFile, extractFilePredicate);
+                        resolve(result);
+                    }
+                    catch (err) {
+                        reject(err);
+                    }
+                    zipFile.close();
+                }));
+            });
+        });
+    }
+    inflateFile(archivePath, extractFilePredicate) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const archiveName = path_1.default.basename(archivePath);
+            return new Promise((resolve, reject) => {
+                yauzl_1.default.open(archivePath, FileCompressor.LIB_OPTIONS, (err, zipFile) => __awaiter(this, void 0, void 0, function* () {
+                    if (err) {
+                        reject(err);
+                    }
+                    try {
+                        const result = yield this.processZipFile(archiveName, zipFile, extractFilePredicate);
+                        resolve(result);
+                    }
+                    catch (err) {
+                        zipFile.close();
+                        reject(err);
+                    }
+                    zipFile.close();
+                }));
+            });
+        });
+    }
+    processZipFile(archiveName, zipFile, extractFilePredicate) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const extractionPath = yield promises_1.default.mkdtemp(path_1.default.join(this.basePath, `${archiveName}-`));
+            let files = [];
+            return new Promise((resolve, reject) => {
+                zipFile.readEntry();
+                zipFile.on("entry", (entry) => {
+                    if (!extractFilePredicate(entry.fileName)) {
+                        zipFile.readEntry();
+                        return;
+                    }
+                    zipFile.openReadStream(entry, (err, readStream) => {
+                        if (err) {
+                            reject(err);
+                        }
+                        const filePath = path_1.default.resolve(extractionPath, entry.fileName);
+                        const fileStream = (0, fs_1.createWriteStream)(filePath).on("finish", () => __awaiter(this, void 0, void 0, function* () {
+                            Logger_1.log.info(`Extracted ${archiveName}:${entry.fileName}`);
+                            files.push(yield this.getFileMetadata(filePath, entry.fileName));
+                            zipFile.readEntry();
+                        }));
+                        readStream.pipe(fileStream);
+                    });
+                });
+                zipFile.once("end", () => {
+                    resolve({ extractionPath, files });
+                });
+            });
+        });
+    }
+    getFileMetadata(filePath, fileName) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { mtime } = yield promises_1.default.stat(filePath);
+            return {
+                path: filePath,
+                name: fileName,
+                mtime,
+                type: "file", // backwards compatibility with the "decompress" output
+            };
+        });
+    }
+}
+exports.FileCompressor = FileCompressor;
+FileCompressor.LIB_OPTIONS = {
+    lazyEntries: true,
+};
+//# sourceMappingURL=FileCompressor.js.map

package/dist/integration-engine/helpers/FileCompressor.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"FileCompressor.js","sourceRoot":"","sources":["../../../src/integration-engine/helpers/FileCompressor.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,oDAA4B;AAC5B,2BAAuC;AACvC,2DAA6B;AAC7B,gDAAwB;AACxB,kDAAuD;AAGvD,qCAA+B;AAO/B,MAAa,cAAc;IAKvB,YAA6B,QAAgB;QAAhB,aAAQ,GAAR,QAAQ,CAAQ;IAAG,CAAC;IAEpC,aAAa,CAAC,MAAc,EAAE,oBAAyC;;YAChF,MAAM,WAAW,GAAG,gBAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC1D,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACnC,eAAK,CAAC,UAAU,CAAC,MAAM,EAAE,cAAc,CAAC,WAAW,EAAE,CAAO,GAAG,EAAE,OAAO,EAAE,EAAE;oBACxE,IAAI,GAAG,EAAE;wBACL,MAAM,CAAC,GAAG,CAAC,CAAC;qBACf;oBAED,IAAI;wBACA,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE,OAAO,EAAE,oBAAoB,CAAC,CAAC;wBACrF,OAAO,CAAC,MAAM,CAAC,CAAC;qBACnB;oBAAC,OAAO,GAAG,EAAE;wBACV,MAAM,CAAC,GAAG,CAAC,CAAC;qBACf;oBAED,OAAO,CAAC,KAAK,EAAE,CAAC;gBACpB,CAAC,CAAA,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;QACP,CAAC;KAAA;IAEY,WAAW,CAAC,WAAmB,EAAE,oBAAyC;;YACnF,MAAM,WAAW,GAAG,cAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC/C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACnC,eAAK,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,CAAC,WAAW,EAAE,CAAO,GAAG,EAAE,OAAO,EAAE,EAAE;oBACvE,IAAI,GAAG,EAAE;wBACL,MAAM,CAAC,GAAG,CAAC,CAAC;qBACf;oBAED,IAAI;wBACA,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE,OAAO,EAAE,oBAAoB,CAAC,CAAC;wBACrF,OAAO,CAAC,MAAM,CAAC,CAAC;qBACnB;oBAAC,OAAO,GAAG,EAAE;wBACV,OAAO,CAAC,KAAK,EAAE,CAAC;wBAChB,MAAM,CAAC,GAAG,CAAC,CAAC;qBACf;oBAED,OAAO,CAAC,KAAK,EAAE,CAAC;gBACpB,CAAC,CAAA,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;QACP,CAAC;KAAA;IAEa,cAAc,CACxB,WAAmB,EACnB,OAAgB,EAChB,oBAAyC;;YAEzC,MAAM,cAAc,GAAG,MAAM,kBAAE,CAAC,OAAO,CAAC,cAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC;YACrF,IAAI,KAAK,GAA4B,EAAE,CAAC;YAExC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACnC,OAAO,CAAC,SAAS,EAAE,CAAC;gBACpB,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE;oBACjC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE;wBACvC,OAAO,CAAC,SAAS,EAAE,CAAC;wBACpB,OAAO;qBACV;oBAED,OAAO,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,EAAE;wBAC9C,IAAI,GAAG,EAAE;4BACL,MAAM,CAAC,GAAG,CAAC,CAAC;yBACf;wBAED,MAAM,QAAQ,GAAG,cAAI,CAAC,OAAO,CAAC,cAAc,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;wBAC9D,MAAM,UAAU,GAAG,IAAA,sBAAiB,EAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAS,EAAE;4BACnE,YAAG,CAAC,IAAI,CAAC,aAAa,WAAW,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;4BACvD,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;4BAEjE,OAAO,CAAC,SAAS,EAAE,CAAC;wBACxB,CAAC,CAAA,CAAC,CAAC;wBAEH,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;oBAChC,CAAC,CAAC,CAAC;gBACP,CAAC,CAAC,CAAC;gBAEH,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE;oBACrB,OAAO,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;gBACvC,CAAC,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;QACP,CAAC;KAAA;IAEa,eAAe,CAAC,QAAgB,EAAE,QAAgB;;YAC5D,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,kBAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC1C,OAAO;gBACH,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,QAAQ;gBACd,KAAK;gBACL,IAAI,EAAE,MAAM,EAAE,uDAAuD;aACxE,CAAC;QACN,CAAC;KAAA;;AA/FL,wCAgGC;AA/FkB,0BAAW,GAAY;IAClC,WAAW,EAAE,IAAI;CACpB,CAAC"}

package/dist/integration-engine/helpers/interfaces/IInflatedFileMetadata.d.ts ADDED Viewed

@@ -0,0 +1,6 @@
+export interface IInflatedFileMetadata {
+    path: string;
+    name: string;
+    mtime: Date;
+    type: string;
+}

package/dist/integration-engine/helpers/interfaces/IInflatedFileMetadata.js ADDED Viewed

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=IInflatedFileMetadata.js.map

package/dist/integration-engine/helpers/interfaces/IInflatedFileMetadata.js.map ADDED Viewed

	@@ -0,0 +1 @@
1	+ {"version":3,"file":"IInflatedFileMetadata.js","sourceRoot":"","sources":["../../../../src/integration-engine/helpers/interfaces/IInflatedFileMetadata.ts"],"names":[],"mappings":""}

package/dist/integration-engine/helpers/interfaces/Predicate.d.ts ADDED Viewed

	@@ -0,0 +1 @@
1	+ export declare type Predicate<T extends any[]> = (...params: T) => boolean;

package/dist/integration-engine/helpers/interfaces/Predicate.js ADDED Viewed

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=Predicate.js.map

package/dist/integration-engine/helpers/interfaces/Predicate.js.map ADDED Viewed

	@@ -0,0 +1 @@
1	+ {"version":3,"file":"Predicate.js","sourceRoot":"","sources":["../../../../src/integration-engine/helpers/interfaces/Predicate.ts"],"names":[],"mappings":""}

package/dist/integration-engine/models/RedisModel.d.ts CHANGED Viewed

@@ -1,6 +1,8 @@
+import { IRedisModel, IRedisSettings } from "./";
 import { Validator } from "@golemio/validator";
+import { ReadStream } from "fs";
 import { Redis } from "ioredis";
-import { IRedisModel, IRedisSettings } from "./";
+import { Readable } from "stream";
 export declare class RedisModel implements IRedisModel {
     /** Model name */
     name: string;
@@ -28,6 +30,14 @@ export declare class RedisModel implements IRedisModel {
     hset: (key: string, data: any) => Promise<any>;
     get: (key: string) => Promise<any>;
     hget: (key: string) => Promise<any>;
+    /**
+     * Stream data into a key from file read stream via SET/APPEND
+     */
+    pipeStream: (key: string, readStream: ReadStream) => Promise<void>;
+    /**
+     * Get a readable stream of data from a key via GETRANGE
+     */
+    getReadableStream: (key: string) => Readable;
     delete: (key?: string) => Promise<any>;
     truncate: (namespace?: boolean) => Promise<number>;
     flush: () => Promise<"OK">;

package/dist/integration-engine/models/RedisModel.js CHANGED Viewed

@@ -8,12 +8,17 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.RedisModel = void 0;
-const errors_1 = require("@golemio/errors");
-const utils_1 = require("@golemio/utils");
 const connectors_1 = require("../connectors");
 const helpers_1 = require("../helpers");
+const errors_1 = require("@golemio/errors");
+const utils_1 = require("@golemio/utils");
+const redis_rstream_1 = __importDefault(require("redis-rstream"));
+const redis_wstream_1 = __importDefault(require("redis-wstream"));
 class RedisModel {
     constructor(name, settings, validator = null) {
         var _a;
@@ -88,6 +93,27 @@ class RedisModel {
             // getting and decoding the data from redis hash
             return this.decodeDataAfterGet(yield this.connection.hget(this.prefix, key));
         });
+        /**
+         * Stream data into a key from file read stream via SET/APPEND
+         */
+        this.pipeStream = (key, readStream) => __awaiter(this, void 0, void 0, function* () {
+            const redisWriteStream = new redis_wstream_1.default(this.connection, `${this.prefix}:${key}`);
+            return new Promise((resolve, reject) => {
+                redisWriteStream.on("finish", () => {
+                    resolve();
+                });
+                redisWriteStream.on("error", (err) => {
+                    reject(err);
+                });
+                readStream.pipe(redisWriteStream);
+            });
+        });
+        /**
+         * Get a readable stream of data from a key via GETRANGE
+         */
+        this.getReadableStream = (key) => {
+            return new redis_rstream_1.default(this.connection, `${this.prefix}:${key}`);
+        };
         this.delete = (key) => __awaiter(this, void 0, void 0, function* () {
             if (key) {
                 return this.connection.del(`${this.prefix}:${key}`);

package/dist/integration-engine/models/RedisModel.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"RedisModel.js","sourceRoot":"","sources":["../../../src/integration-engine/models/RedisModel.ts"],"names":[],"mappings":"~~;;;;;;;;;;;;~~AAAA,4CAA8C;AAC9C,0CAAgD;~~AAGhD~~,~~8CAAgD~~;~~AAChD~~,~~wCAAkC~~;~~AAGlC~~,MAAa,UAAU;IAgBnB,YAAY,IAAY,EAAE,QAAwB,EAAE,YAA8B,IAAI;;QAetF;;;;;;WAMG;QACI,QAAG,GAAG,CAAO,GAAW,EAAE,IAAS,EAAE,UAAmB,EAAE,eAAwB,EAAgB,EAAE;YACvG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAE1B,IAAI,IAAI,YAAY,KAAK,EAAE;gBACvB,8BAA8B;gBAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;gBAEtC,KAAK,MAAM,QAAQ,IAAI,IAAI,EAAE;oBACzB,MAAM,QAAQ,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;oBAClE,MAAM,WAAW,GAAG,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;oBAExD,IAAI,UAAU,EAAE;wBACZ,OAAO,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;qBACzD;yBAAM,IAAI,eAAe,EAAE;wBACxB,iCAAiC;wBACjC,OAAO,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,eAAe,CAAC,CAAC;qBACpE;yBAAM;wBACH,OAAO,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;qBAC3C;iBACJ;gBAED,2BAA2B;gBAC3B,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;aACvB;iBAAM;gBACH,MAAM,QAAQ,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,CAAC;gBAC9D,MAAM,WAAW,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;gBAEpD,IAAI,UAAU,EAAE;oBACZ,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;iBACnE;qBAAM,IAAI,eAAe,EAAE;oBACxB,iCAAiC;oBACjC,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,eAAe,CAAC,CAAC;iBAC9E;qBAAM;oBACH,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;iBACrD;aACJ;QACL,CAAC,CAAA,CAAC;QAEK,SAAI,GAAG,CAAO,GAAW,EAAE,IAAS,EAAgB,EAAE;YACzD,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAE1B,IAAI,IAAI,YAAY,KAAK,EAAE;gBACvB,8BAA8B;gBAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;gBAEtC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;oBACf,MAAM,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;oBAChC,6CAA6C;oBAC7C,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpE,CAAC,CAAC,CAAC;gBAEH,2BAA2B;gBAC3B,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;aACvB;iBAAM;gBACH,MAAM,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBACnC,6CAA6C;gBAC7C,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC;aAChF;QACL,CAAC,CAAA,CAAC;QAEK,QAAG,GAAG,CAAO,GAAW,EAAgB,EAAE;YAC7C,gDAAgD;YAChD,OAAO,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC;QACvF,CAAC,CAAA,CAAC;QAEK,SAAI,GAAG,CAAO,GAAW,EAAgB,EAAE;YAC9C,gDAAgD;YAChD,OAAO,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;QACjF,CAAC,CAAA,CAAC;QAEK,WAAM,GAAG,CAAO,GAAY,EAAgB,EAAE;YACjD,IAAI,GAAG,EAAE;gBACL,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC;aACvD;YACD,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACjD,CAAC,CAAA,CAAC;QAEK,aAAQ,GAAG,CAAC,SAAS,GAAG,IAAI,EAAmB,EAAE;YACpD,MAAM,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC;YACxD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC;YAE5C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACnC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAc,EAAE,EAAE;oBACjC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;wBACjB,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;qBAC5B;gBACL,CAAC,CAAC,CAAC;gBAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;oBACvB,aAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBACf,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;gBACvB,CAAC,CAAC,CAAC;gBAEH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAS,EAAE;;oBACxB,IAAI;wBACA,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;wBACrC,MAAM,WAAW,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,MAAM,CAAC,CAAC,GAAW,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,GAAI,EAAE,CAAC,CAAC,CAAY,EAAE,CAAC,CAAC,mCAAI,CAAC,CAAC;wBACzF,aAAG,CAAC,IAAI,CAAC,yBAAyB,MAAM,aAAa,WAAW,QAAQ,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,MAAM,mCAAI,CAAC,WAAW,CAAC,CAAC;wBACxG,OAAO,CAAC,WAAW,CAAC,CAAC;qBACxB;oBAAC,OAAO,GAAG,EAAE;wBACV,aAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;wBACf,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;qBACtB;gBACL,CAAC,CAAA,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;QACP,CAAC,CAAC;QAEK,UAAK,GAAG,GAAkB,EAAE;YAC/B,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QACrC,CAAC,CAAC;QAEM,aAAQ,GAAG,CAAO,IAAS,EAAE,EAAE;YACnC,kBAAkB;YAClB,IAAI,IAAI,CAAC,SAAS,EAAE;gBAChB,IAAI;oBACA,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;iBACvC;gBAAC,OAAO,GAAG,EAAE;oBACV,MAAM,IAAI,oBAAW,CAAC,8BAA8B,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;iBACrF;aACJ;iBAAM,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,wBAAwB,EAAE;gBACzD,aAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,+BAA+B,CAAC,CAAC;aACzD;QACL,CAAC,CAAA,CAAC;QAEM,aAAQ,GAAG,CAAC,GAAW,EAAE,IAAS,EAAE,EAAE;YAC1C,0CAA0C;YAC1C,IAAI,IAAI,CAAC,wBAAwB,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;gBAC3D,MAAM,IAAI,oBAAW,CAAC,oCAAoC,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;aAC5F;YACD,OAAO,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAA,sBAAc,EAAS,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACnF,CAAC,CAAC;~~QAxJE~~,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,UAAU,GAAG,2BAAc,CAAC,aAAa,EAAE,CAAC;QACjD,IAAI,CAAC,wBAAwB,GAAG,QAAQ,CAAC,wBAAwB,CAAC;QAClE,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAE3B,IAAI,CAAC,oBAAoB,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC;QACzG,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC;QAEnG,IAAI,MAAA,IAAI,CAAC,SAAS,0CAAE,SAAS,EAAE;YAC3B,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,aAAG,CAAC,CAAC;SACjC;IACL,CAAC;~~CA6IJ~~;~~AA1KD~~,~~gCA0KC~~"}
1	+ {"version":3,"file":"RedisModel.js","sourceRoot":"","sources":["../../../src/integration-engine/models/RedisModel.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,8CAAgD;AAChD,wCAAkC;AAElC,4CAA8C;AAC9C,0CAAgD;AAIhD,kEAAyC;AACzC,kEAAyC;AAGzC,MAAa,UAAU;IAgBnB,YAAY,IAAY,EAAE,QAAwB,EAAE,YAA8B,IAAI;;QAetF;;;;;;WAMG;QACI,QAAG,GAAG,CAAO,GAAW,EAAE,IAAS,EAAE,UAAmB,EAAE,eAAwB,EAAgB,EAAE;YACvG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAE1B,IAAI,IAAI,YAAY,KAAK,EAAE;gBACvB,8BAA8B;gBAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;gBAEtC,KAAK,MAAM,QAAQ,IAAI,IAAI,EAAE;oBACzB,MAAM,QAAQ,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;oBAClE,MAAM,WAAW,GAAG,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;oBAExD,IAAI,UAAU,EAAE;wBACZ,OAAO,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;qBACzD;yBAAM,IAAI,eAAe,EAAE;wBACxB,iCAAiC;wBACjC,OAAO,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,eAAe,CAAC,CAAC;qBACpE;yBAAM;wBACH,OAAO,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;qBAC3C;iBACJ;gBAED,2BAA2B;gBAC3B,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;aACvB;iBAAM;gBACH,MAAM,QAAQ,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,CAAC;gBAC9D,MAAM,WAAW,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;gBAEpD,IAAI,UAAU,EAAE;oBACZ,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;iBACnE;qBAAM,IAAI,eAAe,EAAE;oBACxB,iCAAiC;oBACjC,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,eAAe,CAAC,CAAC;iBAC9E;qBAAM;oBACH,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;iBACrD;aACJ;QACL,CAAC,CAAA,CAAC;QAEK,SAAI,GAAG,CAAO,GAAW,EAAE,IAAS,EAAgB,EAAE;YACzD,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAE1B,IAAI,IAAI,YAAY,KAAK,EAAE;gBACvB,8BAA8B;gBAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;gBAEtC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;oBACf,MAAM,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;oBAChC,6CAA6C;oBAC7C,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpE,CAAC,CAAC,CAAC;gBAEH,2BAA2B;gBAC3B,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;aACvB;iBAAM;gBACH,MAAM,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBACnC,6CAA6C;gBAC7C,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC;aAChF;QACL,CAAC,CAAA,CAAC;QAEK,QAAG,GAAG,CAAO,GAAW,EAAgB,EAAE;YAC7C,gDAAgD;YAChD,OAAO,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC;QACvF,CAAC,CAAA,CAAC;QAEK,SAAI,GAAG,CAAO,GAAW,EAAgB,EAAE;YAC9C,gDAAgD;YAChD,OAAO,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;QACjF,CAAC,CAAA,CAAC;QAEF;;WAEG;QACI,eAAU,GAAG,CAAO,GAAW,EAAE,UAAsB,EAAiB,EAAE;YAC7E,MAAM,gBAAgB,GAAa,IAAI,uBAAY,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC;YAE9F,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACnC,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;oBAC/B,OAAO,EAAE,CAAC;gBACd,CAAC,CAAC,CAAC;gBAEH,gBAAgB,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;oBACjC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC,CAAC,CAAC;gBAEH,UAAU,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YACtC,CAAC,CAAC,CAAC;QACP,CAAC,CAAA,CAAC;QAEF;;WAEG;QACI,sBAAiB,GAAG,CAAC,GAAW,EAAY,EAAE;YACjD,OAAO,IAAI,uBAAY,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC;QACtE,CAAC,CAAC;QAEK,WAAM,GAAG,CAAO,GAAY,EAAgB,EAAE;YACjD,IAAI,GAAG,EAAE;gBACL,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC;aACvD;YACD,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACjD,CAAC,CAAA,CAAC;QAEK,aAAQ,GAAG,CAAC,SAAS,GAAG,IAAI,EAAmB,EAAE;YACpD,MAAM,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC;YACxD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC;YAE5C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBACnC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAc,EAAE,EAAE;oBACjC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;wBACjB,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;qBAC5B;gBACL,CAAC,CAAC,CAAC;gBAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;oBACvB,aAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBACf,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;gBACvB,CAAC,CAAC,CAAC;gBAEH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAS,EAAE;;oBACxB,IAAI;wBACA,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;wBACrC,MAAM,WAAW,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,MAAM,CAAC,CAAC,GAAW,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,GAAI,EAAE,CAAC,CAAC,CAAY,EAAE,CAAC,CAAC,mCAAI,CAAC,CAAC;wBACzF,aAAG,CAAC,IAAI,CAAC,yBAAyB,MAAM,aAAa,WAAW,QAAQ,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,MAAM,mCAAI,CAAC,WAAW,CAAC,CAAC;wBACxG,OAAO,CAAC,WAAW,CAAC,CAAC;qBACxB;oBAAC,OAAO,GAAG,EAAE;wBACV,aAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;wBACf,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;qBACtB;gBACL,CAAC,CAAA,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;QACP,CAAC,CAAC;QAEK,UAAK,GAAG,GAAkB,EAAE;YAC/B,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QACrC,CAAC,CAAC;QAEM,aAAQ,GAAG,CAAO,IAAS,EAAE,EAAE;YACnC,kBAAkB;YAClB,IAAI,IAAI,CAAC,SAAS,EAAE;gBAChB,IAAI;oBACA,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;iBACvC;gBAAC,OAAO,GAAG,EAAE;oBACV,MAAM,IAAI,oBAAW,CAAC,8BAA8B,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;iBACrF;aACJ;iBAAM,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,wBAAwB,EAAE;gBACzD,aAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,+BAA+B,CAAC,CAAC;aACzD;QACL,CAAC,CAAA,CAAC;QAEM,aAAQ,GAAG,CAAC,GAAW,EAAE,IAAS,EAAE,EAAE;YAC1C,0CAA0C;YAC1C,IAAI,IAAI,CAAC,wBAAwB,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;gBAC3D,MAAM,IAAI,oBAAW,CAAC,oCAAoC,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;aAC5F;YACD,OAAO,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAA,sBAAc,EAAS,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACnF,CAAC,CAAC;QAlLE,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,UAAU,GAAG,2BAAc,CAAC,aAAa,EAAE,CAAC;QACjD,IAAI,CAAC,wBAAwB,GAAG,QAAQ,CAAC,wBAAwB,CAAC;QAClE,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAE3B,IAAI,CAAC,oBAAoB,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC;QACzG,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC;QAEnG,IAAI,MAAA,IAAI,CAAC,SAAS,0CAAE,SAAS,EAAE;YAC3B,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,aAAG,CAAC,CAAC;SACjC;IACL,CAAC;CAuKJ;AApMD,gCAoMC"}

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
     "name": "@golemio/core",
-    "version": "1.2.24",
+    "version": "1.2.25-dev.722047718",
     "description": "Golemio Core Module",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
@@ -36,13 +36,13 @@
         "@types/chai-as-promised": "7.1.2",
         "@types/content-type": "^1.1.3",
         "@types/debug": "0.0.31",
-        "@types/decompress": "^4.2.3",
         "@types/mocha": "^8.2.0",
         "@types/node": "^16.11.35",
         "@types/sinon": "7.0.13",
         "@types/ssh2-sftp-client": "^5.3.1",
         "@types/supertest": "^2.0.11",
         "@types/uuid": "^8.3.4",
+        "@types/yauzl": "^2.10.0",
         "chai": "4.2.0",
         "chai-as-promised": "7.1.1",
         "cross-env": "^7.0.3",
@@ -97,7 +97,6 @@
         "class-validator": "^0.13.2",
         "content-type": "^1.0.4",
         "debug": "3.1.0",
-        "decompress": "^4.2.1",
         "dotenv": "^8.2.0",
         "express": "^4.16.4",
         "express-validator": "^5.3.1",
@@ -120,12 +119,15 @@
         "pino-pretty": "^9.1.1",
         "prom-client": "^14.0.0",
         "redis": "^3.1.1",
+        "redis-rstream": "^1.0.1",
         "redis-semaphore": "^5.2.0",
+        "redis-wstream": "^1.0.0",
         "request": "2.88.0",
         "request-promise": "^4.2.5",
         "sequelize": "^6.21.2",
         "ssh2-sftp-client": "^7.1.0",
-        "xml2js-es6-promise": "^1.1.1"
+        "xml2js-es6-promise": "^1.1.1",
+        "yauzl": "^2.10.0"
     },
     "resolutions": {}
-}
+}