@ductape/sdk 0.0.3-beta11 → 0.0.3-beta13

package/dist/processor/services/processor.service.js

@@ -52,10 +52,11 @@ const types_1 = require("../../types");
 const logs_service_1 = __importDefault(require("../../logs/logs.service"));
 const inputs_service_1 = __importDefault(require("../../inputs/inputs.service"));
 const processor_utils_1 = require("../utils/processor.utils");
-const http_client_1 = __importDefault(require("../../clients/http.client"));
+const axios_1 = __importDefault(require("axios"));
 const processorApi_service_1 = require("../../api/services/processorApi.service");
 const expo_client_1 = __importDefault(require("../../clients/expo.client"));
 const handlebars_1 = require("handlebars");
+const functions_utils_1 = require("../../products/utils/functions.utils");
 const string_utils_1 = require("../../products/utils/string.utils");
 const create_productFeature_validator_1 = require("../../products/validators/joi-validators/create.productFeature.validator");
 const validators_1 = require("../../products/validators");
@@ -68,8 +69,6 @@ const postgres_repo_1 = require("../repos/postgres.repo");
 const storage_util_1 = require("../utils/storage.util");
 const sms_repo_1 = require("../repos/sms.repo");
 const quota_service_1 = __importDefault(require("./quota.service"));
-const fallback_service_1 = __importDefault(require("./fallback.service"));
-const productsApi_service_1 = require("../../api/services/productsApi.service");
 async function loadBrokerService() {
     if (typeof window === 'undefined') {
         const { createBrokerService } = await Promise.resolve().then(() => __importStar(require('./messagebrokers')));
@@ -90,14 +89,18 @@ class ProcessorService {
         this.public_key = public_key;
         this.user_id = user_id;
         this.token = token;
+        this.published = false;
         this.productBuilderService = new products_service_1.default({
             workspace_id,
             public_key,
             user_id,
             token,
             env_type,
+            redis_client
         });
         this.inputService = new inputs_service_1.default();
+        this.requestTime = 0;
+        this.totalRequests = 0;
         this.processingOutput = {
             success: [],
             failure: [],
@@ -110,7 +113,6 @@ class ProcessorService {
         if (redis_client) {
             this.redisClient = redis_client;
         }
-        this.productApi = new productsApi_service_1.ProductsApiService(env_type);
     }
     async generateSession(payload) {
         try {
@@ -274,7 +276,6 @@ class ProcessorService {
     }
     async processQuota(data) {
         this.start = Date.now();
-        this.input = data;
         const { product: product_tag, env, input, tag: quota_tag } = data;
         try {
             this.productTag = product_tag;
@@ -296,6 +297,9 @@ class ProcessorService {
                 data: input,
             };
             this.quota = this.fetchQuota(quota_tag, additional_logs);
+            if (!this.quota) {
+                throw new Error(`Quota ${quota_tag} not found`);
+            }
             this.logService.setFeatureId(this.quota._id);
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Process quota initiated', data: Object.assign(Object.assign({}, data), { input: (0, processor_utils_1.anonymizeObject)(data.input) }), status: types_1.LogEventStatus.PROCESSING }));
             this.processEnv = this.fetchEnv(env, additional_logs);
@@ -307,32 +311,91 @@ class ProcessorService {
             this.validateJSONFeatureInput(input, quotaInput, additional_logs);
             // split processes
             //this.sequenceLevels = this.splitSequenceIntoLevels(sequence, additional_logs);
-            return await this.runQuotaOptions(options, additional_logs);
+            return await this.runQuotaOptions(options, input, additional_logs);
         }
         catch (e) {
             this.end = Date.now();
-            this.logService.publish();
+            await this.logService.publish();
             throw e;
         }
     }
-    async runQuotaOptions(options, additional_logs) {
-        const quotaManager = new quota_service_1.default(options);
+    async runQuotaOptions(options, input, additional_logs) {
+        const quotaManager = new quota_service_1.default(options, this.redisClient);
         const getNextProvider = quotaManager.getNextProvider();
-        await this.processEvent({
+        const quotaInput = await this.mapQuotaFallbackInput(getNextProvider.input, input, getNextProvider.app);
+        return await this.processEvent({
+            app: getNextProvider.app,
             type: getNextProvider.type,
             event: getNextProvider.event,
-            input: getNextProvider.input,
+            input: quotaInput,
             retries: getNextProvider.retries,
             allow_fail: false
         });
     }
-    async runFallbackOptions(options, additional_logs) {
-        const fallbackManager = new fallback_service_1.default(options);
-        await fallbackManager.executeActionWithFallback(this.processEvent);
+    async mapQuotaFallbackInput(providerInput, quotaInput, app) {
+        if (Array.isArray(providerInput)) {
+            return providerInput.map(async (item) => await this.mapQuotaFallbackInput(item, quotaInput, app));
+        }
+        else if (providerInput && typeof providerInput === 'object') {
+            const result = {};
+            for (const key in providerInput) {
+                result[key] = await this.mapQuotaFallbackInput(providerInput[key], quotaInput, app);
+            }
+            return result;
+        }
+        else {
+            // Base case: primitive value
+            return await this.overrideQuotaFallbackInput(providerInput, quotaInput, app);
+        }
+    }
+    async overrideQuotaFallbackInput(value, quotaInput, app) {
+        if (value.startsWith('$Input{')) {
+            return quotaInput[(0, string_utils_1.extractStages)(value)[0]];
+        }
+        else if (value.startsWith('$Auth')) {
+            return await this.generateAuthValue((0, string_utils_1.extractStages)(value), app, [], {});
+        }
+        else if (value.startsWith('$')) {
+            this.input = Object.assign(Object.assign({}, this.input), { input: quotaInput });
+            return await this.generateOperatorValues(value, '', {});
+        }
+        else {
+            return value;
+        }
+    }
+    async runFallbackOptions(options, input, additional_logs) {
+        return await this.executeActionWithFallback(options, input);
+    }
+    async executeActionWithFallback(providers, input) {
+        let index = 0;
+        for (const provider of providers) {
+            try {
+                const payload = {
+                    app: provider.app,
+                    type: provider.type,
+                    event: provider.event,
+                    input: await this.mapQuotaFallbackInput(provider.input, input, provider.app),
+                    retries: provider.retries,
+                    allow_fail: false
+                };
+                const result = await this.processEvent(payload);
+                if (result.status === types_1.LogEventStatus.FAIL) {
+                    throw new Error(result.errors);
+                }
+                index++;
+            }
+            catch (error) {
+                console.log(error);
+                if (index > 0) {
+                    throw error;
+                }
+                console.warn(`${provider.app || provider.event} failed: ${error}`);
+            }
+        }
+        //throw new Error("All providers failed.");
     }
     async processFallback(data) {
         this.start = Date.now();
-        this.input = data;
         const { product: product_tag, env, input, tag: fallback_tag } = data;
         try {
             this.productTag = product_tag;
@@ -349,12 +412,15 @@ class ProcessorService {
                 product_id: this.productId,
                 workspace_id: this.workspace_id,
                 env,
-                type: types_1.LogEventTypes.QUOTA,
+                type: types_1.LogEventTypes.FALLBACK,
                 process_id,
                 data: input,
             };
             this.fallback = this.fetchFallback(fallback_tag, additional_logs);
-            this.logService.setFeatureId(this.quota._id);
+            if (!this.fallback) {
+                throw new Error(`Fallback "${fallback_tag}" not found`);
+            }
+            this.logService.setFeatureId(this.fallback._id);
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Process fallback initiated', data: Object.assign(Object.assign({}, data), { input: (0, processor_utils_1.anonymizeObject)(data.input) }), status: types_1.LogEventStatus.PROCESSING }));
             this.processEnv = this.fetchEnv(env, additional_logs);
             if (!this.processEnv.active) {
@@ -365,15 +431,15 @@ class ProcessorService {
             this.validateJSONFeatureInput(input, fallbackInput, additional_logs);
             // split processes
             //this.sequenceLevels = this.splitSequenceIntoLevels(sequence, additional_logs);
-            return await this.runFallbackOptions(options, additional_logs);
+            return await this.runFallbackOptions(options, input, additional_logs);
         }
         catch (e) {
             this.end = Date.now();
-            this.logService.publish();
+            await this.logService.publish();
             throw e;
         }
     }
-    async processFeature(data) {
+    async processFeature(data, awaitResolution = false) {
         this.start = Date.now();
         this.input = data;
         const { product: product_tag, env, input, tag: feature_tag } = data;
@@ -385,12 +451,14 @@ class ProcessorService {
                 name: 'Process feature',
                 type: types_1.LogEventTypes.FEATURE,
             };
-            console.log(additional_logs);
             await this.intializeProduct(additional_logs);
             this.component = types_1.LogEventTypes.FEATURE;
             const process_id = this.process_id || (0, processor_utils_1.generateObjectId)();
             this.process_id = process_id;
             this.feature = this.fetchFeature(feature_tag, additional_logs);
+            if (!this.feature) {
+                throw new Error(`Feature "${feature_tag}" not found`);
+            }
             this.baseLogs = {
                 product_tag: this.productTag,
                 product_id: this.productId,
@@ -412,7 +480,7 @@ class ProcessorService {
             // validate feature input and log failure
             this.validateJSONFeatureInput(input, featureInput, additional_logs);
             // split processes
-            this.sequenceLevels = this.splitSequenceIntoLevels(sequence, additional_logs);
+            this.sequenceLevels = await this.splitSequenceIntoLevels(sequence, additional_logs);
             await this.processSequenceLevels(additional_logs);
             return { process_id };
             //return this.generateOutput(output as unknown as Record<string, IFeatureOutput>);
@@ -421,7 +489,7 @@ class ProcessorService {
             this.end = Date.now();
             if (this.logService) {
                 this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Process feature - failed', data: Object.assign(Object.assign({}, data), { input: (0, processor_utils_1.anonymizeObject)(data.input) }), status: types_1.LogEventStatus.PROCESSING }));
-                this.logService.publish();
+                await this.logService.publish();
                 if (passedValidation) {
                     return { process_id: this.process_id };
                 }
@@ -525,7 +593,7 @@ class ProcessorService {
             throw e;
         }
     }
-    splitSequenceIntoLevels(data, additional_logs) {
+    async splitSequenceIntoLevels(data, additional_logs) {
         try {
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Split sequence - initiated', data: {}, status: types_1.LogEventStatus.PROCESSING }));
             const levels = {};
@@ -567,7 +635,7 @@ class ProcessorService {
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Process levels - initiated', data: { levels: this.sequenceLevels }, status: types_1.LogEventStatus.PROCESSING }));
             const levelEvents = {};
             Object.entries(this.sequenceLevels).forEach(([level, sequences]) => {
-                levelEvents[parseInt(level)] = this.fetchLevelEvents(sequences);
+                levelEvents[parseInt(level)] = this.fetchLevelEvents(sequences, parseInt(level));
             });
             let previousLevelComplete = true;
             for (const level of Object.keys(levelEvents)
@@ -580,8 +648,9 @@ class ProcessorService {
                     break;
                 }
             }
-            if (previousLevelComplete) {
-                this.logService.publish();
+            if (previousLevelComplete && !this.published) {
+                this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { successful_feature_execution: true, message: 'Process feature - success', data: {}, status: types_1.LogEventStatus.SUCCESS }));
+                await this.logService.publish();
                 this.end = Date.now();
                 this.writeResult(types_1.LogEventStatus.SUCCESS);
             }
@@ -592,46 +661,62 @@ class ProcessorService {
         }
     }
     async processLevelEvents(events, additional_logs) {
-        const promises = events.map((event) => {
-            const dependants = this.fetchActionRequestDependents(event.input, additional_logs);
-            if (this.checkDependentsSuccess(dependants)) {
-                // TODO: comparison to see if all depending events are in success || dependants is empty
-                return this.processEvent(event);
-            }
-            else {
-                this.addToWaitingOutput(event, dependants);
-            }
-        });
-        return Promise.all(promises);
+        try {
+            const promises = events.map((event) => {
+                const dependants = this.fetchActionRequestDependents(event.input, additional_logs);
+                const passed = this.checkDependentsSuccess(dependants);
+                if (passed) {
+                    // TODO: comparison to see if all depending events are in success || dependants is empty
+                    return this.processEvent(event);
+                }
+                else {
+                    this.addToWaitingOutput(event, dependants);
+                }
+            });
+            return Promise.all(promises);
+        }
+        catch (e) {
+            throw e;
+        }
     }
     async processFailedEvents(additional_logs) {
-        const { failure } = this.processingOutput;
-        const promises = failure.map((failed) => {
-            if (failed.retries_left > 0 && new Date().getTime() > failed.retry_at) {
-                this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Reprocess failed events - initiated', data: Object.assign({}, failed), status: types_1.LogEventStatus.PROCESSING }));
-                return this.processEvent(failed.event); // process events should also take care of this.processingOutput
-            }
-            if (failed.retries_left === 0 && !failed.allow_fail) {
-                this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Reprocess failed events - failed', data: Object.assign(Object.assign({}, failed), { reason: 'Ran out of Retries' }), status: types_1.LogEventStatus.FAIL }));
-                throw new Error(`Event ${failed.event.event} failed in sequence ${failed.event.sequence_tag}, ran out of retries and the feature cannot run without it succeeding`);
-            }
-        });
-        Promise.all(promises);
+        try {
+            const { failure } = this.processingOutput;
+            const promises = failure.map((failed) => {
+                if (failed.retries_left > 0 && new Date().getTime() > failed.retry_at) {
+                    this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Reprocess failed events - initiated', data: Object.assign({}, failed), status: types_1.LogEventStatus.PROCESSING }));
+                    return this.processEvent(failed.event); // process events should also take care of this.processingOutput
+                }
+                if (failed.retries_left === 0 && !failed.allow_fail) {
+                    this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Reprocess failed events - failed', data: Object.assign(Object.assign({}, failed), { reason: 'Ran out of Retries' }), status: types_1.LogEventStatus.FAIL }));
+                    throw new Error(`Event ${failed.event.event} failed in sequence ${failed.event.sequence_tag}, ran out of retries and the feature cannot run without it succeeding`);
+                }
+            });
+            Promise.all(promises);
+        }
+        catch (e) {
+            throw e;
+        }
     }
     async processWaitingEvents(additional_logs) {
-        const { waiting } = this.processingOutput;
-        const promises = waiting.map((waiting) => {
-            const { dependants } = waiting;
-            if (this.checkDependentsSuccess(dependants)) {
-                // TODO: comparison to see if all depending events are in success || dependants is empty
-                this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Reprocess waiting events - initiated', data: Object.assign({}, waiting), status: types_1.LogEventStatus.PROCESSING }));
-                return this.processEvent(waiting.event);
-            }
-            else {
-                this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Reprocess waiting events - waiting', data: Object.assign({}, waiting), status: types_1.LogEventStatus.WAITING }));
-            }
-        });
-        return Promise.all(promises);
+        try {
+            const { waiting } = this.processingOutput;
+            const promises = waiting.map((waiting) => {
+                const { dependants } = waiting;
+                if (this.checkDependentsSuccess(dependants)) {
+                    // TODO: comparison to see if all depending events are in success || dependants is empty
+                    this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Reprocess waiting events - initiated', data: Object.assign({}, waiting), status: types_1.LogEventStatus.PROCESSING }));
+                    return this.processEvent(waiting.event);
+                }
+                else {
+                    this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Reprocess waiting events - waiting', data: Object.assign({}, waiting), status: types_1.LogEventStatus.WAITING }));
+                }
+            });
+            return Promise.all(promises);
+        }
+        catch (e) {
+            throw e;
+        }
     }
     checkDependentsSuccess(dependants) {
         let pass = true;
@@ -661,6 +746,12 @@ class ProcessorService {
             if (input.data) {
                 dependents.push(...this.fetchDependents(input.data, additional_logs));
             }
+            if (input.fileName) {
+                dependents.push(...this.valueStringDepsCheck(input.fileName));
+            }
+            if (input.buffer) {
+                dependents.push(...this.valueStringDepsCheck(input.buffer));
+            }
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Fetch request dependencies - success', data: { input: (0, processor_utils_1.anonymizeObject)(input), dependents }, status: types_1.LogEventStatus.SUCCESS }));
             return dependents;
         }
@@ -685,6 +776,20 @@ class ProcessorService {
                                     event_tag: stages[1],
                                 });
                             }
+                            else if (values[i].startsWith('$')) {
+                                const funcArgs = (0, functions_utils_1.extractFunctionAndArgs)(values[i]);
+                                if (funcArgs.args.length) {
+                                    funcArgs.args.map((arg) => {
+                                        if (arg.startsWith('$Sequence')) {
+                                            const stages = this.productBuilderService.extractStages(arg);
+                                            dependants.push({
+                                                sequence_tag: stages[0],
+                                                event_tag: stages[1],
+                                            });
+                                        }
+                                    });
+                                }
+                            }
                         }
                     }
                     else {
@@ -692,10 +797,7 @@ class ProcessorService {
                     }
                 }
                 else if (typeof value === 'string') {
-                    if (value.startsWith('$Sequence')) {
-                        const stages = this.productBuilderService.extractStages(value);
-                        dependants.push({ sequence_tag: stages[0], event_tag: stages[1] });
-                    }
+                    dependants.push(...this.valueStringDepsCheck(value.trim()));
                 }
             }
             return dependants;
@@ -705,6 +807,34 @@ class ProcessorService {
             throw e;
         }
     }
+    valueStringDepsCheck(value) {
+        const dependants = [];
+        if (value.startsWith('$Sequence')) {
+            const stages = this.productBuilderService.extractStages(value);
+            dependants.push({ sequence_tag: stages[0], event_tag: stages[1] });
+        }
+        else if (value.startsWith('$')) {
+            const funcArgs = (0, functions_utils_1.extractFunctionAndArgs)(value);
+            if (funcArgs && funcArgs.args.length) {
+                funcArgs.args.map((arg) => {
+                    if (arg.startsWith('$Sequence')) {
+                        const stages = this.productBuilderService.extractStages(arg);
+                        dependants.push({
+                            sequence_tag: stages[0],
+                            event_tag: stages[1],
+                        });
+                    }
+                    else {
+                        const args = arg.split(',');
+                        args.map((arg) => {
+                            dependants.push(...this.valueStringDepsCheck(arg.trim()));
+                        });
+                    }
+                });
+            }
+        }
+        return dependants;
+    }
     async constructJSONDataPayloads(object, additional_logs, samples, event, loopIndex = 0) {
         try {
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Construct JSON payloads - initiated', data: { object, samples }, status: types_1.LogEventStatus.PROCESSING }));
@@ -861,7 +991,7 @@ class ProcessorService {
             return await this.sumValues(value, app, additional_logs, sample, index, key, loopIndex);
         }
         else if (value.startsWith('$Substract(') && value.endsWith(')')) {
-            return await this.sumValues(value, app, additional_logs, sample, index, key, loopIndex);
+            return await this.subtractValues(value, app, additional_logs, sample, index, key, loopIndex);
         }
         else if (value.startsWith('$Pick(') && value.endsWith(')')) {
             return await this.pickValue(value, app, additional_logs, sample, index, key, loopIndex);
@@ -1253,7 +1383,7 @@ class ProcessorService {
         }
     }
     decorateValue(value, sample) {
-        if (sample.decorator) {
+        if (sample && sample.decorator) {
             if (sample.decoratorPosition &&
                 sample.decoratorPosition == types_1.DecoratorPostions.APPEND &&
                 !value.endsWith(sample.decorator)) {
@@ -1349,12 +1479,14 @@ class ProcessorService {
               return this.runFallback(event as unknown as IFallbackEvent);
             }*/
             if (event.type === types_1.FeatureEventTypes.ACTION) {
+                console.log("APP AND EVENT!!!!", event.app);
                 const additional_logs = {
-                    parent_tag: (0, string_utils_1.extractOriginAndTag)(event.app),
+                    parent_tag: event.app,
                     child_tag: event.event,
                     type: types_1.LogEventTypes.ACTION,
                     name: 'Process feature action',
                 };
+                console.log("EVENTER!!!!!!===>>>>>>", event);
                 return this.runAction(event, additional_logs);
             }
             if (event.type === types_1.FeatureEventTypes.DB_ACTION) {
@@ -1378,8 +1510,12 @@ class ProcessorService {
                 return this.runStorage(event, additional_logs);
             }
             if (event.type === types_1.FeatureEventTypes.FEATURE) {
-                // this.processFeature({})
-                // this.processFeature TODO: do some processing to get this to reuse this.processFeature from abov
+                const additional_logs = {
+                    parent_tag: event.event,
+                    type: types_1.LogEventTypes.FEATURE,
+                    name: 'Process sub-feature',
+                };
+                return this.runFeature(event, additional_logs);
             }
             if (event.type === types_1.FeatureEventTypes.NOTIFICATION) {
                 const [parent_tag, child_tag] = event.event.split(':');
@@ -1400,11 +1536,9 @@ class ProcessorService {
                     name: 'Process feature broker event',
                 };
                 try {
-                    console.log("GRENADYE ALASOOOO!");
                     return this.runBrokerPublish(event, additional_logs);
                 }
                 catch (e) {
-                    console.log("GRENADYE ALASOOOO!!!!");
                 }
             }
             if (event.type === types_1.FeatureEventTypes.JOB) {
@@ -1420,13 +1554,30 @@ class ProcessorService {
             throw e;
         }
     }
+    async runFeature(event, additional_logs) {
+        const freshInstance = Object.assign(Object.create(Object.getPrototypeOf(this)), this);
+        const result = await freshInstance.processFeature({
+            product: this.productTag,
+            tag: event.event,
+            env: this.processEnv.slug,
+            input: event.input,
+        }, true);
+        const output = await freshInstance.generateOutput(result.process_id);
+        if (output.status === types_1.LogEventStatus.FAIL) {
+            await this.addToFailureOutput(output.errors, event, { process_id: result.process_id }, Object.assign(Object.assign({}, additional_logs), { process_id: result.process_id }));
+        }
+        else if (output.status === types_1.LogEventStatus.SUCCESS) {
+            await this.addToSuccessOutput(event, output.data, Object.assign(Object.assign({}, additional_logs), { process_id: result.process_id }));
+        }
+        return output;
+    }
     runJob(event, additional_logs) {
         throw new Error('Method not implemented.');
     }
     async processFailedAndWaiting() { }
     async generateOutput(process_id) {
         var _a, _b, _c, _d;
-        const result = (await this.processorApiService.fetchResult(process_id, this.getUserAccess()));
+        const result = await this.processorApiService.fetchResult(process_id, this.getUserAccess());
         if (!result) {
             throw new Error(`Invalid process id ${process_id}`);
         }
@@ -1452,7 +1603,7 @@ class ProcessorService {
                 return { process_id, status: result.status, errors };
             }
         }
-        else {
+        else if (result) {
             if ((_a = result.result.success[0]) === null || _a === void 0 ? void 0 : _a.output) {
                 return { process_id, status: result.status, data: (_b = result.result.success[0]) === null || _b === void 0 ? void 0 : _b.output };
             }
@@ -1492,7 +1643,7 @@ class ProcessorService {
             // validate feature input and log failure
             this.validateJSONFeatureInput(result.input.input, featureInput, additional_logs);
             // split processes
-            this.sequenceLevels = this.splitSequenceIntoLevels(sequence, additional_logs);
+            this.sequenceLevels = await this.splitSequenceIntoLevels(sequence, additional_logs);
             await this.processSequenceLevels(additional_logs);
         }
         else {
@@ -1534,7 +1685,7 @@ class ProcessorService {
             // validate feature input and log failure
             this.validateJSONFeatureInput(result.input.input, featureInput, additional_logs);
             // split processes
-            this.sequenceLevels = this.splitSequenceIntoLevels(sequence, additional_logs);
+            this.sequenceLevels = await this.splitSequenceIntoLevels(sequence, additional_logs);
             await this.processSequenceLevels(additional_logs);
         }
         else {
@@ -1547,19 +1698,19 @@ class ProcessorService {
             await this.processEvent(event);
             this.end = Date.now();
             this.writeResult(types_1.LogEventStatus.SUCCESS);
-            this.logService.publish();
+            await this.logService.publish();
         }
         return { process_id };
     }
-    fetchLevelEvents(level) {
+    fetchLevelEvents(sequence, level) {
         const events = [];
-        for (let i = 0; i < level.length; i++) {
-            events.push(...this.appendSequenceDataToLevelEvents(level[i]));
+        for (let i = 0; i < sequence.length; i++) {
+            events.push(...this.appendSequenceDataToLevelEvents(sequence[i], level));
         }
         return events;
     }
-    appendSequenceDataToLevelEvents(sequence) {
-        const { events, level, tag } = sequence;
+    appendSequenceDataToLevelEvents(sequence, level) {
+        const { events, tag } = sequence;
         for (let i = 0; i < events.length; i++) {
             events[i].sequence_level = level;
             events[i].sequence_tag = tag;
@@ -1688,15 +1839,13 @@ class ProcessorService {
         // generate indexes
         return (0, processor_utils_1.generateIndexes)(operator, iter, init, valueValue);
     }
-    async runAction(event, additional_logs, returnValue = false) {
+    async runAction(event, additional_logs, returnValue = true) {
         try {
-            console.log("RUNNING ACTION!!!!!");
             const { event: action_tag, app: access_tag, condition, cache: cache_tag } = event;
             let indexes = [];
             if (condition &&
                 condition.type === types_1.Conditions.CHECK &&
                 (await this.processConditionalCheck(event, additional_logs))) {
-                console.log("RUNNING ACTION SKIPPED!!!!!");
                 // if it fails, it would add to skipped queue
                 return;
             }
@@ -1704,6 +1853,7 @@ class ProcessorService {
                 indexes = await this.extractLoopIndexes(event, additional_logs);
             }
             let app = await this.fetchThirdPartyApp(access_tag);
+            console.log("RUNNING ACTION APP ====>>>>", app.envs, access_tag);
             const { actions, envs: appEnvs, retries, workspace_id: recipient_workspace_id, active } = app;
             const productApp = this.productBuilderService.fetchApp(access_tag);
             const { envs: productEnvs, version } = productApp;
@@ -1721,6 +1871,7 @@ class ProcessorService {
                 throw new Error(`Action ${action_tag} not found in ${access_tag}`);
             }
             const { query, headers, body, params, request_type, method, base_url, resource } = action;
+            console.log("BASE URLS!!!!!!", base_url, env.base_url);
             let request_base_url = base_url;
             if (env.base_url) {
                 request_base_url = env.base_url;
@@ -1754,9 +1905,12 @@ class ProcessorService {
                     }
                 }
             }
-            if (request_type === types_1.DataFormats.JSON || !request_type) {
+            if (request_type === types_1.DataFormats.JSON || request_type === types_1.DataFormats.URLENCODED || !request_type) {
                 if (indexes.length == 0) {
                     payloads = await this.constructJSONDataPayloads(event.input, additional_logs, samples, event);
+                    if (request_type === types_1.DataFormats.URLENCODED) {
+                        payloads.body = (0, processor_utils_1.toFormUrlEncoded)(payloads.body);
+                    }
                     additional_logs.recipient_workspace_id = recipient_workspace_id;
                     result = await this.processRequest({ request_base_url, resource, method, env, payloads, app_id: app._id }, event, retries, additional_logs, returnValue);
                 }
@@ -1802,6 +1956,8 @@ class ProcessorService {
         try {
             const results = await this.sendActionRequest(request_base_url, resource, payloads, method, env.slug);
             const end = Date.now();
+            this.requestTime += end - start;
+            this.totalRequests += 1;
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Process http request - success', successful_execution: true, data: { response: (0, processor_utils_1.anonymizeObject)(results) }, status: types_1.LogEventStatus.SUCCESS, app_id, action: event.event, start,
                 end }));
             await this.addToSuccessOutput(event, results, additional_logs);
@@ -1814,10 +1970,12 @@ class ProcessorService {
         }
         catch (e) {
             const end = Date.now();
+            this.requestTime += end - start;
+            this.totalRequests += 1;
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Process http request - failed', failed_execution: true, data: { e }, status: types_1.LogEventStatus.FAIL, app_id, action: event.event, start,
                 end }));
             try {
-                const value = this.addToFailureOutput(e, event, {
+                const value = await this.addToFailureOutput(e, event, {
                     request_base_url,
                     resource,
                     method,
@@ -1858,7 +2016,7 @@ class ProcessorService {
         }
         // addToSkippedOutput()
     }
-    addToFailureOutput(e, event, payload, additional_logs, policy = {}) {
+    async addToFailureOutput(e, event, payload, additional_logs, policy = {}) {
         try {
             const exists = this.processingOutput.failure.findIndex((item) => {
                 if (item.event.sequence_tag) {
@@ -1917,11 +2075,16 @@ class ProcessorService {
                 }, retry_at);
             }
             if (allow_fail === false && retries_left === 0) {
+                if (this.feature) {
+                    additional_logs.failed_feature_execution = true;
+                }
                 this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Ran out of retries - failed', data: Object.assign(Object.assign({}, output), { payload: (0, processor_utils_1.anonymizeObject)(output.payload) }), status: types_1.LogEventStatus.FAIL }));
                 //throw new Error("Run out of retries")
                 this.end = Date.now();
                 this.writeResult(types_1.LogEventStatus.FAIL);
-                this.logService.publish();
+                await this.logService.publish();
+                this.published = true;
+                //throw new Error("Terminate Process")
             }
             return output;
         }
@@ -1988,20 +2151,24 @@ class ProcessorService {
                 }
             }
             const authHeaders = headers;
+            if (!base_url.endsWith('/') && !resource.startsWith('/')) {
+                base_url = base_url + '/';
+            }
             const request = {
                 url: resource,
                 method,
+                baseURL: base_url,
                 data: body,
                 params: query,
                 headers: authHeaders,
+                timeout: 15000,
+                withCredentials: false
             };
-            if (!base_url.endsWith('/') && !resource.startsWith('/')) {
-                base_url = base_url + '/';
-            }
-            const response = await (0, http_client_1.default)(base_url).request(request);
+            const response = await axios_1.default.request(request);
             return response.data;
         }
         catch (e) {
+            console.error("LOG ERROR", e);
             throw e;
         }
     }
@@ -2046,13 +2213,13 @@ class ProcessorService {
             this.end = Date.now();
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Storing file - success', data: { input: this.clone, result }, status: types_1.LogEventStatus.SUCCESS }));
             this.writeResult(types_1.LogEventStatus.SUCCESS);
-            this.logService.publish();
+            await this.logService.publish();
             return result;
         }
         catch (e) {
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Storing file - failed', data: { e }, status: types_1.LogEventStatus.FAIL }));
             this.end = Date.now();
-            this.logService.publish();
+            await this.logService.publish();
             return { process_id: this.process_id };
         }
     }
@@ -2095,13 +2262,13 @@ class ProcessorService {
             this.end = Date.now();
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Subscribe to topic - success', data: { input: this.clone, result }, status: types_1.LogEventStatus.SUCCESS }));
             this.writeResult(types_1.LogEventStatus.SUCCESS);
-            this.logService.publish();
+            await this.logService.publish();
             return result;
         }
         catch (e) {
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Subscribe to topic - failed', data: { e }, status: types_1.LogEventStatus.FAIL }));
             this.end = Date.now();
-            this.logService.publish();
+            await this.logService.publish();
             return { process_id: this.process_id };
         }
     }
@@ -2118,7 +2285,6 @@ class ProcessorService {
         };
         try {
             this.validateActionDataMappingInput(data.input, types_1.FeatureEventTypes.PUBLISH);
-            console.log("JAPANESE MIRRORS", data.input);
             this.start = Date.now();
             // clone
             this.clone = (0, processor_utils_1.structuredClone)(data.input);
@@ -2132,7 +2298,6 @@ class ProcessorService {
             this.process_id = process_id;
             const productEnv = this.fetchEnv(data.env, additional_logs);
             this.processEnv = productEnv;
-            console.log("JAPANESE MIRRORS 2", productEnv);
             if (!productEnv.active) {
                 throw new Error(`Environment ${data.env} is not active`);
             }
@@ -2150,14 +2315,13 @@ class ProcessorService {
             this.end = Date.now();
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Publishing to topic - success', data: { input: (0, processor_utils_1.anonymizeObject)(this.clone), result }, status: types_1.LogEventStatus.SUCCESS }));
             this.writeResult(types_1.LogEventStatus.SUCCESS);
-            this.logService.publish();
+            await this.logService.publish();
             return result;
         }
         catch (e) {
-            console.log(e);
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Publishing to topic - failed', data: { e }, status: types_1.LogEventStatus.FAIL }));
             this.end = Date.now();
-            this.logService.publish();
+            await this.logService.publish();
             return { process_id: this.process_id };
         }
     }
@@ -2170,7 +2334,6 @@ class ProcessorService {
             body: payload.body,
             data: (0, processor_utils_1.convertStringToObject)(payload.data),
         };
-        console.log("MESSAGE!!!", message);
         try {
             await (0, expo_client_1.default)().post('', message, (0, processor_utils_1.generateAxiosConfig)());
         }
@@ -2189,7 +2352,6 @@ class ProcessorService {
         try {
             const admin = require('firebase-admin');
             const serviceAccount = credentials;
-            console.log("MESSAGE", message);
             admin.initializeApp({
                 credential: admin.credential.cert(serviceAccount),
             });
@@ -2320,11 +2482,6 @@ class ProcessorService {
                 const url = new URL(callbacks.url);
                 try {
                     this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Send callback - initiated', data: {}, status: types_1.LogEventStatus.PROCESSING }));
-                    console.log("CALLBACK!!!!", {
-                        url,
-                        payload,
-                        method: callbacks.method
-                    });
                     await this.sendActionRequest(url.origin, url.pathname, payload, callbacks.method, '');
                     this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { name: 'Send callback - success', data: {}, status: types_1.LogEventStatus.SUCCESS }));
                 }
@@ -2339,7 +2496,6 @@ class ProcessorService {
                     const SmsClient = await (0, sms_repo_1.loadSMSClient)();
                     const smsClient = new SmsClient(smses);
                     this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { name: 'Send sms - initiated', data: { message: input.sms.body, config: (0, processor_utils_1.anonymizeObject)(smses) }, status: types_1.LogEventStatus.SUCCESS }));
-                    console.log("SMS!!!!", input.sms, smses);
                     const res = await smsClient.sendMessage(input.sms.body, input.sms.recipients);
                     this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { name: 'Send sms - success', data: res, status: types_1.LogEventStatus.SUCCESS }));
                 }
@@ -2350,9 +2506,8 @@ class ProcessorService {
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { successful_execution: true, message: 'Attempt notification - success', data: notification, status: types_1.LogEventStatus.SUCCESS }));
         }
         catch (e) {
-            console.log(e);
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { failed_execution: true, message: 'Attempt notification - failed', data: { e: e.toString() }, status: types_1.LogEventStatus.FAIL }));
-            //this.logService.publish();
+            //await this.logService.publish();
             throw e;
         }
     }
@@ -2431,9 +2586,8 @@ class ProcessorService {
             }
         }
         catch (e) {
-            console.log(e);
             this.logService.add(Object.assign(Object.assign({}, this.baseLogs), { failed_execution: true, message: 'Attempt migration - failed', data: e, status: types_1.LogEventStatus.FAIL }));
-            this.logService.publish();
+            await this.logService.publish();
         }
     }
     /*async runFunction(data: IFeatureEvent, additional_logs: Partial<ILogData>): Promise<any> {
@@ -2526,7 +2680,7 @@ class ProcessorService {
           data: e,
           status: LogEventStatus.FAIL,
         });
-        this.logService.publish();
+        await this.logService.publish();
       }
     }*/
     async runStorage(data, additional_logs = {}) {
@@ -2566,7 +2720,6 @@ class ProcessorService {
             input.buffer = input.buffer ? await this.generateStringValues(input.buffer, '', additional_logs, []) : undefined;
             input.fileName = input.fileName ? await this.generateStringValues(input.fileName, '', additional_logs, []) : undefined;
             input.mimeType = input.mimeType ? await this.generateStringValues(input.mimeType, '', additional_logs, []) : undefined;
-            console.log("CHINYERE", input.buffer, input.fileName);
             result = await this.processStorageRequest(data, input, storageEnv, additional_logs);
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { successful_execution: true, message: 'Store file - success', data: { result }, status: types_1.LogEventStatus.PROCESSING }));
             if (cache_tag && this.redisClient) {
@@ -2589,7 +2742,6 @@ class ProcessorService {
             return result;
         }
         catch (e) {
-            console.log(e);
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { failed_execution: true, message: 'Attempt storage - failed', data: { e }, status: types_1.LogEventStatus.FAIL }));
             throw e;
         }
@@ -2668,8 +2820,6 @@ class ProcessorService {
                 if (!MongoDBHandler) {
                     throw new Error(`Running in browser, mongo handler not loaded.`);
                 }
-                console.log("PRIVATE KEY!!!", this.productBuilderService.fetchProduct().private_key);
-                console.log("AMAZONS", (0, processor_utils_1.decrypt)(databaseEnv.connection_url, this.productBuilderService.fetchProduct().private_key));
                 const mongoHandler = new MongoDBHandler((0, processor_utils_1.decrypt)(databaseEnv.connection_url, this.productBuilderService.fetchProduct().private_key));
                 if (databaseAction.type === types_1.DatabaseActionTypes.UPDATE) {
                     const filterTemplate = typeof databaseAction.filterTemplate === 'string'
@@ -2691,7 +2841,7 @@ class ProcessorService {
                     result = await pgHandler[databaseAction.type](databaseAction.data, template, input.data);
                 }
             }
-            //this.logService.publish();
+            //await this.logService.publish();
             // const result = this.processDBRequest(db_action, input, database_tag, databaseEnv, action_tag, additional_logs);
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { successful_execution: true, message: 'Run database action query - success', data: { result: (0, processor_utils_1.anonymizeObject)(result) }, status: types_1.LogEventStatus.SUCCESS }));
             await this.addToSuccessOutput(db_action, result, additional_logs);
@@ -2817,7 +2967,6 @@ class ProcessorService {
         }
         catch (e) {
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { failed_execution: true, message: 'Attempt publish to broker topic - failed', data: { e }, status: types_1.LogEventStatus.FAIL }));
-            console.log("JERMOOOOO!!!");
             throw e;
         }
     }
@@ -2854,7 +3003,7 @@ class ProcessorService {
         }
         catch (e) {
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { failed_execution: true, message: 'Run storage request - failed', data: { e }, status: types_1.LogEventStatus.FAIL }));
-            const value = this.addToFailureOutput(e, data, { storageEnv }, additional_logs);
+            const value = await this.addToFailureOutput(e, data, { storageEnv }, additional_logs);
             throw e;
         }
     }
@@ -2877,7 +3026,7 @@ class ProcessorService {
         }
         catch (e) {
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { failed_execution: true, message: 'Running database query - failed', data: { payload: (0, processor_utils_1.anonymizeObject)(input), e }, status: types_1.LogEventStatus.FAIL }));
-            const value = this.addToFailureOutput(e, db_action, {
+            const value = await this.addToFailureOutput(e, db_action, {
                 db_action,
                 input,
                 database_tag,
@@ -2962,18 +3111,17 @@ class ProcessorService {
                 retries: retries || 0,
                 allow_fail: false,
             }, additional_logs, true);
-            console.log("RESULT ===>>>>", result);
             this.end = Date.now();
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Execute action - success', data: { input: (0, processor_utils_1.anonymizeObject)(input), result: (0, processor_utils_1.anonymizeObject)(result) }, status: types_1.LogEventStatus.SUCCESS }));
             this.writeResult(types_1.LogEventStatus.SUCCESS);
-            this.logService.publish();
+            await this.logService.publish();
             return result;
         }
         catch (e) {
             console.log(e);
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Execute action - failed', data: { e }, status: types_1.LogEventStatus.FAIL }));
             this.end = Date.now();
-            this.logService.publish();
+            await this.logService.publish();
             return { process_id: this.process_id };
         }
     }
@@ -3021,13 +3169,13 @@ class ProcessorService {
             this.end = Date.now();
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Execute database action', data: { input: (0, processor_utils_1.anonymizeObject)(input), result: (0, processor_utils_1.anonymizeObject)(result) }, status: types_1.LogEventStatus.SUCCESS }));
             this.writeResult(types_1.LogEventStatus.SUCCESS);
-            this.logService.publish();
+            await this.logService.publish();
             return result;
         }
         catch (e) {
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Execute database action', data: { e }, status: types_1.LogEventStatus.FAIL }));
             this.end = Date.now();
-            this.logService.publish();
+            await this.logService.publish();
             return { process_id: this.process_id };
         }
     }
@@ -3076,13 +3224,13 @@ class ProcessorService {
             this.end = Date.now();
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Send notification - success', data: { input: (0, processor_utils_1.anonymizeObject)(input), result: (0, processor_utils_1.anonymizeObject)(result) }, status: types_1.LogEventStatus.SUCCESS }));
             this.writeResult(types_1.LogEventStatus.SUCCESS);
-            this.logService.publish();
+            await this.logService.publish();
             return { process_id };
         }
         catch (e) {
             this.logService.add(Object.assign(Object.assign(Object.assign({}, this.baseLogs), additional_logs), { message: 'Send notification - failed', data: { e }, status: types_1.LogEventStatus.FAIL }));
             this.end = Date.now();
-            this.logService.publish();
+            await this.logService.publish();
             return { process_id: this.process_id };
         }
     }