@budibase/server 2.6.19-alpha.5 → 2.6.19-alpha.51

package/dist/threads/automation.js

@@ -1,450 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.removeStalled = exports.execute = void 0;
-const utils_1 = __importDefault(require("./utils"));
-utils_1.default.threadSetup();
-const utils_2 = require("../automations/utils");
-const actions = __importStar(require("../automations/actions"));
-const automationUtils = __importStar(require("../automations/automationUtils"));
-const AutomationEmitter_1 = __importDefault(require("../events/AutomationEmitter"));
-const utils_3 = require("../db/utils");
-const triggerInfo_1 = require("../automations/triggerInfo");
-const constants_1 = require("../constants");
-const logging_1 = require("../automations/logging");
-const types_1 = require("@budibase/types");
-const backend_core_1 = require("@budibase/backend-core");
-const string_templates_1 = require("@budibase/string-templates");
-const fp_1 = require("lodash/fp");
-const sdkUtils = __importStar(require("../sdk/utils"));
-const environment_1 = __importDefault(require("../environment"));
-const FILTER_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.FILTER.stepId;
-const LOOP_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.LOOP.stepId;
-const CRON_STEP_ID = triggerInfo_1.definitions.CRON.stepId;
-const STOPPED_STATUS = { success: true, status: types_1.AutomationStatus.STOPPED };
-function getLoopIterations(loopStep, input) {
-    const binding = automationUtils.typecastForLooping(loopStep, input);
-    if (!binding) {
-        return 0;
-    }
-    if (Array.isArray(binding)) {
-        return binding.length;
-    }
-    if (typeof binding === "string") {
-        return automationUtils.stringSplit(binding).length;
-    }
-    return 0;
-}
-/**
- * The automation orchestrator is a class responsible for executing automations.
- * It handles the context of the automation and makes sure each step gets the correct
- * inputs and handles any outputs.
- */
-class Orchestrator {
-    constructor(job) {
-        let automation = job.data.automation;
-        let triggerOutput = job.data.event;
-        const metadata = triggerOutput.metadata;
-        this._chainCount = metadata ? metadata.automationChainCount : 0;
-        this._appId = triggerOutput.appId;
-        this._job = job;
-        const triggerStepId = automation.definition.trigger.stepId;
-        triggerOutput = this.cleanupTriggerOutputs(triggerStepId, triggerOutput);
-        // remove from context
-        delete triggerOutput.appId;
-        delete triggerOutput.metadata;
-        // step zero is never used as the template string is zero indexed for customer facing
-        this._context = { steps: [{}], trigger: triggerOutput };
-        this._automation = automation;
-        // create an emitter which has the chain count for this automation run in it, so it can block
-        // excessive chaining if required
-        this._emitter = new AutomationEmitter_1.default(this._chainCount + 1);
-        this.executionOutput = { trigger: {}, steps: [] };
-        // setup the execution output
-        const triggerId = automation.definition.trigger.id;
-        this.updateExecutionOutput(triggerId, triggerStepId, null, triggerOutput);
-    }
-    cleanupTriggerOutputs(stepId, triggerOutput) {
-        if (stepId === CRON_STEP_ID) {
-            triggerOutput.timestamp = Date.now();
-        }
-        return triggerOutput;
-    }
-    getStepFunctionality(stepId) {
-        return __awaiter(this, void 0, void 0, function* () {
-            let step = yield actions.getAction(stepId);
-            if (step == null) {
-                throw `Cannot find automation step by name ${stepId}`;
-            }
-            return step;
-        });
-    }
-    getMetadata() {
-        return __awaiter(this, void 0, void 0, function* () {
-            const metadataId = (0, utils_3.generateAutomationMetadataID)(this._automation._id);
-            const db = backend_core_1.context.getAppDB();
-            let metadata;
-            try {
-                metadata = yield db.get(metadataId);
-            }
-            catch (err) {
-                metadata = {
-                    _id: metadataId,
-                    errorCount: 0,
-                };
-            }
-            return metadata;
-        });
-    }
-    stopCron(reason) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this._job.opts.repeat) {
-                return;
-            }
-            backend_core_1.logging.logWarn(`CRON disabled reason=${reason} - ${this._appId}/${this._automation._id}`);
-            const automation = this._automation;
-            const trigger = automation.definition.trigger;
-            yield (0, utils_2.disableCronById)(this._job.id);
-            this.updateExecutionOutput(trigger.id, trigger.stepId, {}, {
-                status: types_1.AutomationStatus.STOPPED_ERROR,
-                success: false,
-            });
-            yield (0, logging_1.storeLog)(automation, this.executionOutput);
-        });
-    }
-    checkIfShouldStop(metadata) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!metadata.errorCount || !this._job.opts.repeat) {
-                return false;
-            }
-            if (metadata.errorCount >= constants_1.MAX_AUTOMATION_RECURRING_ERRORS) {
-                yield this.stopCron("errors");
-                return true;
-            }
-            return false;
-        });
-    }
-    updateMetadata(metadata) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const output = this.executionOutput, automation = this._automation;
-            if (!output || !(0, utils_2.isRecurring)(automation)) {
-                return;
-            }
-            const count = metadata.errorCount;
-            const isError = (0, utils_2.isErrorInOutput)(output);
-            // nothing to do in this scenario, escape
-            if (!count && !isError) {
-                return;
-            }
-            if (isError) {
-                metadata.errorCount = count ? count + 1 : 1;
-            }
-            else {
-                metadata.errorCount = 0;
-            }
-            const db = backend_core_1.context.getAppDB();
-            try {
-                yield db.put(metadata);
-            }
-            catch (err) {
-                backend_core_1.logging.logAlertWithInfo("Failed to write automation metadata", db.name, automation._id, err);
-            }
-        });
-    }
-    updateExecutionOutput(id, stepId, inputs, outputs) {
-        const stepObj = { id, stepId, inputs, outputs };
-        // replacing trigger when disabling CRON
-        if (stepId === CRON_STEP_ID &&
-            outputs.status === types_1.AutomationStatus.STOPPED_ERROR) {
-            this.executionOutput.trigger = stepObj;
-            this.executionOutput.steps = [stepObj];
-            return;
-        }
-        // first entry is always the trigger (constructor)
-        if (this.executionOutput.steps.length === 0 ||
-            this.executionOutput.trigger.id === id) {
-            this.executionOutput.trigger = stepObj;
-        }
-        this.executionOutput.steps.push(stepObj);
-    }
-    updateContextAndOutput(loopStepNumber, step, output, result) {
-        if (!loopStepNumber) {
-            throw new Error("No loop step number provided.");
-        }
-        this.executionOutput.steps.splice(loopStepNumber, 0, {
-            id: step.id,
-            stepId: step.stepId,
-            outputs: Object.assign(Object.assign({}, output), { success: result.success, status: result.status }),
-            inputs: step.inputs,
-        });
-        this._context.steps.splice(loopStepNumber, 0, Object.assign(Object.assign({}, output), { success: result.success, status: result.status }));
-    }
-    execute() {
-        var _a;
-        return __awaiter(this, void 0, void 0, function* () {
-            // this will retrieve from context created at start of thread
-            this._context.env = yield sdkUtils.getEnvironmentVariables();
-            let automation = this._automation;
-            let stopped = false;
-            let loopStep = undefined;
-            let stepCount = 0;
-            let loopStepNumber = undefined;
-            let loopSteps = [];
-            let metadata;
-            let wasLoopStep = false;
-            // check if this is a recurring automation,
-            if ((0, utils_3.isProdAppID)(this._appId) && (0, utils_2.isRecurring)(automation)) {
-                metadata = yield this.getMetadata();
-                const shouldStop = yield this.checkIfShouldStop(metadata);
-                if (shouldStop) {
-                    return;
-                }
-            }
-            for (let step of automation.definition.steps) {
-                stepCount++;
-                let input, iterations = 1, iterationCount = 0;
-                if (step.stepId === LOOP_STEP_ID) {
-                    loopStep = step;
-                    loopStepNumber = stepCount;
-                    continue;
-                }
-                if (loopStep) {
-                    input = yield (0, string_templates_1.processObject)(loopStep.inputs, this._context);
-                    iterations = getLoopIterations(loopStep, input);
-                }
-                for (let index = 0; index < iterations; index++) {
-                    let originalStepInput = (0, fp_1.cloneDeep)(step.inputs);
-                    // Handle if the user has set a max iteration count or if it reaches the max limit set by us
-                    if (loopStep && input.binding) {
-                        let newInput = yield (0, string_templates_1.processObject)(loopStep.inputs, (0, fp_1.cloneDeep)(this._context));
-                        let tempOutput = { items: loopSteps, iterations: iterationCount };
-                        try {
-                            newInput.binding = automationUtils.typecastForLooping(loopStep, newInput);
-                        }
-                        catch (err) {
-                            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
-                                status: constants_1.AutomationErrors.INCORRECT_TYPE,
-                                success: false,
-                            });
-                            loopSteps = undefined;
-                            loopStep = undefined;
-                            break;
-                        }
-                        let item = [];
-                        if (typeof loopStep.inputs.binding === "string" &&
-                            loopStep.inputs.option === "String") {
-                            item = automationUtils.stringSplit(newInput.binding);
-                        }
-                        else if (Array.isArray(loopStep.inputs.binding)) {
-                            item = loopStep.inputs.binding;
-                        }
-                        this._context.steps[loopStepNumber] = {
-                            currentItem: item[index],
-                        };
-                        // The "Loop" binding in the front end is "fake", so replace it here so the context can understand it
-                        // Pretty hacky because we need to account for the row object
-                        for (let [key, value] of Object.entries(originalStepInput)) {
-                            if (typeof value === "object") {
-                                for (let [innerKey, innerValue] of Object.entries(originalStepInput[key])) {
-                                    if (typeof innerValue === "string") {
-                                        originalStepInput[key][innerKey] =
-                                            automationUtils.substituteLoopStep(innerValue, `steps.${loopStepNumber}`);
-                                    }
-                                    else if (typeof value === "object") {
-                                        for (let [innerObject, innerValue] of Object.entries(originalStepInput[key][innerKey])) {
-                                            originalStepInput[key][innerKey][innerObject] =
-                                                automationUtils.substituteLoopStep(innerValue, `steps.${loopStepNumber}`);
-                                        }
-                                    }
-                                }
-                            }
-                            else {
-                                if (typeof value === "string") {
-                                    originalStepInput[key] = automationUtils.substituteLoopStep(value, `steps.${loopStepNumber}`);
-                                }
-                            }
-                        }
-                        if (index === environment_1.default.AUTOMATION_MAX_ITERATIONS ||
-                            index === parseInt(loopStep.inputs.iterations)) {
-                            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
-                                status: constants_1.AutomationErrors.MAX_ITERATIONS,
-                                success: true,
-                            });
-                            loopSteps = undefined;
-                            loopStep = undefined;
-                            break;
-                        }
-                        let isFailure = false;
-                        const currentItem = (_a = this._context.steps[loopStepNumber]) === null || _a === void 0 ? void 0 : _a.currentItem;
-                        if (currentItem && typeof currentItem === "object") {
-                            isFailure = Object.keys(currentItem).some(value => {
-                                return currentItem[value] === (loopStep === null || loopStep === void 0 ? void 0 : loopStep.inputs.failure);
-                            });
-                        }
-                        else {
-                            isFailure = currentItem && currentItem === loopStep.inputs.failure;
-                        }
-                        if (isFailure) {
-                            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
-                                status: constants_1.AutomationErrors.FAILURE_CONDITION,
-                                success: false,
-                            });
-                            loopSteps = undefined;
-                            loopStep = undefined;
-                            break;
-                        }
-                    }
-                    // execution stopped, record state for that
-                    if (stopped) {
-                        this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS);
-                        continue;
-                    }
-                    // If it's a loop step, we need to manually add the bindings to the context
-                    let stepFn = yield this.getStepFunctionality(step.stepId);
-                    let inputs = yield (0, string_templates_1.processObject)(originalStepInput, this._context);
-                    inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs);
-                    try {
-                        // appId is always passed
-                        const outputs = yield stepFn({
-                            inputs: inputs,
-                            appId: this._appId,
-                            emitter: this._emitter,
-                            context: this._context,
-                        });
-                        this._context.steps[stepCount] = outputs;
-                        // if filter causes us to stop execution don't break the loop, set a var
-                        // so that we can finish iterating through the steps and record that it stopped
-                        if (step.stepId === FILTER_STEP_ID && !outputs.result) {
-                            stopped = true;
-                            this.updateExecutionOutput(step.id, step.stepId, step.inputs, Object.assign(Object.assign({}, outputs), STOPPED_STATUS));
-                            continue;
-                        }
-                        if (loopStep && loopSteps) {
-                            loopSteps.push(outputs);
-                        }
-                        else {
-                            this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs);
-                        }
-                    }
-                    catch (err) {
-                        console.error(`Automation error - ${step.stepId} - ${err}`);
-                        return err;
-                    }
-                    if (loopStep) {
-                        iterationCount++;
-                        if (index === iterations - 1) {
-                            loopStep = undefined;
-                            this._context.steps.splice(loopStepNumber, 1);
-                            break;
-                        }
-                    }
-                }
-                if (loopStep && iterations === 0) {
-                    loopStep = undefined;
-                    this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
-                        id: step.id,
-                        stepId: step.stepId,
-                        outputs: { status: types_1.AutomationStatus.NO_ITERATIONS, success: true },
-                        inputs: {},
-                    });
-                    this._context.steps.splice(loopStepNumber, 1);
-                    iterations = 1;
-                }
-                // Delete the step after the loop step as it's irrelevant, since information is included
-                // in the loop step
-                if (wasLoopStep && !loopStep) {
-                    this._context.steps.splice(loopStepNumber + 1, 1);
-                    wasLoopStep = false;
-                }
-                if (loopSteps && loopSteps.length) {
-                    let tempOutput = {
-                        success: true,
-                        items: loopSteps,
-                        iterations: iterationCount,
-                    };
-                    this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
-                        id: step.id,
-                        stepId: step.stepId,
-                        outputs: tempOutput,
-                        inputs: step.inputs,
-                    });
-                    this._context.steps[loopStepNumber] = tempOutput;
-                    wasLoopStep = true;
-                    loopSteps = [];
-                }
-            }
-            // store the logs for the automation run
-            yield (0, logging_1.storeLog)(this._automation, this.executionOutput);
-            if ((0, utils_3.isProdAppID)(this._appId) && (0, utils_2.isRecurring)(automation) && metadata) {
-                yield this.updateMetadata(metadata);
-            }
-            return this.executionOutput;
-        });
-    }
-}
-function execute(job, callback) {
-    const appId = job.data.event.appId;
-    if (!appId) {
-        throw new Error("Unable to execute, event doesn't contain app ID.");
-    }
-    return backend_core_1.context.doInAppContext(appId, () => __awaiter(this, void 0, void 0, function* () {
-        const envVars = yield sdkUtils.getEnvironmentVariables();
-        // put into automation thread for whole context
-        yield backend_core_1.context.doInEnvironmentContext(envVars, () => __awaiter(this, void 0, void 0, function* () {
-            const automationOrchestrator = new Orchestrator(job);
-            try {
-                const response = yield automationOrchestrator.execute();
-                callback(null, response);
-            }
-            catch (err) {
-                callback(err);
-            }
-        }));
-    }));
-}
-exports.execute = execute;
-const removeStalled = (job) => __awaiter(void 0, void 0, void 0, function* () {
-    const appId = job.data.event.appId;
-    if (!appId) {
-        throw new Error("Unable to execute, event doesn't contain app ID.");
-    }
-    yield backend_core_1.context.doInAppContext(appId, () => __awaiter(void 0, void 0, void 0, function* () {
-        const automationOrchestrator = new Orchestrator(job);
-        yield automationOrchestrator.stopCron("stalled");
-    }));
-});
-exports.removeStalled = removeStalled;

package/dist/threads/definitions.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/dist/threads/index.js

@@ -1,140 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Thread = exports.ThreadType = void 0;
-const worker_farm_1 = __importDefault(require("worker-farm"));
-const environment_1 = __importDefault(require("../environment"));
-exports.ThreadType = {
-    QUERY: "query",
-    AUTOMATION: "automation",
-};
-function typeToFile(type) {
-    let filename = null;
-    switch (type) {
-        case exports.ThreadType.QUERY:
-            filename = "./query";
-            break;
-        case exports.ThreadType.AUTOMATION:
-            filename = "./automation";
-            break;
-        default:
-            throw "Unknown thread type";
-    }
-    // have to use require here, to make it work with worker-farm
-    return require.resolve(filename);
-}
-class Thread {
-    constructor(type, opts = { timeoutMs: null, count: 1 }) {
-        this.type = type;
-        this.count = opts.count ? opts.count : 1;
-        this.disableThreading = this.shouldDisableThreading();
-        if (!this.disableThreading) {
-            const workerOpts = {
-                autoStart: true,
-                maxConcurrentWorkers: this.count,
-                workerOptions: {
-                    env: Object.assign(Object.assign({}, process.env), { FORKED_PROCESS: "1" }),
-                },
-            };
-            if (opts.timeoutMs) {
-                this.timeoutMs = opts.timeoutMs;
-                workerOpts.maxCallTime = opts.timeoutMs;
-            }
-            this.workers = (0, worker_farm_1.default)(workerOpts, typeToFile(type), ["execute"]);
-            Thread.workerRefs.push(this.workers);
-        }
-    }
-    shouldDisableThreading() {
-        return !!(environment_1.default.isTest() ||
-            environment_1.default.DISABLE_THREADING ||
-            this.count === 0 ||
-            environment_1.default.isInThread());
-    }
-    run(job) {
-        const timeout = this.timeoutMs;
-        return new Promise((resolve, reject) => {
-            function fire(worker) {
-                worker.execute(job, (err, response) => {
-                    if (err && err.type === "TimeoutError") {
-                        reject(new Error(`Query response time exceeded ${timeout}ms timeout.`));
-                    }
-                    else if (err) {
-                        reject(err);
-                    }
-                    else {
-                        resolve(response);
-                    }
-                });
-            }
-            // if in test then don't use threading
-            if (this.disableThreading) {
-                Promise.resolve().then(() => __importStar(require(typeToFile(this.type)))).then((thread) => {
-                    fire(thread);
-                });
-            }
-            else {
-                fire(this.workers);
-            }
-        });
-    }
-    static stopThreads() {
-        return new Promise(resolve => {
-            if (Thread.workerRefs.length === 0) {
-                resolve();
-            }
-            let count = 0;
-            function complete() {
-                count++;
-                if (count >= Thread.workerRefs.length) {
-                    resolve();
-                }
-            }
-            for (let worker of Thread.workerRefs) {
-                worker_farm_1.default.end(worker, complete);
-            }
-            Thread.workerRefs = [];
-        });
-    }
-    static shutdown() {
-        return __awaiter(this, void 0, void 0, function* () {
-            yield Thread.stopThreads();
-            console.log("Threads shutdown");
-        });
-    }
-}
-exports.Thread = Thread;
-Thread.workerRefs = [];