@budibase/server 1.2.44 → 1.2.47

package/coverage/lcov-report/utilities/statusCodes.js.html

@@ -91,7 +91,7 @@
             <div class='footer quiet pad2 space-top1 center small'>
                 Code coverage generated by
                 <a href="https://istanbul.js.org/" target="_blank" rel="noopener noreferrer">istanbul</a>
-                at Fri Aug 19 2022 14:16:53 GMT+0000 (Coordinated Universal Time)
+                at Sat Aug 20 2022 11:44:08 GMT+0000 (Coordinated Universal Time)
             </div>
         <script src="../prettify.js"></script>
         <script>

package/coverage/lcov-report/utilities/usageQuota/index.html

@@ -116,7 +116,7 @@
             <div class='footer quiet pad2 space-top1 center small'>
                 Code coverage generated by
                 <a href="https://istanbul.js.org/" target="_blank" rel="noopener noreferrer">istanbul</a>
-                at Fri Aug 19 2022 14:16:53 GMT+0000 (Coordinated Universal Time)
+                at Sat Aug 20 2022 11:44:08 GMT+0000 (Coordinated Universal Time)
             </div>
         <script src="../../prettify.js"></script>
         <script>

package/coverage/lcov-report/utilities/usageQuota/rows.js.html

@@ -307,7 +307,7 @@ exports.getUniqueRows = async appIds => {
             <div class='footer quiet pad2 space-top1 center small'>
                 Code coverage generated by
                 <a href="https://istanbul.js.org/" target="_blank" rel="noopener noreferrer">istanbul</a>
-                at Fri Aug 19 2022 14:16:53 GMT+0000 (Coordinated Universal Time)
+                at Sat Aug 20 2022 11:44:08 GMT+0000 (Coordinated Universal Time)
             </div>
         <script src="../../prettify.js"></script>
         <script>

package/coverage/lcov-report/utilities/usageQuota/usageQuoteReset.js.html

@@ -124,7 +124,7 @@ function <span class="fstat-no" title="function not covered" >resetQuotasIfRequi
             <div class='footer quiet pad2 space-top1 center small'>
                 Code coverage generated by
                 <a href="https://istanbul.js.org/" target="_blank" rel="noopener noreferrer">istanbul</a>
-                at Fri Aug 19 2022 14:16:53 GMT+0000 (Coordinated Universal Time)
+                at Sat Aug 20 2022 11:44:08 GMT+0000 (Coordinated Universal Time)
             </div>
         <script src="../../prettify.js"></script>
         <script>

package/coverage/lcov-report/utilities/users.js.html

@@ -205,7 +205,7 @@ exports.publicApiUserFix = ctx => {
             <div class='footer quiet pad2 space-top1 center small'>
                 Code coverage generated by
                 <a href="https://istanbul.js.org/" target="_blank" rel="noopener noreferrer">istanbul</a>
-                at Fri Aug 19 2022 14:16:53 GMT+0000 (Coordinated Universal Time)
+                at Sat Aug 20 2022 11:44:08 GMT+0000 (Coordinated Universal Time)
             </div>
         <script src="../prettify.js"></script>
         <script>

package/coverage/lcov-report/utilities/workerRequests.js.html

@@ -535,7 +535,7 @@ exports.getChecklist = <span class="fstat-no" title="function not covered" >as</
             <div class='footer quiet pad2 space-top1 center small'>
                 Code coverage generated by
                 <a href="https://istanbul.js.org/" target="_blank" rel="noopener noreferrer">istanbul</a>
-                at Fri Aug 19 2022 14:16:53 GMT+0000 (Coordinated Universal Time)
+                at Sat Aug 20 2022 11:44:08 GMT+0000 (Coordinated Universal Time)
             </div>
         <script src="../prettify.js"></script>
         <script>

package/coverage/lcov.info

@@ -4619,10 +4619,10 @@ BRH:21
 end_of_record
 TN:
 SF:src/automations/bullboard.js
-FN:18,cleanup
-FN:24,(anonymous_1)
-FN:29,(anonymous_2)
-FN:48,(anonymous_3)
+FN:20,cleanup
+FN:26,(anonymous_1)
+FN:31,(anonymous_2)
+FN:50,(anonymous_3)
 FNF:4
 FNH:0
 FNDA:0,cleanup
@@ -4637,41 +4637,43 @@ DA:5,45
 DA:8,45
 DA:9,45
 DA:10,45
-DA:12,45
+DA:11,45
 DA:13,45
 DA:14,45
-DA:16,45
-DA:19,0
-DA:22,45
+DA:15,45
+DA:17,45
+DA:18,45
+DA:21,0
 DA:24,45
-DA:26,0
-DA:27,0
+DA:26,45
+DA:28,0
 DA:29,0
-DA:30,0
-DA:34,0
-DA:35,0
+DA:31,0
+DA:32,0
 DA:36,0
 DA:37,0
 DA:38,0
+DA:39,0
 DA:40,0
-DA:44,0
-DA:45,0
-DA:48,45
-DA:49,0
-DA:50,0
+DA:42,0
+DA:46,0
+DA:47,0
+DA:50,45
 DA:51,0
 DA:52,0
-DA:56,45
-LF:33
-LH:16
+DA:53,0
+DA:54,0
+DA:58,45
+LF:35
+LH:18
 BRDA:5,0,0,45
 BRDA:5,0,1,0
-BRDA:13,1,0,45
-BRDA:13,1,1,45
-BRDA:26,2,0,0
-BRDA:26,2,1,0
-BRDA:49,3,0,0
-BRDA:49,3,1,0
+BRDA:14,1,0,45
+BRDA:14,1,1,45
+BRDA:28,2,0,0
+BRDA:28,2,1,0
+BRDA:51,3,0,0
+BRDA:51,3,1,0
 BRF:8
 BRH:3
 end_of_record
@@ -6380,9 +6382,9 @@ FNH:2
 FNDA:6,(anonymous_0)
 FNDA:5,(anonymous_1)
 FNDA:0,(anonymous_2)
-DA:1,3
-DA:2,3
-DA:5,3
+DA:1,44
+DA:2,44
+DA:5,44
 DA:15,6
 DA:16,6
 DA:23,5
@@ -6391,7 +6393,7 @@ DA:26,4
 DA:38,0
 DA:39,0
 DA:41,0
-DA:51,3
+DA:51,44
 LF:12
 LH:9
 BRDA:21,0,0,0
@@ -7647,8 +7649,9 @@ FN:97,(anonymous_7)
 FN:102,(anonymous_8)
 FN:109,(anonymous_9)
 FN:113,(anonymous_10)
-FNF:11
-FNH:6
+FN:117,(anonymous_11)
+FNF:12
+FNH:7
 FNDA:8,newJob
 FNDA:45,(anonymous_1)
 FNDA:32,(anonymous_2)
@@ -7660,6 +7663,7 @@ FNDA:253,(anonymous_7)
 FNDA:0,(anonymous_8)
 FNDA:0,(anonymous_9)
 FNDA:0,(anonymous_10)
+FNDA:45,(anonymous_11)
 DA:1,45
 DA:13,8
 DA:33,45
@@ -7682,7 +7686,7 @@ DA:91,0
 DA:98,253
 DA:110,0
 DA:114,0
-DA:118,45
+DA:122,45
 LF:23
 LH:18
 BRDA:32,0,0,0

package/dist/automations/bullboard.js

@@ -18,10 +18,12 @@ const Queue = env.isTest()
 const { JobQueues } = require("../constants");
 const { utils } = require("@budibase/backend-core/redis");
 const { opts, redisProtocolUrl } = utils.getRedisOptions();
+const listeners = require("./listeners");
 const CLEANUP_PERIOD_MS = 60 * 1000;
 const queueConfig = redisProtocolUrl || { redis: opts };
 let cleanupInternal = null;
 let automationQueue = new Queue(JobQueues.AUTOMATIONS, queueConfig);
+listeners.addListeners(automationQueue);
 function cleanup() {
     return __awaiter(this, void 0, void 0, function* () {
         yield automationQueue.clean(CLEANUP_PERIOD_MS, "completed");

package/dist/automations/listeners.js

@@ -0,0 +1,102 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.addListeners = void 0;
+const automation = __importStar(require("../threads/automation"));
+const addListeners = (queue) => {
+    logging(queue);
+    handleStalled(queue);
+};
+exports.addListeners = addListeners;
+const handleStalled = (queue) => {
+    queue.on("stalled", (job) => __awaiter(void 0, void 0, void 0, function* () {
+        yield automation.removeStalled(job);
+    }));
+};
+const logging = (queue) => {
+    var _a;
+    if ((_a = process.env.NODE_DEBUG) === null || _a === void 0 ? void 0 : _a.includes("bull")) {
+        queue
+            .on("error", (error) => {
+            // An error occurred.
+            console.error(`automation-event=error error=${JSON.stringify(error)}`);
+        })
+            .on("waiting", (jobId) => {
+            // A Job is waiting to be processed as soon as a worker is idling.
+            console.log(`automation-event=waiting jobId=${jobId}`);
+        })
+            .on("active", (job, jobPromise) => {
+            // A job has started. You can use `jobPromise.cancel()`` to abort it.
+            console.log(`automation-event=active jobId=${job.id}`);
+        })
+            .on("stalled", (job) => {
+            // A job has been marked as stalled. This is useful for debugging job
+            // workers that crash or pause the event loop.
+            console.error(`automation-event=stalled jobId=${job.id} job=${JSON.stringify(job)}`);
+        })
+            .on("progress", (job, progress) => {
+            // A job's progress was updated!
+            console.log(`automation-event=progress jobId=${job.id} progress=${progress}`);
+        })
+            .on("completed", (job, result) => {
+            // A job successfully completed with a `result`.
+            console.log(`automation-event=completed jobId=${job.id} result=${result}`);
+        })
+            .on("failed", (job, err) => {
+            // A job failed with reason `err`!
+            console.log(`automation-event=failed jobId=${job.id} error=${err}`);
+        })
+            .on("paused", () => {
+            // The queue has been paused.
+            console.log(`automation-event=paused`);
+        })
+            .on("resumed", (job) => {
+            // The queue has been resumed.
+            console.log(`automation-event=paused jobId=${job.id}`);
+        })
+            .on("cleaned", (jobs, type) => {
+            // Old jobs have been cleaned from the queue. `jobs` is an array of cleaned
+            // jobs, and `type` is the type of jobs cleaned.
+            console.log(`automation-event=cleaned length=${jobs.length} type=${type}`);
+        })
+            .on("drained", () => {
+            // Emitted every time the queue has processed all the waiting jobs (even if there can be some delayed jobs not yet processed)
+            console.log(`automation-event=drained`);
+        })
+            .on("removed", (job) => {
+            // A job successfully removed.
+            console.log(`automation-event=removed jobId=${job.id}`);
+        });
+    }
+};

package/dist/automations/utils.js

@@ -52,10 +52,13 @@ const pro_1 = require("@budibase/pro");
 const WH_STEP_ID = triggerInfo_1.definitions.WEBHOOK.stepId;
 const CRON_STEP_ID = triggerInfo_1.definitions.CRON.stepId;
 const Runner = new threads_1.Thread(threads_1.ThreadType.AUTOMATION);
+const jobMessage = (job, message) => {
+    return `app=${job.data.event.appId} automation=${job.data.automation._id} jobId=${job.id} trigger=${job.data.automation.definition.trigger.event} : ${message}`;
+};
 function processEvent(job) {
     return __awaiter(this, void 0, void 0, function* () {
         try {
-            console.log(`${job.data.automation.appId} automation ${job.data.automation._id} running`);
+            console.log(jobMessage(job, "running"));
             // need to actually await these so that an error can be captured properly
             const tenantId = backend_core_1.tenancy.getTenantIDFromAppID(job.data.event.appId);
             return yield backend_core_1.tenancy.doInTenant(tenantId, () => __awaiter(this, void 0, void 0, function* () {
@@ -65,7 +68,7 @@ function processEvent(job) {
         }
         catch (err) {
             const errJson = JSON.stringify(err);
-            console.error(`${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${errJson}`);
+            console.error(jobMessage(job, `was unable to run - ${errJson}`));
             console.trace(err);
             return { err };
         }
@@ -119,6 +122,7 @@ function disableCron(jobId, jobKey) {
     return __awaiter(this, void 0, void 0, function* () {
         yield bullboard_1.queue.removeRepeatableByKey(jobKey);
         yield bullboard_1.queue.removeJobs(jobId);
+        console.log(`jobId=${jobId} disabled`);
     });
 }
 exports.disableCron = disableCron;

package/dist/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@budibase/server",
     "email": "hi@budibase.com",
-    "version": "1.2.43",
+    "version": "1.2.46",
     "description": "Budibase Web Server",
     "main": "src/index.ts",
     "repository": {
@@ -77,11 +77,11 @@
     "license": "GPL-3.0",
     "dependencies": {
         "@apidevtools/swagger-parser": "10.0.3",
-        "@budibase/backend-core": "^1.2.43",
-        "@budibase/client": "^1.2.43",
-        "@budibase/pro": "1.2.43",
-        "@budibase/string-templates": "^1.2.43",
-        "@budibase/types": "^1.2.43",
+        "@budibase/backend-core": "^1.2.46",
+        "@budibase/client": "^1.2.46",
+        "@budibase/pro": "1.2.46",
+        "@budibase/string-templates": "^1.2.46",
+        "@budibase/types": "^1.2.46",
         "@bull-board/api": "3.7.0",
         "@bull-board/koa": "3.9.4",
         "@elastic/elasticsearch": "7.10.0",

package/dist/threads/automation.js

@@ -12,7 +12,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.execute = void 0;
+exports.removeStalled = exports.execute = void 0;
 const utils_1 = __importDefault(require("./utils"));
 utils_1.default.threadSetup();
 const utils_2 = require("../automations/utils");
@@ -130,22 +130,30 @@ class Orchestrator {
             return metadata;
         });
     }
-    checkIfShouldStop(metadata) {
+    stopCron(reason) {
         var _a, _b;
         return __awaiter(this, void 0, void 0, function* () {
-            if (!metadata.errorCount || !this._repeat) {
-                return false;
+            if (!this._repeat) {
+                return;
             }
+            logWarn(`CRON disabled reason=${reason} - ${this._appId}/${this._automation._id}`);
             const automation = this._automation;
             const trigger = automation.definition.trigger;
+            yield (0, utils_2.disableCron)((_a = this._repeat) === null || _a === void 0 ? void 0 : _a.jobId, (_b = this._repeat) === null || _b === void 0 ? void 0 : _b.jobKey);
+            this.updateExecutionOutput(trigger.id, trigger.stepId, {}, {
+                status: types_1.AutomationStatus.STOPPED_ERROR,
+                success: false,
+            });
+            yield (0, logging_1.storeLog)(automation, this.executionOutput);
+        });
+    }
+    checkIfShouldStop(metadata) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!metadata.errorCount || !this._repeat) {
+                return false;
+            }
             if (metadata.errorCount >= constants_1.MAX_AUTOMATION_RECURRING_ERRORS) {
-                logWarn(`CRON disabled due to errors - ${this._appId}/${this._automation._id}`);
-                yield (0, utils_2.disableCron)((_a = this._repeat) === null || _a === void 0 ? void 0 : _a.jobId, (_b = this._repeat) === null || _b === void 0 ? void 0 : _b.jobKey);
-                this.updateExecutionOutput(trigger.id, trigger.stepId, {}, {
-                    status: types_1.AutomationStatus.STOPPED_ERROR,
-                    success: false,
-                });
-                yield (0, logging_1.storeLog)(automation, this.executionOutput);
+                yield this.stopCron("errors");
                 return true;
             }
             return false;
@@ -396,3 +404,11 @@ function execute(input, callback) {
     }));
 }
 exports.execute = execute;
+const removeStalled = (input) => __awaiter(void 0, void 0, void 0, function* () {
+    const appId = input.data.event.appId;
+    yield doInAppContext(appId, () => __awaiter(void 0, void 0, void 0, function* () {
+        const automationOrchestrator = new Orchestrator(input.data.automation, input.data.event, input.opts);
+        yield automationOrchestrator.stopCron("stalled");
+    }));
+});
+exports.removeStalled = removeStalled;