npm - @budibase/server - Versions diffs - 2.6.19-alpha.30 → 2.6.19-alpha.34 - Mend

@budibase/server 2.6.19-alpha.30 → 2.6.19-alpha.34

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (17) hide show

package/builder/assets/{index.8469b14c.css → index.a96c516c.css} +1 -1
package/builder/assets/{index.3dd3d237.js → index.b6e30d5b.js} +3 -3
package/builder/index.html +2 -2
package/dist/automation.js +148 -90
package/dist/automation.js.map +3 -3
package/dist/index.js +234 -164
package/dist/index.js.map +3 -3
package/dist/query.js +27 -13
package/dist/query.js.map +2 -2
package/package.json +9 -8
package/src/automations/logging/index.ts +21 -0
package/src/environment.ts +1 -0
package/src/middleware/builder.ts +22 -13
package/src/threads/automation.ts +49 -28
package/src/threads/index.ts +9 -3
package/src/threads/utils.ts +2 -0
package/src/utilities/fileSystem/app.ts +14 -4

package/builder/index.html CHANGED Viewed

@@ -8,8 +8,8 @@
 	<link rel="preconnect" href="https://fonts.gstatic.com" />
 	<link href="https://fonts.googleapis.com/css2?family=Source+Sans+Pro:wght@400;600;700&display=swap"
 		rel="stylesheet" />
-  <script type="module" crossorigin src="/builder/assets/index.3dd3d237.js"></script>
-  <link rel="stylesheet" href="/builder/assets/index.8469b14c.css">
+  <script type="module" crossorigin src="/builder/assets/index.b6e30d5b.js"></script>
+  <link rel="stylesheet" href="/builder/assets/index.a96c516c.css">
 </head>
 <body id="app">

package/dist/automation.js CHANGED Viewed

@@ -128,6 +128,7 @@ var init_environment = __esm({
       ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
       SELF_HOSTED: process.env.SELF_HOSTED,
       HTTP_MB_LIMIT: process.env.HTTP_MB_LIMIT,
+      FORKED_PROCESS_NAME: process.env.FORKED_PROCESS_NAME || "main",
       // old
       CLIENT_ID: process.env.CLIENT_ID,
       _set(key, value) {
@@ -3440,9 +3441,76 @@ var init_users3 = __esm({
 // ../backend-core/src/redis/redlockImpl.ts
 var redlockImpl_exports = {};
 __export(redlockImpl_exports, {
-  doWithLock: () => doWithLock
+  doWithLock: () => doWithLock,
+  newRedlock: () => newRedlock
 });
-var import_redlock, getClient, OPTIONS, newRedlock, doWithLock;
+async function getClient(type, opts) {
+  if (type === "custom" /* CUSTOM */) {
+    return newRedlock(opts);
+  }
+  if (environment_default2.isTest() && type !== "try_once" /* TRY_ONCE */) {
+    return newRedlock(OPTIONS.TEST);
+  }
+  switch (type) {
+    case "try_once" /* TRY_ONCE */: {
+      return newRedlock(OPTIONS.TRY_ONCE);
+    }
+    case "try_twice" /* TRY_TWICE */: {
+      return newRedlock(OPTIONS.TRY_TWICE);
+    }
+    case "default" /* DEFAULT */: {
+      return newRedlock(OPTIONS.DEFAULT);
+    }
+    case "delay_500" /* DELAY_500 */: {
+      return newRedlock(OPTIONS.DELAY_500);
+    }
+    default: {
+      throw new Error(`Could not get redlock client: ${type}`);
+    }
+  }
+}
+async function newRedlock(opts = {}) {
+  let options2 = { ...OPTIONS.DEFAULT, ...opts };
+  const redisWrapper = await getLockClient();
+  const client3 = redisWrapper.getClient();
+  return new import_redlock.default([client3], options2);
+}
+function getLockName(opts) {
+  const prefix = opts.systemLock ? "system" : getTenantId();
+  let name = `lock:${prefix}_${opts.name}`;
+  if (opts.resource) {
+    name = name + `_${opts.resource}`;
+  }
+  return name;
+}
+async function doWithLock(opts, task) {
+  const redlock = await getClient(opts.type, opts.customOptions);
+  let lock;
+  try {
+    const name = getLockName(opts);
+    lock = await redlock.lock(name, opts.ttl);
+    const result = await task();
+    return { executed: true, result };
+  } catch (e) {
+    console.warn("lock error");
+    if (e.name === "LockError") {
+      if (opts.type === "try_once" /* TRY_ONCE */) {
+        return { executed: false };
+      } else {
+        console.error(e);
+        throw e;
+      }
+    } else {
+      console.error(e);
+      throw e;
+    }
+  } finally {
+    if (lock) {
+      await lock.unlock();
+    }
+  }
+}
+var import_redlock, OPTIONS;
 var init_redlockImpl = __esm({
   "../backend-core/src/redis/redlockImpl.ts"() {
     import_redlock = __toESM(require("redlock"));
@@ -3450,33 +3518,14 @@ var init_redlockImpl = __esm({
     init_src();
     init_context2();
     init_environment3();
-    getClient = async (type, opts) => {
-      if (type === "custom" /* CUSTOM */) {
-        return newRedlock(opts);
-      }
-      if (environment_default2.isTest() && type !== "try_once" /* TRY_ONCE */) {
-        return newRedlock(OPTIONS.TEST);
-      }
-      switch (type) {
-        case "try_once" /* TRY_ONCE */: {
-          return newRedlock(OPTIONS.TRY_ONCE);
-        }
-        case "default" /* DEFAULT */: {
-          return newRedlock(OPTIONS.DEFAULT);
-        }
-        case "delay_500" /* DELAY_500 */: {
-          return newRedlock(OPTIONS.DELAY_500);
-        }
-        default: {
-          throw new Error(`Could not get redlock client: ${type}`);
-        }
-      }
-    };
     OPTIONS = {
       TRY_ONCE: {
         // immediately throws an error if the lock is already held
         retryCount: 0
       },
+      TRY_TWICE: {
+        retryCount: 1
+      },
       TEST: {
         // higher retry count in unit tests
         // due to high contention.
@@ -3503,44 +3552,6 @@ var init_redlockImpl = __esm({
         retryDelay: 500
       }
     };
-    newRedlock = async (opts = {}) => {
-      let options2 = { ...OPTIONS.DEFAULT, ...opts };
-      const redisWrapper = await getLockClient();
-      const client3 = redisWrapper.getClient();
-      return new import_redlock.default([client3], options2);
-    };
-    doWithLock = async (opts, task) => {
-      const redlock = await getClient(opts.type, opts.customOptions);
-      let lock;
-      try {
-        const prefix = opts.systemLock ? "system" : getTenantId();
-        let name = `lock:${prefix}_${opts.name}`;
-        if (opts.resource) {
-          name = name + `_${opts.resource}`;
-        }
-        lock = await redlock.lock(name, opts.ttl);
-        const result = await task();
-        return { executed: true, result };
-      } catch (e) {
-        console.warn("lock error");
-        if (e.name === "LockError") {
-          if (opts.type === "try_once" /* TRY_ONCE */) {
-            console.warn(e);
-            return { executed: false };
-          } else {
-            console.error(e);
-            throw e;
-          }
-        } else {
-          console.error(e);
-          throw e;
-        }
-      } finally {
-        if (lock) {
-          await lock.unlock();
-        }
-      }
-    };
   }
 });
@@ -3647,6 +3658,7 @@ var init_logger = __esm({
         }
         const mergingObject = {
           err: error,
+          pid: process.pid,
           ...contextObject
         };
         if (objects.length) {
@@ -26770,8 +26782,10 @@ function makeVariableKey(queryId, variable) {
 }
 function threadSetup() {
   if (environment_default.isTest() || environment_default.DISABLE_THREADING || !environment_default.isInThread()) {
+    console.debug(`[${environment_default.FORKED_PROCESS_NAME}] thread setup skipped`);
     return;
   }
+  console.debug(`[${environment_default.FORKED_PROCESS_NAME}] thread setup running`);
   init8();
 }
 async function checkCacheForDynamicVariable(queryId, variable) {
@@ -26845,13 +26859,17 @@ var _Thread = class {
     this.count = opts.count ? opts.count : 1;
     this.disableThreading = this.shouldDisableThreading();
     if (!this.disableThreading) {
+      console.debug(
+        `[${environment_default.FORKED_PROCESS_NAME}] initialising worker farm type=${type}`
+      );
       const workerOpts = {
         autoStart: true,
         maxConcurrentWorkers: this.count,
         workerOptions: {
           env: {
             ...process.env,
-            FORKED_PROCESS: "1"
+            FORKED_PROCESS: "1",
+            FORKED_PROCESS_NAME: type
           }
         }
       };
@@ -26861,6 +26879,10 @@ var _Thread = class {
       }
       this.workers = (0, import_worker_farm.default)(workerOpts, typeToFile(type), ["execute"]);
       _Thread.workerRefs.push(this.workers);
+    } else {
+      console.debug(
+        `[${environment_default.FORKED_PROCESS_NAME}] skipping worker farm type=${type}`
+      );
     }
   }
   shouldDisableThreading() {
@@ -26872,9 +26894,7 @@ var _Thread = class {
       function fire(worker) {
         worker.execute(job, (err, response2) => {
           if (err && err.type === "TimeoutError") {
-            reject(
-              new Error(`Query response time exceeded ${timeout2}ms timeout.`)
-            );
+            reject(new Error(`Thread timeout exceeded ${timeout2}ms timeout.`));
           } else if (err) {
             reject(err);
           } else {
@@ -31950,10 +31970,29 @@ init_constants6();
 init_environment();
 init_src4();
 init_src2();
+var import_object_sizeof = __toESM(require("object-sizeof"));
+var MAX_LOG_SIZE_MB = 5;
+var MB_IN_BYTES = 1024 * 1024;
+function sanitiseResults(results) {
+  const message = `[removed] - max results size of ${MAX_LOG_SIZE_MB}MB exceeded`;
+  for (let step of results.steps) {
+    step.inputs = {
+      message
+    };
+    step.outputs = {
+      message,
+      success: step.outputs.success
+    };
+  }
+}
 async function storeLog2(automation, results) {
   if (environment_default.DISABLE_AUTOMATION_LOGS) {
     return;
   }
+  const bytes = (0, import_object_sizeof.default)(results);
+  if (bytes / MB_IN_BYTES > MAX_LOG_SIZE_MB) {
+    sanitiseResults(results);
+  }
   await automations_exports.logs.storeLog(automation, results);
 }
@@ -31962,6 +32001,7 @@ init_src();
 init_src2();
 var import_string_templates8 = __toESM(require_src2());
 var import_fp8 = require("lodash/fp");
+var import_perf_hooks2 = require("perf_hooks");
 init_utils15();
 init_environment();
 utils_default.threadSetup();
@@ -31969,8 +32009,8 @@ var FILTER_STEP_ID = BUILTIN_ACTION_DEFINITIONS.FILTER.stepId;
 var LOOP_STEP_ID = BUILTIN_ACTION_DEFINITIONS.LOOP.stepId;
 var CRON_STEP_ID2 = definitions.CRON.stepId;
 var STOPPED_STATUS = { success: true, status: "stopped" /* STOPPED */ };
-function getLoopIterations(loopStep, input) {
-  const binding = typecastForLooping(loopStep, input);
+function getLoopIterations(loopStep) {
+  let binding = loopStep.inputs.binding;
   if (!binding) {
     return 0;
   }
@@ -31986,7 +32026,6 @@ var Orchestrator = class {
   constructor(job) {
     let automation = job.data.automation;
     let triggerOutput = job.data.event;
-    let timeout2 = job.data.event.timeout;
     const metadata = triggerOutput.metadata;
     this._chainCount = metadata ? metadata.automationChainCount : 0;
     this._appId = triggerOutput.appId;
@@ -32120,7 +32159,7 @@ var Orchestrator = class {
     });
   }
   async execute() {
-    var _a;
+    var _a, _b;
     this._context.env = await getEnvironmentVariables2();
     let automation = this._automation;
     let stopped = false;
@@ -32139,6 +32178,7 @@ var Orchestrator = class {
         return;
       }
     }
+    const start2 = import_perf_hooks2.performance.now();
     for (let step of automation.definition.steps) {
       if (timeoutFlag) {
         break;
@@ -32157,20 +32197,16 @@ var Orchestrator = class {
       }
       if (loopStep) {
         input = await (0, import_string_templates8.processObject)(loopStep.inputs, this._context);
-        iterations = getLoopIterations(loopStep, input);
+        iterations = getLoopIterations(loopStep);
       }
       for (let index2 = 0; index2 < iterations; index2++) {
         let originalStepInput = (0, import_fp8.cloneDeep)(step.inputs);
         if (loopStep && input.binding) {
-          let newInput = await (0, import_string_templates8.processObject)(
-            loopStep.inputs,
-            (0, import_fp8.cloneDeep)(this._context)
-          );
           let tempOutput = { items: loopSteps, iterations: iterationCount };
           try {
-            newInput.binding = typecastForLooping(
+            loopStep.inputs.binding = typecastForLooping(
               loopStep,
-              newInput
+              loopStep.inputs
             );
           } catch (err) {
             this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
@@ -32183,7 +32219,7 @@ var Orchestrator = class {
           }
           let item = [];
           if (typeof loopStep.inputs.binding === "string" && loopStep.inputs.option === "String") {
-            item = stringSplit(newInput.binding);
+            item = stringSplit(loopStep.inputs.binding);
           } else if (Array.isArray(loopStep.inputs.binding)) {
             item = loopStep.inputs.binding;
           }
@@ -32326,7 +32362,21 @@ var Orchestrator = class {
         loopSteps = [];
       }
     }
-    await storeLog2(this._automation, this.executionOutput);
+    const end2 = import_perf_hooks2.performance.now();
+    const executionTime = end2 - start2;
+    console.info(`Execution time: ${executionTime} milliseconds`, {
+      _logKey: "automation",
+      executionTime
+    });
+    try {
+      await storeLog2(this._automation, this.executionOutput);
+    } catch (e) {
+      if (e.status === 413 && ((_b = e.request) == null ? void 0 : _b.data)) {
+        delete e.request.data;
+        e.request.data = { message: "removed due to large size" };
+      }
+      logging_exports.logAlert("Error writing automation log", e);
+    }
     if (isProdAppID2(this._appId) && isRecurring(automation) && metadata) {
       await this.updateMetadata(metadata);
     }
@@ -32335,20 +32385,28 @@ var Orchestrator = class {
 };
 function execute3(job, callback) {
   const appId = job.data.event.appId;
+  const automationId = job.data.automation._id;
   if (!appId) {
     throw new Error("Unable to execute, event doesn't contain app ID.");
   }
-  return context_exports.doInAppContext(appId, async () => {
-    const envVars = await getEnvironmentVariables2();
-    await context_exports.doInEnvironmentContext(envVars, async () => {
-      const automationOrchestrator = new Orchestrator(job);
-      try {
-        const response2 = await automationOrchestrator.execute();
-        callback(null, response2);
-      } catch (err) {
-        callback(err);
-      }
-    });
+  if (!automationId) {
+    throw new Error("Unable to execute, event doesn't contain automation ID.");
+  }
+  return context_exports.doInAutomationContext({
+    appId,
+    automationId,
+    task: async () => {
+      const envVars = await getEnvironmentVariables2();
+      await context_exports.doInEnvironmentContext(envVars, async () => {
+        const automationOrchestrator = new Orchestrator(job);
+        try {
+          const response2 = await automationOrchestrator.execute();
+          callback(null, response2);
+        } catch (err) {
+          callback(err);
+        }
+      });
+    }
   });
 }
 function executeSynchronously(job) {