@budibase/server 1.0.105-alpha.19 → 1.0.105-alpha.21

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.105-alpha.19",
+  "version": "1.0.105-alpha.21",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -68,9 +68,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.105-alpha.19",
-    "@budibase/client": "^1.0.105-alpha.19",
-    "@budibase/string-templates": "^1.0.105-alpha.19",
+    "@budibase/backend-core": "^1.0.105-alpha.21",
+    "@budibase/client": "^1.0.105-alpha.21",
+    "@budibase/string-templates": "^1.0.105-alpha.21",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -174,5 +174,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "31d2ae8c6b4628b020dc01f01acfc0b09a401456"
+  "gitHead": "9ca0d4f0bfbd9313451c3da656390b4641a75f17"
 }

package/src/automations/actions.js

@@ -13,6 +13,7 @@ const integromat = require("./steps/integromat")
 let filter = require("./steps/filter")
 let delay = require("./steps/delay")
 let queryRow = require("./steps/queryRows")
+let loop = require("./steps/loop")
 const env = require("../environment")
 
 const ACTION_IMPLS = {
@@ -27,6 +28,7 @@ const ACTION_IMPLS = {
   DELAY: delay.run,
   FILTER: filter.run,
   QUERY_ROWS: queryRow.run,
+  LOOP: loop.run,
   // these used to be lowercase step IDs, maintain for backwards compat
   discord: discord.run,
   slack: slack.run,
@@ -45,6 +47,7 @@ const ACTION_DEFINITIONS = {
   DELAY: delay.definition,
   FILTER: filter.definition,
   QUERY_ROWS: queryRow.definition,
+  LOOP: loop.definition,
   // these used to be lowercase step IDs, maintain for backwards compat
   discord: discord.definition,
   slack: slack.definition,

package/src/automations/automationUtils.js

@@ -1,4 +1,5 @@
 const { getTable } = require("../api/controllers/table/utils")
+const { findHBSBlocks } = require("@budibase/string-templates")
 
 /**
  * When values are input to the system generally they will be of type string as this is required for template strings.
@@ -74,3 +75,14 @@ exports.getError = err => {
   }
   return typeof err !== "string" ? err.toString() : err
 }
+
+exports.substituteLoopStep = (hbsString, substitute) => {
+  let blocks = findHBSBlocks(hbsString)
+  for (let block of blocks) {
+    let oldBlock = block
+    block = block.replace(/loop/, substitute)
+    hbsString = hbsString.replace(new RegExp(oldBlock, "g"), block)
+  }
+
+  return hbsString
+}

package/src/automations/steps/loop.js

@@ -0,0 +1,50 @@
+exports.definition = {
+  name: "Looping",
+  icon: "Reuse",
+  tagline: "Loop the block",
+  description: "Loop",
+  stepId: "LOOP",
+  internal: true,
+  inputs: {},
+  schema: {
+    inputs: {
+      properties: {
+        option: {
+          customType: "loopOption",
+          title: "Input type",
+        },
+        binding: {
+          type: "string",
+          title: "Binding / Value",
+        },
+        iterations: {
+          type: "number",
+          title: "Max loop iterations",
+        },
+        failure: {
+          type: "string",
+          title: "Failure Condition",
+        },
+      },
+      required: ["type", "value", "iterations", "failure"],
+    },
+    outputs: {
+      properties: {
+        items: {
+          customType: "item",
+          description: "The item currently being executed",
+        },
+        success: {
+          type: "boolean",
+          description: "Whether the message loop was successfully",
+        },
+        iterations: {
+          type: "number",
+          descriptions: "The amount of times the block ran",
+        },
+      },
+      required: ["success", "items", "iterations"],
+    },
+  },
+  type: "LOGIC",
+}

package/src/constants/index.js

@@ -190,5 +190,11 @@ exports.WebhookType = {
   AUTOMATION: "automation",
 }
 
+exports.AutomationErrors = {
+  INCORRECT_TYPE: "INCORRECT_TYPE",
+  MAX_ITERATIONS: "MAX_ITERATIONS_REACHED",
+  FAILURE_CONDITION: "FAILURE_CONDITION_MET",
+}
+
 // pass through the list from the auth/core lib
 exports.ObjectStoreBuckets = ObjectStoreBuckets

package/src/environment.js

@@ -59,6 +59,7 @@ module.exports = {
   LOG_LEVEL: process.env.LOG_LEVEL,
   AUTOMATION_DIRECTORY: process.env.AUTOMATION_DIRECTORY,
   AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET,
+  AUTOMATION_MAX_ITERATIONS: process.env.AUTOMATION_MAX_ITERATIONS,
   SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
   DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
   POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,

package/src/threads/automation.js

@@ -8,10 +8,14 @@ const { DocumentTypes } = require("../db/utils")
 const { doInTenant } = require("@budibase/backend-core/tenancy")
 const { definitions: triggerDefs } = require("../automations/triggerInfo")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
-
+const { AutomationErrors } = require("../constants")
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
+const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
+
 const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: false, status: "STOPPED" }
+const { cloneDeep } = require("lodash/fp")
+const env = require("../environment")
 
 /**
  * The automation orchestrator is a class responsible for executing automations.
@@ -74,50 +78,208 @@ class Orchestrator {
     this.executionOutput.steps.push(stepObj)
   }
 
+  updateContextAndOutput(loopStepNumber, step, output, result) {
+    this.executionOutput.steps.splice(loopStepNumber, 0, {
+      id: step.id,
+      stepId: step.stepId,
+      outputs: {
+        ...output,
+        success: result.success,
+        status: result.status,
+      },
+      inputs: step.inputs,
+    })
+    this._context.steps.splice(loopStepNumber, 0, {
+      ...output,
+      success: result.success,
+      status: result.status,
+    })
+  }
+
   async execute() {
     let automation = this._automation
     const app = await this.getApp()
     let stopped = false
+    let loopStep
+
+    let stepCount = 0
+    let loopStepNumber
+    let loopSteps = []
     for (let step of automation.definition.steps) {
-      // execution stopped, record state for that
-      if (stopped) {
-        this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS)
+      stepCount++
+      let input
+      if (step.stepId === LOOP_STEP_ID) {
+        loopStep = step
+        loopStepNumber = stepCount
         continue
       }
-      let stepFn = await this.getStepFunctionality(step.stepId)
-      step.inputs = await processObject(step.inputs, this._context)
-      step.inputs = automationUtils.cleanInputValues(
-        step.inputs,
-        step.schema.inputs
-      )
-      // appId is always passed
-      try {
-        let tenantId = app.tenantId || DEFAULT_TENANT_ID
-        const outputs = await doInTenant(tenantId, () => {
-          return stepFn({
-            inputs: step.inputs,
-            appId: this._appId,
-            emitter: this._emitter,
-            context: this._context,
-          })
-        })
-        this._context.steps.push(outputs)
-        // if filter causes us to stop execution don't break the loop, set a var
-        // so that we can finish iterating through the steps and record that it stopped
-        if (step.stepId === FILTER_STEP_ID && !outputs.success) {
-          stopped = true
-          this.updateExecutionOutput(step.id, step.stepId, step.inputs, {
-            ...outputs,
-            ...STOPPED_STATUS,
-          })
+
+      if (loopStep) {
+        input = await processObject(loopStep.inputs, this._context)
+      }
+      let iterations = loopStep ? input.binding.length : 1
+      let iterationCount = 0
+      for (let index = 0; index < iterations; index++) {
+        let originalStepInput = cloneDeep(step.inputs)
+
+        // Handle if the user has set a max iteration count or if it reaches the max limit set by us
+        if (loopStep) {
+          // lets first of all handle the input
+          // if the input is array then use it, if it is a string then split it on every new line
+          let newInput = await processObject(
+            loopStep.inputs,
+            cloneDeep(this._context)
+          )
+          newInput = automationUtils.cleanInputValues(
+            newInput,
+            loopStep.schema.inputs
+          )
+          this._context.steps[loopStepNumber] = {
+            currentItem: newInput.binding[index],
+          }
+
+          let tempOutput = { items: loopSteps, iterations: iterationCount }
+          if (
+            (loopStep.inputs.option === "Array" &&
+              !Array.isArray(newInput.binding)) ||
+            (loopStep.inputs.option === "String" &&
+              typeof newInput.binding !== "string")
+          ) {
+            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
+              status: AutomationErrors.INCORRECT_TYPE,
+              success: false,
+            })
+            loopSteps = null
+            loopStep = null
+            break
+          }
+
+          // The "Loop" binding in the front end is "fake", so replace it here so the context can understand it
+          // Pretty hacky because we need to account for the row object
+          for (let [key, value] of Object.entries(originalStepInput)) {
+            if (typeof value === "object") {
+              for (let [innerKey, innerValue] of Object.entries(
+                originalStepInput[key]
+              )) {
+                if (typeof innerValue === "string") {
+                  originalStepInput[key][innerKey] =
+                    automationUtils.substituteLoopStep(
+                      innerValue,
+                      `steps.${loopStepNumber}`
+                    )
+                }
+              }
+            } else {
+              if (typeof value === "string") {
+                originalStepInput[key] = automationUtils.substituteLoopStep(
+                  value,
+                  `steps.${loopStepNumber}`
+                )
+              }
+            }
+          }
+
+          if (
+            index === parseInt(env.AUTOMATION_MAX_ITERATIONS) ||
+            index === loopStep.inputs.iterations
+          ) {
+            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
+              status: AutomationErrors.MAX_ITERATIONS,
+              success: true,
+            })
+            loopSteps = null
+            loopStep = null
+            break
+          }
+
+          if (
+            this._context.steps[loopStepNumber]?.currentItem ===
+            loopStep.inputs.failure
+          ) {
+            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
+              status: AutomationErrors.FAILURE_CONDITION,
+              success: false,
+            })
+            loopSteps = null
+            loopStep = null
+            break
+          }
+        }
+
+        // execution stopped, record state for that
+        if (stopped) {
+          this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS)
           continue
         }
-        this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs)
-      } catch (err) {
-        console.error(`Automation error - ${step.stepId} - ${err}`)
-        return err
+
+        // If it's a loop step, we need to manually add the bindings to the context
+        let stepFn = await this.getStepFunctionality(step.stepId)
+        let inputs = await processObject(originalStepInput, this._context)
+        inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs)
+        try {
+          // appId is always passed
+          let tenantId = app.tenantId || DEFAULT_TENANT_ID
+          const outputs = await doInTenant(tenantId, () => {
+            return stepFn({
+              inputs: inputs,
+              appId: this._appId,
+              emitter: this._emitter,
+              context: this._context,
+            })
+          })
+          this._context.steps[stepCount] = outputs
+          // if filter causes us to stop execution don't break the loop, set a var
+          // so that we can finish iterating through the steps and record that it stopped
+          if (step.stepId === FILTER_STEP_ID && !outputs.success) {
+            stopped = true
+            this.updateExecutionOutput(step.id, step.stepId, step.inputs, {
+              ...outputs,
+              ...STOPPED_STATUS,
+            })
+            continue
+          }
+          if (loopStep && loopSteps) {
+            loopSteps.push(outputs)
+          } else {
+            this.updateExecutionOutput(
+              step.id,
+              step.stepId,
+              step.inputs,
+              outputs
+            )
+          }
+        } catch (err) {
+          console.error(`Automation error - ${step.stepId} - ${err}`)
+          return err
+        }
+        if (loopStep) {
+          iterationCount++
+          if (index === iterations - 1) {
+            loopStep = null
+            this._context.steps.splice(loopStepNumber, 1)
+            break
+          }
+        }
+      }
+
+      if (loopSteps && loopSteps.length) {
+        let tempOutput = {
+          success: true,
+          items: loopSteps,
+          iterations: iterationCount,
+        }
+        this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
+          id: step.id,
+          stepId: step.stepId,
+          outputs: tempOutput,
+          inputs: step.inputs,
+        })
+
+        this._context.steps.splice(loopStepNumber, 0, tempOutput)
+        loopSteps = null
       }
     }
+
     return this.executionOutput
   }
 }