@budibase/server 1.1.32-alpha.4 → 1.1.32

package/src/threads/{automation.ts → automation.js}

@@ -1,47 +1,36 @@
-import { default as threadUtils } from "./utils"
-threadUtils.threadSetup()
-import { isRecurring, disableCron, isErrorInOutput } from "../automations/utils"
-import { default as actions } from "../automations/actions"
-import { default as automationUtils } from "../automations/automationUtils"
-import { default as AutomationEmitter } from "../events/AutomationEmitter"
-import { generateAutomationMetadataID, isProdAppID } from "../db/utils"
-import { definitions as triggerDefs } from "../automations/triggerInfo"
-import { AutomationErrors, MAX_AUTOMATION_RECURRING_ERRORS } from "../constants"
-import { storeLog } from "../automations/logging"
-import { Automation, AutomationStep, AutomationStatus } from "@budibase/types"
-import {
-  LoopStep,
-  LoopStepTypes,
-  LoopInput,
-  AutomationEvent,
-  TriggerOutput,
-  AutomationContext,
-  AutomationMetadata,
-} from "../definitions/automations"
-import { WorkerCallback } from "./definitions"
-const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
-const { logAlertWithInfo, logWarn } = require("@budibase/backend-core/logging")
+require("./utils").threadSetup()
+const actions = require("../automations/actions")
+const automationUtils = require("../automations/automationUtils")
+const AutomationEmitter = require("../events/AutomationEmitter")
 const { processObject } = require("@budibase/string-templates")
+const { DocumentTypes } = require("../db/utils")
+const { definitions: triggerDefs } = require("../automations/triggerInfo")
+const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
+const { AutomationErrors, LoopStepTypes } = require("../constants")
+const { storeLog } = require("../automations/logging")
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
 const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
+
 const CRON_STEP_ID = triggerDefs.CRON.stepId
-const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED }
+const STOPPED_STATUS = { success: true, status: "STOPPED" }
 const { cloneDeep } = require("lodash/fp")
 const env = require("../environment")
 
-function typecastForLooping(loopStep: LoopStep, input: LoopInput) {
+function typecastForLooping(loopStep, input) {
   if (!input || !input.binding) {
     return null
   }
+  const isArray = Array.isArray(input.binding),
+    isString = typeof input.binding === "string"
   try {
     switch (loopStep.inputs.option) {
       case LoopStepTypes.ARRAY:
-        if (typeof input.binding === "string") {
+        if (isString) {
           return JSON.parse(input.binding)
         }
         break
       case LoopStepTypes.STRING:
-        if (Array.isArray(input.binding)) {
+        if (isArray) {
           return input.binding.join(",")
         }
         break
@@ -52,7 +41,7 @@ function typecastForLooping(loopStep: LoopStep, input: LoopInput) {
   return input.binding
 }
 
-function getLoopIterations(loopStep: LoopStep, input: LoopInput) {
+function getLoopIterations(loopStep, input) {
   const binding = typecastForLooping(loopStep, input)
   if (!loopStep || !binding) {
     return 1
@@ -68,24 +57,11 @@ function getLoopIterations(loopStep: LoopStep, input: LoopInput) {
  * inputs and handles any outputs.
  */
 class Orchestrator {
-  _chainCount: number
-  _appId: string
-  _automation: Automation
-  _emitter: any
-  _context: AutomationContext
-  _repeat?: { jobId: string; jobKey: string }
-  executionOutput: AutomationContext
-
-  constructor(automation: Automation, triggerOutput: TriggerOutput, opts: any) {
-    const metadata = triggerOutput.metadata
-    this._chainCount = metadata ? metadata.automationChainCount : 0
-    this._appId = triggerOutput.appId as string
-    if (opts?.repeat) {
-      this._repeat = {
-        jobId: opts.repeat.jobId,
-        jobKey: opts.repeat.key,
-      }
-    }
+  constructor(automation, triggerOutput = {}) {
+    this._metadata = triggerOutput.metadata
+    this._chainCount = this._metadata ? this._metadata.automationChainCount : 0
+    this._appId = triggerOutput.appId
+    this._app = null
     const triggerStepId = automation.definition.trigger.stepId
     triggerOutput = this.cleanupTriggerOutputs(triggerStepId, triggerOutput)
     // remove from context
@@ -103,14 +79,14 @@ class Orchestrator {
     this.updateExecutionOutput(triggerId, triggerStepId, null, triggerOutput)
   }
 
-  cleanupTriggerOutputs(stepId: string, triggerOutput: TriggerOutput) {
+  cleanupTriggerOutputs(stepId, triggerOutput) {
     if (stepId === CRON_STEP_ID) {
       triggerOutput.timestamp = Date.now()
     }
     return triggerOutput
   }
 
-  async getStepFunctionality(stepId: string) {
+  async getStepFunctionality(stepId) {
     let step = await actions.getAction(stepId)
     if (step == null) {
       throw `Cannot find automation step by name ${stepId}`
@@ -118,107 +94,25 @@ class Orchestrator {
     return step
   }
 
-  async getMetadata(): Promise<AutomationMetadata> {
-    const metadataId = generateAutomationMetadataID(this._automation._id)
-    const db = getAppDB()
-    let metadata: AutomationMetadata
-    try {
-      metadata = await db.get(metadataId)
-    } catch (err) {
-      metadata = {
-        _id: metadataId,
-        errorCount: 0,
-      }
-    }
-    return metadata
-  }
-
-  async checkIfShouldStop(metadata: AutomationMetadata): Promise<boolean> {
-    if (!metadata.errorCount || !this._repeat) {
-      return false
-    }
-    const automation = this._automation
-    const trigger = automation.definition.trigger
-    if (metadata.errorCount >= MAX_AUTOMATION_RECURRING_ERRORS) {
-      logWarn(
-        `CRON disabled due to errors - ${this._appId}/${this._automation._id}`
-      )
-      await disableCron(this._repeat?.jobId, this._repeat?.jobKey)
-      this.updateExecutionOutput(
-        trigger.id,
-        trigger.stepId,
-        {},
-        {
-          status: AutomationStatus.STOPPED_ERROR,
-          success: false,
-        }
-      )
-      await storeLog(automation, this.executionOutput)
-      return true
-    }
-    return false
-  }
-
-  async updateMetadata(metadata: AutomationMetadata) {
-    const output = this.executionOutput,
-      automation = this._automation
-    if (!output || !isRecurring(automation)) {
-      return
-    }
-    const count = metadata.errorCount
-    const isError = isErrorInOutput(output)
-    // nothing to do in this scenario, escape
-    if (!count && !isError) {
-      return
-    }
-    if (isError) {
-      metadata.errorCount = count ? count + 1 : 1
-    } else {
-      metadata.errorCount = 0
+  async getApp() {
+    if (this._app) {
+      return this._app
     }
     const db = getAppDB()
-    try {
-      await db.put(metadata)
-    } catch (err) {
-      logAlertWithInfo(
-        "Failed to write automation metadata",
-        db.name,
-        automation._id,
-        err
-      )
-    }
+    this._app = await db.get(DocumentTypes.APP_METADATA)
+    return this._app
   }
 
-  updateExecutionOutput(id: string, stepId: string, inputs: any, outputs: any) {
+  updateExecutionOutput(id, stepId, inputs, outputs) {
     const stepObj = { id, stepId, inputs, outputs }
-    // replacing trigger when disabling CRON
-    if (
-      stepId === CRON_STEP_ID &&
-      outputs.status === AutomationStatus.STOPPED_ERROR
-    ) {
-      this.executionOutput.trigger = stepObj
-      this.executionOutput.steps = [stepObj]
-      return
-    }
     // first entry is always the trigger (constructor)
-    if (
-      this.executionOutput.steps.length === 0 ||
-      this.executionOutput.trigger.id === id
-    ) {
+    if (this.executionOutput.steps.length === 0) {
       this.executionOutput.trigger = stepObj
     }
     this.executionOutput.steps.push(stepObj)
   }
 
-  updateContextAndOutput(
-    loopStepNumber: number | undefined,
-    step: AutomationStep,
-    output: any,
-    result: { success: boolean; status: string }
-  ) {
-    if (!loopStepNumber) {
-      throw new Error("No loop step number provided.")
-    }
+  updateContextAndOutput(loopStepNumber, step, output, result) {
     this.executionOutput.steps.splice(loopStepNumber, 0, {
       id: step.id,
       stepId: step.stepId,
@@ -239,22 +133,11 @@ class Orchestrator {
   async execute() {
     let automation = this._automation
     let stopped = false
-    let loopStep: AutomationStep | undefined = undefined
+    let loopStep = null
 
     let stepCount = 0
-    let loopStepNumber: any = undefined
-    let loopSteps: LoopStep[] | undefined = []
-    let metadata
-
-    // check if this is a recurring automation,
-    if (isProdAppID(this._appId) && isRecurring(automation)) {
-      metadata = await this.getMetadata()
-      const shouldStop = await this.checkIfShouldStop(metadata)
-      if (shouldStop) {
-        return
-      }
-    }
-
+    let loopStepNumber = null
+    let loopSteps = []
     for (let step of automation.definition.steps) {
       stepCount++
       let input,
@@ -268,7 +151,7 @@ class Orchestrator {
 
       if (loopStep) {
         input = await processObject(loopStep.inputs, this._context)
-        iterations = getLoopIterations(loopStep as LoopStep, input)
+        iterations = getLoopIterations(loopStep, input)
       }
 
       for (let index = 0; index < iterations; index++) {
@@ -283,17 +166,14 @@ class Orchestrator {
 
           let tempOutput = { items: loopSteps, iterations: iterationCount }
           try {
-            newInput.binding = typecastForLooping(
-              loopStep as LoopStep,
-              newInput
-            )
+            newInput.binding = typecastForLooping(loopStep, newInput)
           } catch (err) {
             this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
               status: AutomationErrors.INCORRECT_TYPE,
               success: false,
             })
-            loopSteps = undefined
-            loopStep = undefined
+            loopSteps = null
+            loopStep = null
             break
           }
 
@@ -343,8 +223,8 @@ class Orchestrator {
               status: AutomationErrors.MAX_ITERATIONS,
               success: true,
             })
-            loopSteps = undefined
-            loopStep = undefined
+            loopSteps = null
+            loopStep = null
             break
           }
 
@@ -352,7 +232,7 @@ class Orchestrator {
           const currentItem = this._context.steps[loopStepNumber]?.currentItem
           if (currentItem && typeof currentItem === "object") {
             isFailure = Object.keys(currentItem).some(value => {
-              return currentItem[value] === loopStep?.inputs.failure
+              return currentItem[value] === loopStep.inputs.failure
             })
           } else {
             isFailure = currentItem && currentItem === loopStep.inputs.failure
@@ -363,8 +243,8 @@ class Orchestrator {
               status: AutomationErrors.FAILURE_CONDITION,
               success: false,
             })
-            loopSteps = undefined
-            loopStep = undefined
+            loopSteps = null
+            loopStep = null
             break
           }
         }
@@ -415,7 +295,7 @@ class Orchestrator {
         if (loopStep) {
           iterationCount++
           if (index === iterations - 1) {
-            loopStep = undefined
+            loopStep = null
             this._context.steps.splice(loopStepNumber, 1)
             break
           }
@@ -436,26 +316,22 @@ class Orchestrator {
         })
 
         this._context.steps.splice(loopStepNumber, 0, tempOutput)
-        loopSteps = undefined
+        loopSteps = null
       }
     }
 
     // store the logs for the automation run
     await storeLog(this._automation, this.executionOutput)
-    if (isProdAppID(this._appId) && isRecurring(automation) && metadata) {
-      await this.updateMetadata(metadata)
-    }
     return this.executionOutput
   }
 }
 
-export function execute(input: AutomationEvent, callback: WorkerCallback) {
+module.exports = (input, callback) => {
   const appId = input.data.event.appId
   doInAppContext(appId, async () => {
     const automationOrchestrator = new Orchestrator(
       input.data.automation,
-      input.data.event,
-      input.opts
+      input.data.event
     )
     try {
       const response = await automationOrchestrator.execute()

package/src/threads/index.ts

@@ -18,23 +18,26 @@ function typeToFile(type: any) {
     default:
       throw "Unknown thread type"
   }
-  // have to use require here, to make it work with worker-farm
   return require.resolve(filename)
 }
 
 export class Thread {
   type: any
   count: any
+  disableThreading: any
   workers: any
   timeoutMs: any
-  disableThreading: boolean
 
   static workerRefs: any[] = []
 
   constructor(type: any, opts: any = { timeoutMs: null, count: 1 }) {
     this.type = type
     this.count = opts.count ? opts.count : 1
-    this.disableThreading = this.shouldDisableThreading()
+    this.disableThreading =
+      env.isTest() ||
+      env.DISABLE_THREADING ||
+      this.count === 0 ||
+      env.isInThread()
     if (!this.disableThreading) {
       const workerOpts: any = {
         autoStart: true,
@@ -44,44 +47,33 @@ export class Thread {
         this.timeoutMs = opts.timeoutMs
         workerOpts.maxCallTime = opts.timeoutMs
       }
-      this.workers = workerFarm(workerOpts, typeToFile(type), ["execute"])
+      this.workers = workerFarm(workerOpts, typeToFile(type))
       Thread.workerRefs.push(this.workers)
     }
   }
 
-  shouldDisableThreading(): boolean {
-    return !!(
-      env.isTest() ||
-      env.DISABLE_THREADING ||
-      this.count === 0 ||
-      env.isInThread()
-    )
-  }
-
   run(data: any) {
-    const timeout = this.timeoutMs
     return new Promise((resolve, reject) => {
-      function fire(worker: any) {
-        worker.execute(data, (err: any, response: any) => {
-          if (err && err.type === "TimeoutError") {
-            reject(
-              new Error(`Query response time exceeded ${timeout}ms timeout.`)
-            )
-          } else if (err) {
-            reject(err)
-          } else {
-            resolve(response)
-          }
-        })
-      }
+      let fncToCall
       // if in test then don't use threading
       if (this.disableThreading) {
-        import(typeToFile(this.type)).then((thread: any) => {
-          fire(thread)
-        })
+        fncToCall = require(typeToFile(this.type))
       } else {
-        fire(this.workers)
+        fncToCall = this.workers
       }
+      fncToCall(data, (err: any, response: any) => {
+        if (err && err.type === "TimeoutError") {
+          reject(
+            new Error(
+              `Query response time exceeded ${this.timeoutMs}ms timeout.`
+            )
+          )
+        } else if (err) {
+          reject(err)
+        } else {
+          resolve(response)
+        }
+      })
     })
   }
 

package/src/threads/{query.ts → query.js}

@@ -1,6 +1,5 @@
-import { default as threadUtils } from "./utils"
+const threadUtils = require("./utils")
 threadUtils.threadSetup()
-import { WorkerCallback, QueryEvent, QueryVariable } from "./definitions"
 const ScriptRunner = require("../utilities/scriptRunner")
 const { integrations } = require("../integrations")
 const { processStringSync } = require("@budibase/string-templates")
@@ -20,22 +19,7 @@ const {
 } = require("../integrations/queries/sql")
 
 class QueryRunner {
-  datasource: any
-  queryVerb: string
-  queryId: string
-  fields: any
-  parameters: any
-  pagination: any
-  transformer: any
-  cachedVariables: any[]
-  ctx: any
-  queryResponse: any
-  noRecursiveQuery: boolean
-  hasRerun: boolean
-  hasRefreshedOAuth: boolean
-  hasDynamicVariables: boolean
-
-  constructor(input: QueryEvent, flags = { noRecursiveQuery: false }) {
+  constructor(input, flags = { noRecursiveQuery: false }) {
     this.datasource = input.datasource
     this.queryVerb = input.queryVerb
     this.fields = input.fields
@@ -53,10 +37,9 @@ class QueryRunner {
     this.queryResponse = {}
     this.hasRerun = false
     this.hasRefreshedOAuth = false
-    this.hasDynamicVariables = false
   }
 
-  async execute(): Promise<any> {
+  async execute() {
     let { datasource, fields, queryVerb, transformer } = this
 
     let datasourceClone = cloneDeep(datasource)
@@ -69,7 +52,7 @@ class QueryRunner {
 
     if (datasourceClone.config.authConfigs) {
       datasourceClone.config.authConfigs =
-        datasourceClone.config.authConfigs.map((config: any) => {
+        datasourceClone.config.authConfigs.map(config => {
           return enrichQueryFields(config, this.ctx)
         })
     }
@@ -155,8 +138,8 @@ class QueryRunner {
     }
 
     // map into JSON if just raw primitive here
-    if (rows.find((row: any) => typeof row !== "object")) {
-      rows = rows.map((value: any) => ({ value }))
+    if (rows.find(row => typeof row !== "object")) {
+      rows = rows.map(value => ({ value }))
     }
 
     // get all the potential fields in the schema
@@ -169,7 +152,7 @@ class QueryRunner {
     return { rows, keys, info, extra, pagination }
   }
 
-  async runAnotherQuery(queryId: string, parameters: any) {
+  async runAnotherQuery(queryId, parameters) {
     const db = getAppDB()
     const query = await db.get(queryId)
     const datasource = await db.get(query.datasourceId)
@@ -180,13 +163,12 @@ class QueryRunner {
         fields: query.fields,
         parameters,
         transformer: query.transformer,
-        queryId,
       },
       { noRecursiveQuery: true }
     ).execute()
   }
 
-  async refreshOAuth2(ctx: any) {
+  async refreshOAuth2(ctx) {
     const { oauth2, providerType, _id } = ctx.user
     const { configId } = ctx.auth
 
@@ -218,7 +200,7 @@ class QueryRunner {
     return resp
   }
 
-  async getDynamicVariable(variable: QueryVariable) {
+  async getDynamicVariable(variable) {
     let { parameters } = this
     const queryId = variable.queryId,
       name = variable.name
@@ -251,7 +233,7 @@ class QueryRunner {
     if (!this.noRecursiveQuery) {
       // need to see if this uses any variables
       const stringFields = JSON.stringify(fields)
-      const foundVars = dynamicVars.filter((variable: QueryVariable) => {
+      const foundVars = dynamicVars.filter(variable => {
         // don't allow a query to use its own dynamic variable (loop)
         if (variable.queryId === this.queryId) {
           return false
@@ -260,9 +242,7 @@ class QueryRunner {
         const regex = new RegExp(`{{[ ]*${variable.name}[ ]*}}`)
         return regex.test(stringFields)
       })
-      const dynamics = foundVars.map((dynVar: QueryVariable) =>
-        this.getDynamicVariable(dynVar)
-      )
+      const dynamics = foundVars.map(dynVar => this.getDynamicVariable(dynVar))
       const responses = await Promise.all(dynamics)
       for (let i = 0; i < foundVars.length; i++) {
         const variable = foundVars[i]
@@ -284,7 +264,7 @@ class QueryRunner {
   }
 }
 
-export function execute(input: QueryEvent, callback: WorkerCallback) {
+module.exports = (input, callback) => {
   doInAppContext(input.appId, async () => {
     const Runner = new QueryRunner(input)
     try {

package/src/threads/{utils.ts → utils.js}

@@ -1,11 +1,10 @@
-import { QueryVariable } from "./definitions"
 const env = require("../environment")
 const db = require("../db")
 const redis = require("@budibase/backend-core/redis")
 const { SEPARATOR } = require("@budibase/backend-core/db")
 
 const VARIABLE_TTL_SECONDS = 3600
-let client: any
+let client
 
 async function getClient() {
   if (!client) {
@@ -15,16 +14,10 @@ async function getClient() {
 }
 
 process.on("exit", async () => {
-  if (client) {
-    await client.finish()
-  }
+  if (client) await client.finish()
 })
 
-function makeVariableKey(queryId: string, variable: string) {
-  return `${queryId}${SEPARATOR}${variable}`
-}
-
-export function threadSetup() {
+exports.threadSetup = () => {
   // don't run this if not threading
   if (env.isTest() || env.DISABLE_THREADING) {
     return
@@ -34,15 +27,16 @@ export function threadSetup() {
   db.init()
 }
 
-export async function checkCacheForDynamicVariable(
-  queryId: string,
-  variable: string
-) {
+function makeVariableKey(queryId, variable) {
+  return `${queryId}${SEPARATOR}${variable}`
+}
+
+exports.checkCacheForDynamicVariable = async (queryId, variable) => {
   const cache = await getClient()
   return cache.get(makeVariableKey(queryId, variable))
 }
 
-export async function invalidateDynamicVariables(cachedVars: QueryVariable[]) {
+exports.invalidateDynamicVariables = async cachedVars => {
   const cache = await getClient()
   let promises = []
   for (let variable of cachedVars) {
@@ -53,11 +47,7 @@ export async function invalidateDynamicVariables(cachedVars: QueryVariable[]) {
   await Promise.all(promises)
 }
 
-export async function storeDynamicVariable(
-  queryId: string,
-  variable: string,
-  value: any
-) {
+exports.storeDynamicVariable = async (queryId, variable, value) => {
   const cache = await getClient()
   await cache.store(
     makeVariableKey(queryId, variable),
@@ -66,7 +56,7 @@ export async function storeDynamicVariable(
   )
 }
 
-export function formatResponse(resp: any) {
+exports.formatResponse = resp => {
   if (typeof resp === "string") {
     try {
       resp = JSON.parse(resp)
@@ -77,7 +67,7 @@ export function formatResponse(resp: any) {
   return resp
 }
 
-export function hasExtraData(response: any) {
+exports.hasExtraData = response => {
   return (
     typeof response === "object" &&
     !Array.isArray(response) &&
@@ -86,12 +76,3 @@ export function hasExtraData(response: any) {
     response.info != null
   )
 }
-
-export default {
-  hasExtraData,
-  formatResponse,
-  storeDynamicVariable,
-  invalidateDynamicVariables,
-  checkCacheForDynamicVariable,
-  threadSetup,
-}