@budibase/server 1.1.32 → 1.1.33-alpha.0

package/src/threads/{automation.js → automation.ts}

@@ -1,36 +1,47 @@
-require("./utils").threadSetup()
-const actions = require("../automations/actions")
-const automationUtils = require("../automations/automationUtils")
-const AutomationEmitter = require("../events/AutomationEmitter")
-const { processObject } = require("@budibase/string-templates")
-const { DocumentTypes } = require("../db/utils")
-const { definitions: triggerDefs } = require("../automations/triggerInfo")
+import { default as threadUtils } from "./utils"
+threadUtils.threadSetup()
+import { isRecurring, disableCron, isErrorInOutput } from "../automations/utils"
+import { default as actions } from "../automations/actions"
+import { default as automationUtils } from "../automations/automationUtils"
+import { default as AutomationEmitter } from "../events/AutomationEmitter"
+import { generateAutomationMetadataID, isProdAppID } from "../db/utils"
+import { definitions as triggerDefs } from "../automations/triggerInfo"
+import { AutomationErrors, MAX_AUTOMATION_RECURRING_ERRORS } from "../constants"
+import { storeLog } from "../automations/logging"
+import { Automation, AutomationStep, AutomationStatus } from "@budibase/types"
+import {
+  LoopStep,
+  LoopStepTypes,
+  LoopInput,
+  AutomationEvent,
+  TriggerOutput,
+  AutomationContext,
+  AutomationMetadata,
+} from "../definitions/automations"
+import { WorkerCallback } from "./definitions"
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
-const { AutomationErrors, LoopStepTypes } = require("../constants")
-const { storeLog } = require("../automations/logging")
+const { logAlertWithInfo, logWarn } = require("@budibase/backend-core/logging")
+const { processObject } = require("@budibase/string-templates")
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
 const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
-
 const CRON_STEP_ID = triggerDefs.CRON.stepId
-const STOPPED_STATUS = { success: true, status: "STOPPED" }
+const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED }
 const { cloneDeep } = require("lodash/fp")
 const env = require("../environment")
 
-function typecastForLooping(loopStep, input) {
+function typecastForLooping(loopStep: LoopStep, input: LoopInput) {
   if (!input || !input.binding) {
     return null
   }
-  const isArray = Array.isArray(input.binding),
-    isString = typeof input.binding === "string"
   try {
     switch (loopStep.inputs.option) {
       case LoopStepTypes.ARRAY:
-        if (isString) {
+        if (typeof input.binding === "string") {
           return JSON.parse(input.binding)
         }
         break
       case LoopStepTypes.STRING:
-        if (isArray) {
+        if (Array.isArray(input.binding)) {
           return input.binding.join(",")
         }
         break
@@ -41,7 +52,7 @@ function typecastForLooping(loopStep, input) {
   return input.binding
 }
 
-function getLoopIterations(loopStep, input) {
+function getLoopIterations(loopStep: LoopStep, input: LoopInput) {
   const binding = typecastForLooping(loopStep, input)
   if (!loopStep || !binding) {
     return 1
@@ -57,11 +68,24 @@ function getLoopIterations(loopStep, input) {
  * inputs and handles any outputs.
  */
 class Orchestrator {
-  constructor(automation, triggerOutput = {}) {
-    this._metadata = triggerOutput.metadata
-    this._chainCount = this._metadata ? this._metadata.automationChainCount : 0
-    this._appId = triggerOutput.appId
-    this._app = null
+  _chainCount: number
+  _appId: string
+  _automation: Automation
+  _emitter: any
+  _context: AutomationContext
+  _repeat?: { jobId: string; jobKey: string }
+  executionOutput: AutomationContext
+
+  constructor(automation: Automation, triggerOutput: TriggerOutput, opts: any) {
+    const metadata = triggerOutput.metadata
+    this._chainCount = metadata ? metadata.automationChainCount : 0
+    this._appId = triggerOutput.appId as string
+    if (opts?.repeat) {
+      this._repeat = {
+        jobId: opts.repeat.jobId,
+        jobKey: opts.repeat.key,
+      }
+    }
     const triggerStepId = automation.definition.trigger.stepId
     triggerOutput = this.cleanupTriggerOutputs(triggerStepId, triggerOutput)
     // remove from context
@@ -79,14 +103,14 @@ class Orchestrator {
     this.updateExecutionOutput(triggerId, triggerStepId, null, triggerOutput)
   }
 
-  cleanupTriggerOutputs(stepId, triggerOutput) {
+  cleanupTriggerOutputs(stepId: string, triggerOutput: TriggerOutput) {
     if (stepId === CRON_STEP_ID) {
       triggerOutput.timestamp = Date.now()
     }
     return triggerOutput
   }
 
-  async getStepFunctionality(stepId) {
+  async getStepFunctionality(stepId: string) {
     let step = await actions.getAction(stepId)
     if (step == null) {
       throw `Cannot find automation step by name ${stepId}`
@@ -94,25 +118,107 @@ class Orchestrator {
     return step
   }
 
-  async getApp() {
-    if (this._app) {
-      return this._app
+  async getMetadata(): Promise<AutomationMetadata> {
+    const metadataId = generateAutomationMetadataID(this._automation._id)
+    const db = getAppDB()
+    let metadata: AutomationMetadata
+    try {
+      metadata = await db.get(metadataId)
+    } catch (err) {
+      metadata = {
+        _id: metadataId,
+        errorCount: 0,
+      }
+    }
+    return metadata
+  }
+
+  async checkIfShouldStop(metadata: AutomationMetadata): Promise<boolean> {
+    if (!metadata.errorCount || !this._repeat) {
+      return false
+    }
+    const automation = this._automation
+    const trigger = automation.definition.trigger
+    if (metadata.errorCount >= MAX_AUTOMATION_RECURRING_ERRORS) {
+      logWarn(
+        `CRON disabled due to errors - ${this._appId}/${this._automation._id}`
+      )
+      await disableCron(this._repeat?.jobId, this._repeat?.jobKey)
+      this.updateExecutionOutput(
+        trigger.id,
+        trigger.stepId,
+        {},
+        {
+          status: AutomationStatus.STOPPED_ERROR,
+          success: false,
+        }
+      )
+      await storeLog(automation, this.executionOutput)
+      return true
+    }
+    return false
+  }
+
+  async updateMetadata(metadata: AutomationMetadata) {
+    const output = this.executionOutput,
+      automation = this._automation
+    if (!output || !isRecurring(automation)) {
+      return
+    }
+    const count = metadata.errorCount
+    const isError = isErrorInOutput(output)
+    // nothing to do in this scenario, escape
+    if (!count && !isError) {
+      return
+    }
+    if (isError) {
+      metadata.errorCount = count ? count + 1 : 1
+    } else {
+      metadata.errorCount = 0
     }
     const db = getAppDB()
-    this._app = await db.get(DocumentTypes.APP_METADATA)
-    return this._app
+    try {
+      await db.put(metadata)
+    } catch (err) {
+      logAlertWithInfo(
+        "Failed to write automation metadata",
+        db.name,
+        automation._id,
+        err
+      )
+    }
   }
 
-  updateExecutionOutput(id, stepId, inputs, outputs) {
+  updateExecutionOutput(id: string, stepId: string, inputs: any, outputs: any) {
     const stepObj = { id, stepId, inputs, outputs }
+    // replacing trigger when disabling CRON
+    if (
+      stepId === CRON_STEP_ID &&
+      outputs.status === AutomationStatus.STOPPED_ERROR
+    ) {
+      this.executionOutput.trigger = stepObj
+      this.executionOutput.steps = [stepObj]
+      return
+    }
     // first entry is always the trigger (constructor)
-    if (this.executionOutput.steps.length === 0) {
+    if (
+      this.executionOutput.steps.length === 0 ||
+      this.executionOutput.trigger.id === id
+    ) {
       this.executionOutput.trigger = stepObj
     }
     this.executionOutput.steps.push(stepObj)
   }
 
-  updateContextAndOutput(loopStepNumber, step, output, result) {
+  updateContextAndOutput(
+    loopStepNumber: number | undefined,
+    step: AutomationStep,
+    output: any,
+    result: { success: boolean; status: string }
+  ) {
+    if (!loopStepNumber) {
+      throw new Error("No loop step number provided.")
+    }
     this.executionOutput.steps.splice(loopStepNumber, 0, {
       id: step.id,
       stepId: step.stepId,
@@ -133,11 +239,22 @@ class Orchestrator {
   async execute() {
     let automation = this._automation
     let stopped = false
-    let loopStep = null
+    let loopStep: AutomationStep | undefined = undefined
 
     let stepCount = 0
-    let loopStepNumber = null
-    let loopSteps = []
+    let loopStepNumber: any = undefined
+    let loopSteps: LoopStep[] | undefined = []
+    let metadata
+
+    // check if this is a recurring automation,
+    if (isProdAppID(this._appId) && isRecurring(automation)) {
+      metadata = await this.getMetadata()
+      const shouldStop = await this.checkIfShouldStop(metadata)
+      if (shouldStop) {
+        return
+      }
+    }
+
     for (let step of automation.definition.steps) {
       stepCount++
       let input,
@@ -151,7 +268,7 @@ class Orchestrator {
 
       if (loopStep) {
         input = await processObject(loopStep.inputs, this._context)
-        iterations = getLoopIterations(loopStep, input)
+        iterations = getLoopIterations(loopStep as LoopStep, input)
       }
 
       for (let index = 0; index < iterations; index++) {
@@ -166,14 +283,17 @@ class Orchestrator {
 
           let tempOutput = { items: loopSteps, iterations: iterationCount }
           try {
-            newInput.binding = typecastForLooping(loopStep, newInput)
+            newInput.binding = typecastForLooping(
+              loopStep as LoopStep,
+              newInput
+            )
           } catch (err) {
             this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
               status: AutomationErrors.INCORRECT_TYPE,
               success: false,
             })
-            loopSteps = null
-            loopStep = null
+            loopSteps = undefined
+            loopStep = undefined
             break
           }
 
@@ -223,8 +343,8 @@ class Orchestrator {
               status: AutomationErrors.MAX_ITERATIONS,
               success: true,
             })
-            loopSteps = null
-            loopStep = null
+            loopSteps = undefined
+            loopStep = undefined
             break
           }
 
@@ -232,7 +352,7 @@ class Orchestrator {
           const currentItem = this._context.steps[loopStepNumber]?.currentItem
           if (currentItem && typeof currentItem === "object") {
             isFailure = Object.keys(currentItem).some(value => {
-              return currentItem[value] === loopStep.inputs.failure
+              return currentItem[value] === loopStep?.inputs.failure
             })
           } else {
             isFailure = currentItem && currentItem === loopStep.inputs.failure
@@ -243,8 +363,8 @@ class Orchestrator {
               status: AutomationErrors.FAILURE_CONDITION,
               success: false,
             })
-            loopSteps = null
-            loopStep = null
+            loopSteps = undefined
+            loopStep = undefined
             break
           }
         }
@@ -295,7 +415,7 @@ class Orchestrator {
         if (loopStep) {
           iterationCount++
           if (index === iterations - 1) {
-            loopStep = null
+            loopStep = undefined
             this._context.steps.splice(loopStepNumber, 1)
             break
           }
@@ -316,22 +436,26 @@ class Orchestrator {
         })
 
         this._context.steps.splice(loopStepNumber, 0, tempOutput)
-        loopSteps = null
+        loopSteps = undefined
       }
     }
 
     // store the logs for the automation run
     await storeLog(this._automation, this.executionOutput)
+    if (isProdAppID(this._appId) && isRecurring(automation) && metadata) {
+      await this.updateMetadata(metadata)
+    }
     return this.executionOutput
   }
 }
 
-module.exports = (input, callback) => {
+export function execute(input: AutomationEvent, callback: WorkerCallback) {
   const appId = input.data.event.appId
   doInAppContext(appId, async () => {
     const automationOrchestrator = new Orchestrator(
       input.data.automation,
-      input.data.event
+      input.data.event,
+      input.opts
     )
     try {
       const response = await automationOrchestrator.execute()

package/src/threads/definitions.ts

@@ -0,0 +1,18 @@
+export type WorkerCallback = (error: any, response?: any) => void
+
+export interface QueryEvent {
+  appId?: string
+  datasource: any
+  queryVerb: string
+  fields: { [key: string]: any }
+  parameters: { [key: string]: any }
+  pagination?: any
+  transformer: any
+  queryId: string
+  ctx?: any
+}
+
+export interface QueryVariable {
+  queryId: string
+  name: string
+}

package/src/threads/index.ts

@@ -18,26 +18,23 @@ function typeToFile(type: any) {
     default:
       throw "Unknown thread type"
   }
+  // have to use require here, to make it work with worker-farm
   return require.resolve(filename)
 }
 
 export class Thread {
   type: any
   count: any
-  disableThreading: any
   workers: any
   timeoutMs: any
+  disableThreading: boolean
 
   static workerRefs: any[] = []
 
   constructor(type: any, opts: any = { timeoutMs: null, count: 1 }) {
     this.type = type
     this.count = opts.count ? opts.count : 1
-    this.disableThreading =
-      env.isTest() ||
-      env.DISABLE_THREADING ||
-      this.count === 0 ||
-      env.isInThread()
+    this.disableThreading = this.shouldDisableThreading()
     if (!this.disableThreading) {
       const workerOpts: any = {
         autoStart: true,
@@ -47,33 +44,44 @@ export class Thread {
         this.timeoutMs = opts.timeoutMs
         workerOpts.maxCallTime = opts.timeoutMs
       }
-      this.workers = workerFarm(workerOpts, typeToFile(type))
+      this.workers = workerFarm(workerOpts, typeToFile(type), ["execute"])
       Thread.workerRefs.push(this.workers)
     }
   }
 
+  shouldDisableThreading(): boolean {
+    return !!(
+      env.isTest() ||
+      env.DISABLE_THREADING ||
+      this.count === 0 ||
+      env.isInThread()
+    )
+  }
+
   run(data: any) {
+    const timeout = this.timeoutMs
     return new Promise((resolve, reject) => {
-      let fncToCall
+      function fire(worker: any) {
+        worker.execute(data, (err: any, response: any) => {
+          if (err && err.type === "TimeoutError") {
+            reject(
+              new Error(`Query response time exceeded ${timeout}ms timeout.`)
+            )
+          } else if (err) {
+            reject(err)
+          } else {
+            resolve(response)
+          }
+        })
+      }
       // if in test then don't use threading
       if (this.disableThreading) {
-        fncToCall = require(typeToFile(this.type))
+        import(typeToFile(this.type)).then((thread: any) => {
+          fire(thread)
+        })
       } else {
-        fncToCall = this.workers
+        fire(this.workers)
       }
-      fncToCall(data, (err: any, response: any) => {
-        if (err && err.type === "TimeoutError") {
-          reject(
-            new Error(
-              `Query response time exceeded ${this.timeoutMs}ms timeout.`
-            )
-          )
-        } else if (err) {
-          reject(err)
-        } else {
-          resolve(response)
-        }
-      })
     })
   }
 

package/src/threads/{query.js → query.ts}

@@ -1,5 +1,6 @@
-const threadUtils = require("./utils")
+import { default as threadUtils } from "./utils"
 threadUtils.threadSetup()
+import { WorkerCallback, QueryEvent, QueryVariable } from "./definitions"
 const ScriptRunner = require("../utilities/scriptRunner")
 const { integrations } = require("../integrations")
 const { processStringSync } = require("@budibase/string-templates")
@@ -19,7 +20,22 @@ const {
 } = require("../integrations/queries/sql")
 
 class QueryRunner {
-  constructor(input, flags = { noRecursiveQuery: false }) {
+  datasource: any
+  queryVerb: string
+  queryId: string
+  fields: any
+  parameters: any
+  pagination: any
+  transformer: any
+  cachedVariables: any[]
+  ctx: any
+  queryResponse: any
+  noRecursiveQuery: boolean
+  hasRerun: boolean
+  hasRefreshedOAuth: boolean
+  hasDynamicVariables: boolean
+
+  constructor(input: QueryEvent, flags = { noRecursiveQuery: false }) {
     this.datasource = input.datasource
     this.queryVerb = input.queryVerb
     this.fields = input.fields
@@ -37,9 +53,10 @@ class QueryRunner {
     this.queryResponse = {}
     this.hasRerun = false
     this.hasRefreshedOAuth = false
+    this.hasDynamicVariables = false
   }
 
-  async execute() {
+  async execute(): Promise<any> {
     let { datasource, fields, queryVerb, transformer } = this
 
     let datasourceClone = cloneDeep(datasource)
@@ -52,7 +69,7 @@ class QueryRunner {
 
     if (datasourceClone.config.authConfigs) {
       datasourceClone.config.authConfigs =
-        datasourceClone.config.authConfigs.map(config => {
+        datasourceClone.config.authConfigs.map((config: any) => {
           return enrichQueryFields(config, this.ctx)
         })
     }
@@ -138,8 +155,8 @@ class QueryRunner {
     }
 
     // map into JSON if just raw primitive here
-    if (rows.find(row => typeof row !== "object")) {
-      rows = rows.map(value => ({ value }))
+    if (rows.find((row: any) => typeof row !== "object")) {
+      rows = rows.map((value: any) => ({ value }))
     }
 
     // get all the potential fields in the schema
@@ -152,7 +169,7 @@ class QueryRunner {
     return { rows, keys, info, extra, pagination }
   }
 
-  async runAnotherQuery(queryId, parameters) {
+  async runAnotherQuery(queryId: string, parameters: any) {
     const db = getAppDB()
     const query = await db.get(queryId)
     const datasource = await db.get(query.datasourceId)
@@ -163,12 +180,13 @@ class QueryRunner {
         fields: query.fields,
         parameters,
         transformer: query.transformer,
+        queryId,
       },
       { noRecursiveQuery: true }
     ).execute()
   }
 
-  async refreshOAuth2(ctx) {
+  async refreshOAuth2(ctx: any) {
     const { oauth2, providerType, _id } = ctx.user
     const { configId } = ctx.auth
 
@@ -200,7 +218,7 @@ class QueryRunner {
     return resp
   }
 
-  async getDynamicVariable(variable) {
+  async getDynamicVariable(variable: QueryVariable) {
     let { parameters } = this
     const queryId = variable.queryId,
       name = variable.name
@@ -233,7 +251,7 @@ class QueryRunner {
     if (!this.noRecursiveQuery) {
       // need to see if this uses any variables
       const stringFields = JSON.stringify(fields)
-      const foundVars = dynamicVars.filter(variable => {
+      const foundVars = dynamicVars.filter((variable: QueryVariable) => {
         // don't allow a query to use its own dynamic variable (loop)
         if (variable.queryId === this.queryId) {
           return false
@@ -242,7 +260,9 @@ class QueryRunner {
         const regex = new RegExp(`{{[ ]*${variable.name}[ ]*}}`)
         return regex.test(stringFields)
       })
-      const dynamics = foundVars.map(dynVar => this.getDynamicVariable(dynVar))
+      const dynamics = foundVars.map((dynVar: QueryVariable) =>
+        this.getDynamicVariable(dynVar)
+      )
       const responses = await Promise.all(dynamics)
       for (let i = 0; i < foundVars.length; i++) {
         const variable = foundVars[i]
@@ -264,7 +284,7 @@ class QueryRunner {
   }
 }
 
-module.exports = (input, callback) => {
+export function execute(input: QueryEvent, callback: WorkerCallback) {
   doInAppContext(input.appId, async () => {
     const Runner = new QueryRunner(input)
     try {

package/src/threads/{utils.js → utils.ts}

@@ -1,10 +1,11 @@
+import { QueryVariable } from "./definitions"
 const env = require("../environment")
 const db = require("../db")
 const redis = require("@budibase/backend-core/redis")
 const { SEPARATOR } = require("@budibase/backend-core/db")
 
 const VARIABLE_TTL_SECONDS = 3600
-let client
+let client: any
 
 async function getClient() {
   if (!client) {
@@ -14,10 +15,16 @@ async function getClient() {
 }
 
 process.on("exit", async () => {
-  if (client) await client.finish()
+  if (client) {
+    await client.finish()
+  }
 })
 
-exports.threadSetup = () => {
+function makeVariableKey(queryId: string, variable: string) {
+  return `${queryId}${SEPARATOR}${variable}`
+}
+
+export function threadSetup() {
   // don't run this if not threading
   if (env.isTest() || env.DISABLE_THREADING) {
     return
@@ -27,16 +34,15 @@ exports.threadSetup = () => {
   db.init()
 }
 
-function makeVariableKey(queryId, variable) {
-  return `${queryId}${SEPARATOR}${variable}`
-}
-
-exports.checkCacheForDynamicVariable = async (queryId, variable) => {
+export async function checkCacheForDynamicVariable(
+  queryId: string,
+  variable: string
+) {
   const cache = await getClient()
   return cache.get(makeVariableKey(queryId, variable))
 }
 
-exports.invalidateDynamicVariables = async cachedVars => {
+export async function invalidateDynamicVariables(cachedVars: QueryVariable[]) {
   const cache = await getClient()
   let promises = []
   for (let variable of cachedVars) {
@@ -47,7 +53,11 @@ exports.invalidateDynamicVariables = async cachedVars => {
   await Promise.all(promises)
 }
 
-exports.storeDynamicVariable = async (queryId, variable, value) => {
+export async function storeDynamicVariable(
+  queryId: string,
+  variable: string,
+  value: any
+) {
   const cache = await getClient()
   await cache.store(
     makeVariableKey(queryId, variable),
@@ -56,7 +66,7 @@ exports.storeDynamicVariable = async (queryId, variable, value) => {
   )
 }
 
-exports.formatResponse = resp => {
+export function formatResponse(resp: any) {
   if (typeof resp === "string") {
     try {
       resp = JSON.parse(resp)
@@ -67,7 +77,7 @@ exports.formatResponse = resp => {
   return resp
 }
 
-exports.hasExtraData = response => {
+export function hasExtraData(response: any) {
   return (
     typeof response === "object" &&
     !Array.isArray(response) &&
@@ -76,3 +86,12 @@ exports.hasExtraData = response => {
     response.info != null
   )
 }
+
+export default {
+  hasExtraData,
+  formatResponse,
+  storeDynamicVariable,
+  invalidateDynamicVariables,
+  checkCacheForDynamicVariable,
+  threadSetup,
+}