@budibase/server 2.6.19-alpha.5 → 2.6.19-alpha.51

package/src/integrations/googlesheets.ts

@@ -66,10 +66,10 @@ const SCHEMA: Integration = {
     "Create and collaborate on online spreadsheets in real-time and from any device.",
   friendlyName: "Google Sheets",
   type: "Spreadsheet",
-  features: [
-    DatasourceFeature.CONNECTION_CHECKING,
-    DatasourceFeature.FETCH_TABLE_NAMES,
-  ],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+    [DatasourceFeature.FETCH_TABLE_NAMES]: true,
+  },
   datasource: {
     spreadsheetId: {
       display: "Google Sheet URL",
@@ -457,22 +457,41 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   }) {
     try {
       await this.connect()
+      const hasFilters = dataFilters.hasFilters(query.filters)
+      const limit = query.paginate?.limit || 100
+      const page: number =
+        typeof query.paginate?.page === "number"
+          ? query.paginate.page
+          : parseInt(query.paginate?.page || "1")
+      const offset = (page - 1) * limit
       const sheet = this.client.sheetsByTitle[query.sheet]
       let rows: GoogleSpreadsheetRow[] = []
-      if (query.paginate) {
-        const limit = query.paginate.limit || 100
-        let page: number =
-          typeof query.paginate.page === "number"
-            ? query.paginate.page
-            : parseInt(query.paginate.page || "1")
+      if (query.paginate && !hasFilters) {
         rows = await sheet.getRows({
           limit,
-          offset: (page - 1) * limit,
+          offset,
         })
       } else {
         rows = await sheet.getRows()
       }
-      const filtered = dataFilters.runLuceneQuery(rows, query.filters)
+      // this is a special case - need to handle the _id, it doesn't exist
+      // we cannot edit the returned structure from google, it does not have
+      // setter functions and is immutable, easier to update the filters
+      // to look for the _rowNumber property rather than rowNumber
+      if (query.filters?.equal) {
+        const idFilterKeys = Object.keys(query.filters.equal).filter(filter =>
+          filter.includes(GOOGLE_SHEETS_PRIMARY_KEY)
+        )
+        for (let idFilterKey of idFilterKeys) {
+          const id = query.filters.equal[idFilterKey]
+          delete query.filters.equal[idFilterKey]
+          query.filters.equal[`_${GOOGLE_SHEETS_PRIMARY_KEY}`] = id
+        }
+      }
+      let filtered = dataFilters.runLuceneQuery(rows, query.filters)
+      if (hasFilters && query.paginate) {
+        filtered = filtered.slice(offset, offset + limit)
+      }
       const headerValues = sheet.headerValues
       let response = []
       for (let row of filtered) {
@@ -535,7 +554,12 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     const row = rows[query.rowIndex]
     if (row) {
       await row.delete()
-      return [{ deleted: query.rowIndex }]
+      return [
+        {
+          deleted: query.rowIndex,
+          [GOOGLE_SHEETS_PRIMARY_KEY]: query.rowIndex,
+        },
+      ]
     } else {
       throw new Error("Row does not exist.")
     }

package/src/integrations/microsoftSqlServer.ts

@@ -40,10 +40,10 @@ const SCHEMA: Integration = {
     "Microsoft SQL Server is a relational database management system developed by Microsoft. ",
   friendlyName: "MS SQL Server",
   type: "Relational",
-  features: [
-    DatasourceFeature.CONNECTION_CHECKING,
-    DatasourceFeature.FETCH_TABLE_NAMES,
-  ],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+    [DatasourceFeature.FETCH_TABLE_NAMES]: true,
+  },
   datasource: {
     user: {
       type: DatasourceFieldType.STRING,

package/src/integrations/mongodb.ts

@@ -40,7 +40,9 @@ const getSchema = () => {
     type: "Non-relational",
     description:
       "MongoDB is a general purpose, document-based, distributed database built for modern application developers and for the cloud era.",
-    features: [DatasourceFeature.CONNECTION_CHECKING],
+    features: {
+      [DatasourceFeature.CONNECTION_CHECKING]: true,
+    },
     datasource: {
       connectionString: {
         type: DatasourceFieldType.STRING,

package/src/integrations/mysql.ts

@@ -36,10 +36,10 @@ const SCHEMA: Integration = {
   type: "Relational",
   description:
     "MySQL Database Service is a fully managed database service to deploy cloud-native applications. ",
-  features: [
-    DatasourceFeature.CONNECTION_CHECKING,
-    DatasourceFeature.FETCH_TABLE_NAMES,
-  ],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+    [DatasourceFeature.FETCH_TABLE_NAMES]: true,
+  },
   datasource: {
     host: {
       type: DatasourceFieldType.STRING,

package/src/integrations/oracle.ts

@@ -50,10 +50,10 @@ const SCHEMA: Integration = {
   type: "Relational",
   description:
     "Oracle Database is an object-relational database management system developed by Oracle Corporation",
-  features: [
-    DatasourceFeature.CONNECTION_CHECKING,
-    DatasourceFeature.FETCH_TABLE_NAMES,
-  ],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+    [DatasourceFeature.FETCH_TABLE_NAMES]: true,
+  },
   datasource: {
     host: {
       type: DatasourceFieldType.STRING,

package/src/integrations/postgres.ts

@@ -52,10 +52,10 @@ const SCHEMA: Integration = {
   type: "Relational",
   description:
     "PostgreSQL, also known as Postgres, is a free and open-source relational database management system emphasizing extensibility and SQL compliance.",
-  features: [
-    DatasourceFeature.CONNECTION_CHECKING,
-    DatasourceFeature.FETCH_TABLE_NAMES,
-  ],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+    [DatasourceFeature.FETCH_TABLE_NAMES]: true,
+  },
   datasource: {
     host: {
       type: DatasourceFieldType.STRING,

package/src/integrations/redis.ts

@@ -21,7 +21,9 @@ const SCHEMA: Integration = {
     "Redis is a caching tool, providing powerful key-value store capabilities.",
   friendlyName: "Redis",
   type: "Non-relational",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     host: {
       type: "string",
@@ -175,7 +177,13 @@ class RedisIntegration {
       const pipeline = this.client.pipeline(pipelineCommands)
       const result = await pipeline.exec()
 
-      return result.map((output: string | string[]) => output[1])
+      return result?.map((output: any) => {
+        if (typeof output === "string") {
+          return output
+        } else if (Array.isArray(output)) {
+          return output[1]
+        }
+      })
     })
   }
 }

package/src/integrations/s3.ts

@@ -24,7 +24,9 @@ const SCHEMA: Integration = {
     "Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.",
   friendlyName: "Amazon S3",
   type: "Object store",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     region: {
       type: "string",

package/src/integrations/snowflake.ts

@@ -22,7 +22,9 @@ const SCHEMA: Integration = {
     "Snowflake is a solution for data warehousing, data lakes, data engineering, data science, data application development, and securely sharing and consuming shared data.",
   friendlyName: "Snowflake",
   type: "Relational",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     account: {
       type: "string",

package/src/middleware/builder.ts

@@ -9,8 +9,8 @@ import {
   checkDebounce,
   setDebounce,
 } from "../utilities/redis"
-import { db as dbCore, cache, permissions } from "@budibase/backend-core"
-import { BBContext, Database } from "@budibase/types"
+import { db as dbCore, cache } from "@budibase/backend-core"
+import { UserCtx, Database } from "@budibase/types"
 
 const DEBOUNCE_TIME_SEC = 30
 
@@ -23,7 +23,7 @@ const DEBOUNCE_TIME_SEC = 30
  * through the authorized middleware                *
  ****************************************************/
 
-async function checkDevAppLocks(ctx: BBContext) {
+async function checkDevAppLocks(ctx: UserCtx) {
   const appId = ctx.appId
 
   // if any public usage, don't proceed
@@ -35,15 +35,14 @@ async function checkDevAppLocks(ctx: BBContext) {
   if (!appId || !appId.startsWith(APP_DEV_PREFIX)) {
     return
   }
-  if (!(await doesUserHaveLock(appId, ctx.user))) {
-    ctx.throw(400, "User does not hold app lock.")
-  }
 
-  // they do have lock, update it
-  await updateLock(appId, ctx.user)
+  // If this user already owns the lock, then update it
+  if (await doesUserHaveLock(appId, ctx.user)) {
+    await updateLock(appId, ctx.user)
+  }
 }
 
-async function updateAppUpdatedAt(ctx: BBContext) {
+async function updateAppUpdatedAt(ctx: UserCtx) {
   const appId = ctx.appId
   // if debouncing skip this update
   // get methods also aren't updating
@@ -51,20 +50,29 @@ async function updateAppUpdatedAt(ctx: BBContext) {
     return
   }
   await dbCore.doWithDB(appId, async (db: Database) => {
-    const metadata = await db.get(DocumentType.APP_METADATA)
-    metadata.updatedAt = new Date().toISOString()
+    try {
+      const metadata = await db.get(DocumentType.APP_METADATA)
+      metadata.updatedAt = new Date().toISOString()
 
-    metadata.updatedBy = getGlobalIDFromUserMetadataID(ctx.user?.userId!)
+      metadata.updatedBy = getGlobalIDFromUserMetadataID(ctx.user?.userId!)
 
-    const response = await db.put(metadata)
-    metadata._rev = response.rev
-    await cache.app.invalidateAppMetadata(appId, metadata)
-    // set a new debounce record with a short TTL
-    await setDebounce(appId, DEBOUNCE_TIME_SEC)
+      const response = await db.put(metadata)
+      metadata._rev = response.rev
+      await cache.app.invalidateAppMetadata(appId, metadata)
+      // set a new debounce record with a short TTL
+      await setDebounce(appId, DEBOUNCE_TIME_SEC)
+    } catch (err: any) {
+      // if a 409 occurs, then multiple clients connected at the same time - ignore
+      if (err?.status === 409) {
+        return
+      } else {
+        throw err
+      }
+    }
   })
 }
 
-export default async function builder(ctx: BBContext) {
+export default async function builder(ctx: UserCtx) {
   const appId = ctx.appId
   // this only functions within an app context
   if (!appId) {

package/src/sdk/app/automations/index.ts

@@ -1,5 +1,7 @@
 import * as webhook from "./webhook"
+import * as utils from "./utils"
 
 export default {
   webhook,
+  utils,
 }

package/src/sdk/app/automations/utils.ts

@@ -0,0 +1,7 @@
+import { Automation, AutomationActionStepId } from "@budibase/types"
+
+export function checkForCollectStep(automation: Automation) {
+  return automation.definition.steps.some(
+    (step: any) => step.stepId === AutomationActionStepId.COLLECT
+  )
+}

package/src/sdk/plugins/plugins.ts

@@ -36,6 +36,6 @@ export async function processUploaded(plugin: FileType, source?: PluginSource) {
   }
 
   const doc = await pro.plugins.storePlugin(metadata, directory, source)
-  clientAppSocket.emit("plugin-update", { name: doc.name, hash: doc.hash })
+  clientAppSocket?.emit("plugin-update", { name: doc.name, hash: doc.hash })
   return doc
 }

package/src/startup.ts

@@ -16,6 +16,7 @@ import * as bullboard from "./automations/bullboard"
 import * as pro from "@budibase/pro"
 import * as api from "./api"
 import sdk from "./sdk"
+import { initialise as initialiseWebsockets } from "./websockets"
 
 let STARTUP_RAN = false
 
@@ -64,6 +65,7 @@ export async function startup(app?: any, server?: any) {
   fileSystem.init()
   await redis.init()
   eventInit()
+  initialiseWebsockets(app, server)
 
   // run migrations on startup if not done via http
   // not recommended in a clustered environment

package/src/tests/utilities/TestConfiguration.ts

@@ -373,7 +373,7 @@ class TestConfiguration {
 
   // HEADERS
 
-  defaultHeaders(extras = {}) {
+  defaultHeaders(extras = {}, prodApp = false) {
     const tenantId = this.getTenantId()
     const authObj: AuthToken = {
       userId: this.defaultUserValues.globalUserId,
@@ -390,7 +390,9 @@ class TestConfiguration {
       ...extras,
     }
 
-    if (this.appId) {
+    if (prodApp) {
+      headers[constants.Header.APP_ID] = this.prodAppId
+    } else if (this.appId) {
       headers[constants.Header.APP_ID] = this.appId
     }
     return headers

package/src/tests/utilities/structures.ts

@@ -199,6 +199,48 @@ export function loopAutomation(tableId: string, loopOpts?: any): Automation {
   return automation as Automation
 }
 
+export function collectAutomation(tableId?: string): Automation {
+  const automation: any = {
+    name: "looping",
+    type: "automation",
+    definition: {
+      steps: [
+        {
+          id: "b",
+          type: "ACTION",
+          internal: true,
+          stepId: AutomationActionStepId.EXECUTE_SCRIPT,
+          inputs: {
+            code: "return [1,2,3]",
+          },
+          schema: BUILTIN_ACTION_DEFINITIONS.EXECUTE_SCRIPT.schema,
+        },
+        {
+          id: "c",
+          type: "ACTION",
+          internal: true,
+          stepId: AutomationActionStepId.COLLECT,
+          inputs: {
+            collection: "{{ literal steps.1.value }}",
+          },
+          schema: BUILTIN_ACTION_DEFINITIONS.SERVER_LOG.schema,
+        },
+      ],
+      trigger: {
+        id: "a",
+        type: "TRIGGER",
+        event: "row:save",
+        stepId: AutomationTriggerStepId.ROW_SAVED,
+        inputs: {
+          tableId,
+        },
+        schema: TRIGGER_DEFINITIONS.ROW_SAVED.schema,
+      },
+    },
+  }
+  return automation as Automation
+}
+
 export function basicRow(tableId: string) {
   return {
     name: "Test Contact",

package/src/threads/automation.ts

@@ -19,6 +19,7 @@ import {
   AutomationStatus,
   AutomationMetadata,
   AutomationJob,
+  AutomationData,
 } from "@budibase/types"
 import {
   LoopStep,
@@ -30,6 +31,7 @@ import { WorkerCallback } from "./definitions"
 import { context, logging } from "@budibase/backend-core"
 import { processObject } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
+import { performance } from "perf_hooks"
 import * as sdkUtils from "../sdk/utils"
 import env from "../environment"
 const FILTER_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.FILTER.stepId
@@ -37,15 +39,23 @@ const LOOP_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.LOOP.stepId
 const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED }
 
-function getLoopIterations(loopStep: LoopStep, input: LoopInput) {
-  const binding = automationUtils.typecastForLooping(loopStep, input)
+function getLoopIterations(loopStep: LoopStep) {
+  let binding = loopStep.inputs.binding
   if (!binding) {
     return 0
   }
+  const isString = typeof binding === "string"
+  try {
+    if (isString) {
+      binding = JSON.parse(binding)
+    }
+  } catch (err) {
+    // ignore error - wasn't able to parse
+  }
   if (Array.isArray(binding)) {
     return binding.length
   }
-  if (typeof binding === "string") {
+  if (isString) {
     return automationUtils.stringSplit(binding).length
   }
   return 0
@@ -240,7 +250,9 @@ class Orchestrator {
     let loopStepNumber: any = undefined
     let loopSteps: LoopStep[] | undefined = []
     let metadata
+    let timeoutFlag = false
     let wasLoopStep = false
+    let timeout = this._job.data.event.timeout
     // check if this is a recurring automation,
     if (isProdAppID(this._appId) && isRecurring(automation)) {
       metadata = await this.getMetadata()
@@ -249,8 +261,18 @@ class Orchestrator {
         return
       }
     }
-
+    const start = performance.now()
     for (let step of automation.definition.steps) {
+      if (timeoutFlag) {
+        break
+      }
+
+      if (timeout) {
+        setTimeout(() => {
+          timeoutFlag = true
+        }, timeout || 12000)
+      }
+
       stepCount++
       let input: any,
         iterations = 1,
@@ -264,22 +286,17 @@ class Orchestrator {
 
       if (loopStep) {
         input = await processObject(loopStep.inputs, this._context)
-        iterations = getLoopIterations(loopStep as LoopStep, input)
+        iterations = getLoopIterations(loopStep as LoopStep)
       }
       for (let index = 0; index < iterations; index++) {
         let originalStepInput = cloneDeep(step.inputs)
         // Handle if the user has set a max iteration count or if it reaches the max limit set by us
         if (loopStep && input.binding) {
-          let newInput: any = await processObject(
-            loopStep.inputs,
-            cloneDeep(this._context)
-          )
-
           let tempOutput = { items: loopSteps, iterations: iterationCount }
           try {
-            newInput.binding = automationUtils.typecastForLooping(
+            loopStep.inputs.binding = automationUtils.typecastForLooping(
               loopStep as LoopStep,
-              newInput
+              loopStep.inputs as LoopInput
             )
           } catch (err) {
             this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
@@ -290,13 +307,12 @@ class Orchestrator {
             loopStep = undefined
             break
           }
-
           let item = []
           if (
             typeof loopStep.inputs.binding === "string" &&
             loopStep.inputs.option === "String"
           ) {
-            item = automationUtils.stringSplit(newInput.binding)
+            item = automationUtils.stringSplit(loopStep.inputs.binding)
           } else if (Array.isArray(loopStep.inputs.binding)) {
             item = loopStep.inputs.binding
           }
@@ -338,6 +354,7 @@ class Orchestrator {
               }
             }
           }
+
           if (
             index === env.AUTOMATION_MAX_ITERATIONS ||
             index === parseInt(loopStep.inputs.iterations)
@@ -466,8 +483,25 @@ class Orchestrator {
       }
     }
 
+    const end = performance.now()
+    const executionTime = end - start
+
+    console.info(`Execution time: ${executionTime} milliseconds`, {
+      _logKey: "automation",
+      executionTime,
+    })
+
     // store the logs for the automation run
-    await storeLog(this._automation, this.executionOutput)
+    try {
+      await storeLog(this._automation, this.executionOutput)
+    } catch (e: any) {
+      if (e.status === 413 && e.request?.data) {
+        // if content is too large we shouldn't log it
+        delete e.request.data
+        e.request.data = { message: "removed due to large size" }
+      }
+      logging.logAlert("Error writing automation log", e)
+    }
     if (isProdAppID(this._appId) && isRecurring(automation) && metadata) {
       await this.updateMetadata(metadata)
     }
@@ -475,22 +509,56 @@ class Orchestrator {
   }
 }
 
-export function execute(job: Job, callback: WorkerCallback) {
+export function execute(job: Job<AutomationData>, callback: WorkerCallback) {
   const appId = job.data.event.appId
+  const automationId = job.data.automation._id
   if (!appId) {
     throw new Error("Unable to execute, event doesn't contain app ID.")
   }
+  if (!automationId) {
+    throw new Error("Unable to execute, event doesn't contain automation ID.")
+  }
+  return context.doInAutomationContext({
+    appId,
+    automationId,
+    task: async () => {
+      const envVars = await sdkUtils.getEnvironmentVariables()
+      // put into automation thread for whole context
+      await context.doInEnvironmentContext(envVars, async () => {
+        const automationOrchestrator = new Orchestrator(job)
+        try {
+          const response = await automationOrchestrator.execute()
+          callback(null, response)
+        } catch (err) {
+          callback(err)
+        }
+      })
+    },
+  })
+}
+
+export function executeSynchronously(job: Job) {
+  const appId = job.data.event.appId
+  if (!appId) {
+    throw new Error("Unable to execute, event doesn't contain app ID.")
+  }
+
+  const timeoutPromise = new Promise((resolve, reject) => {
+    setTimeout(() => {
+      reject(new Error("Timeout exceeded"))
+    }, job.data.event.timeout || 12000)
+  })
+
   return context.doInAppContext(appId, async () => {
     const envVars = await sdkUtils.getEnvironmentVariables()
     // put into automation thread for whole context
-    await context.doInEnvironmentContext(envVars, async () => {
+    return context.doInEnvironmentContext(envVars, async () => {
       const automationOrchestrator = new Orchestrator(job)
-      try {
-        const response = await automationOrchestrator.execute()
-        callback(null, response)
-      } catch (err) {
-        callback(err)
-      }
+      const response = await Promise.race([
+        automationOrchestrator.execute(),
+        timeoutPromise,
+      ])
+      return response
     })
   })
 }

package/src/threads/index.ts

@@ -38,6 +38,9 @@ export class Thread {
     this.count = opts.count ? opts.count : 1
     this.disableThreading = this.shouldDisableThreading()
     if (!this.disableThreading) {
+      console.debug(
+        `[${env.FORKED_PROCESS_NAME}] initialising worker farm type=${type}`
+      )
       const workerOpts: any = {
         autoStart: true,
         maxConcurrentWorkers: this.count,
@@ -45,6 +48,7 @@ export class Thread {
           env: {
             ...process.env,
             FORKED_PROCESS: "1",
+            FORKED_PROCESS_NAME: type,
           },
         },
       }
@@ -54,6 +58,10 @@ export class Thread {
       }
       this.workers = workerFarm(workerOpts, typeToFile(type), ["execute"])
       Thread.workerRefs.push(this.workers)
+    } else {
+      console.debug(
+        `[${env.FORKED_PROCESS_NAME}] skipping worker farm type=${type}`
+      )
     }
   }
 
@@ -72,9 +80,7 @@ export class Thread {
       function fire(worker: any) {
         worker.execute(job, (err: any, response: any) => {
           if (err && err.type === "TimeoutError") {
-            reject(
-              new Error(`Query response time exceeded ${timeout}ms timeout.`)
-            )
+            reject(new Error(`Thread timeout exceeded ${timeout}ms timeout.`))
           } else if (err) {
             reject(err)
           } else {

package/src/threads/utils.ts

@@ -26,8 +26,10 @@ function makeVariableKey(queryId: string, variable: string) {
 export function threadSetup() {
   // don't run this if not threading
   if (env.isTest() || env.DISABLE_THREADING || !env.isInThread()) {
+    console.debug(`[${env.FORKED_PROCESS_NAME}] thread setup skipped`)
     return
   }
+  console.debug(`[${env.FORKED_PROCESS_NAME}] thread setup running`)
   db.init()
 }