@budibase/server 1.0.123 → 1.0.125

package/src/migrations/functions/usageQuotas/syncDevelopers.ts

@@ -0,0 +1,19 @@
+import { getTenantId } from "@budibase/backend-core/tenancy"
+import { utils } from "@budibase/backend-core"
+import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
+
+export const run = async () => {
+  // get developer count
+  const developerCount = await utils.getBuildersCount()
+
+  // sync developer count
+  const tenantId = getTenantId()
+  console.log(
+    `[Tenant: ${tenantId}] Syncing developer count: ${developerCount}`
+  )
+  await quotas.setUsage(
+    developerCount,
+    StaticQuotaName.DEVELOPERS,
+    QuotaUsageType.STATIC
+  )
+}

package/src/migrations/functions/usageQuotas/syncPublishedApps.ts

@@ -0,0 +1,21 @@
+import { getTenantId } from "@budibase/backend-core/tenancy"
+import { getAllApps } from "@budibase/backend-core/db"
+import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
+
+export const run = async () => {
+  // get app count
+  const opts: any = { dev: false }
+  const prodApps = await getAllApps(opts)
+  const prodAppCount = prodApps ? prodApps.length : 0
+
+  // sync app count
+  const tenantId = getTenantId()
+  console.log(
+    `[Tenant: ${tenantId}] Syncing published app count: ${prodAppCount}`
+  )
+  await quotas.setUsage(
+    prodAppCount,
+    StaticQuotaName.PUBLISHED_APPS,
+    QuotaUsageType.STATIC
+  )
+}

package/src/migrations/functions/usageQuotas/syncRows.ts

@@ -1,10 +1,9 @@
-import { getGlobalDB, getTenantId } from "@budibase/backend-core/tenancy"
+import { getTenantId } from "@budibase/backend-core/tenancy"
 import { getAllApps } from "@budibase/backend-core/db"
-import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
 import { getUniqueRows } from "../../../utilities/usageQuota/rows"
+import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
 
 export const run = async () => {
-  const db = getGlobalDB()
   // get all rows in all apps
   // @ts-ignore
   const allApps = await getAllApps({ all: true })
@@ -16,7 +15,5 @@ export const run = async () => {
   // sync row count
   const tenantId = getTenantId()
   console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
-  const usageDoc = await getUsageQuotaDoc(db)
-  usageDoc.usageQuota.rows = rowCount
-  await db.put(usageDoc)
+  await quotas.setUsage(rowCount, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
 }

package/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts

@@ -0,0 +1,34 @@
+import TestConfig from "../../../../tests/utilities/TestConfiguration"
+import * as syncApps from "../syncApps"
+import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
+
+describe("syncApps", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)
+
+  it("runs successfully", async () => {
+    return config.doInContext(null, async () => {
+      // create the usage quota doc and mock usages
+      await quotas.getQuotaUsage()
+      await quotas.setUsage(3, StaticQuotaName.APPS, QuotaUsageType.STATIC)
+
+      let usageDoc = await quotas.getQuotaUsage()
+      expect(usageDoc.usageQuota.apps).toEqual(3)
+
+      // create an extra app to test the migration
+      await config.createApp("quota-test")
+
+      // migrate
+      await syncApps.run()
+
+      // assert the migration worked
+      usageDoc = await quotas.getQuotaUsage()
+      expect(usageDoc.usageQuota.apps).toEqual(2)
+    })
+  })
+})

package/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts

@@ -0,0 +1,40 @@
+import TestConfig from "../../../../tests/utilities/TestConfiguration"
+import * as syncRows from "../syncRows"
+import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
+
+describe("syncRows", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)
+
+  it("runs successfully", async () => {
+    return config.doInContext(null, async () => {
+      // create the usage quota doc and mock usages
+      await quotas.getQuotaUsage()
+      await quotas.setUsage(300, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
+
+      let usageDoc = await quotas.getQuotaUsage()
+      expect(usageDoc.usageQuota.rows).toEqual(300)
+
+      // app 1
+      await config.createTable()
+      await config.createRow()
+      // app 2
+      await config.createApp("second-app")
+      await config.createTable()
+      await config.createRow()
+      await config.createRow()
+
+      // migrate
+      await syncRows.run()
+
+      // assert the migration worked
+      usageDoc = await quotas.getQuotaUsage()
+      expect(usageDoc.usageQuota.rows).toEqual(3)
+    })
+  })
+})

package/src/migrations/index.ts

@@ -7,6 +7,8 @@ const {
 import * as userEmailViewCasing from "./functions/userEmailViewCasing"
 import * as quota1 from "./functions/quotas1"
 import * as appUrls from "./functions/appUrls"
+import * as developerQuota from "./functions/developerQuota"
+import * as publishedAppsQuota from "./functions/publishedAppsQuota"
 
 export interface Migration {
   type: string
@@ -26,7 +28,7 @@ export interface Migration {
  */
 export interface MigrationOptions {
   tenantIds?: string[]
-  forced?: {
+  force?: {
     [type: string]: string[]
   }
 }
@@ -48,6 +50,16 @@ export const MIGRATIONS: Migration[] = [
     opts: { all: true },
     fn: appUrls.run,
   },
+  {
+    type: MIGRATION_TYPES.GLOBAL,
+    name: "developer_quota",
+    fn: developerQuota.run,
+  },
+  {
+    type: MIGRATION_TYPES.GLOBAL,
+    name: "published_apps_quota",
+    fn: publishedAppsQuota.run,
+  },
 ]
 
 export const migrate = async (options?: MigrationOptions) => {

package/src/threads/automation.js

@@ -8,10 +8,14 @@ const { DocumentTypes } = require("../db/utils")
 const { doInTenant } = require("@budibase/backend-core/tenancy")
 const { definitions: triggerDefs } = require("../automations/triggerInfo")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
-
+const { AutomationErrors } = require("../constants")
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
+const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
+
 const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: false, status: "STOPPED" }
+const { cloneDeep } = require("lodash/fp")
+const env = require("../environment")
 
 /**
  * The automation orchestrator is a class responsible for executing automations.
@@ -74,50 +78,208 @@ class Orchestrator {
     this.executionOutput.steps.push(stepObj)
   }
 
+  updateContextAndOutput(loopStepNumber, step, output, result) {
+    this.executionOutput.steps.splice(loopStepNumber, 0, {
+      id: step.id,
+      stepId: step.stepId,
+      outputs: {
+        ...output,
+        success: result.success,
+        status: result.status,
+      },
+      inputs: step.inputs,
+    })
+    this._context.steps.splice(loopStepNumber, 0, {
+      ...output,
+      success: result.success,
+      status: result.status,
+    })
+  }
+
   async execute() {
     let automation = this._automation
     const app = await this.getApp()
     let stopped = false
+    let loopStep
+
+    let stepCount = 0
+    let loopStepNumber
+    let loopSteps = []
     for (let step of automation.definition.steps) {
-      // execution stopped, record state for that
-      if (stopped) {
-        this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS)
+      stepCount++
+      let input
+      if (step.stepId === LOOP_STEP_ID) {
+        loopStep = step
+        loopStepNumber = stepCount
         continue
       }
-      let stepFn = await this.getStepFunctionality(step.stepId)
-      step.inputs = await processObject(step.inputs, this._context)
-      step.inputs = automationUtils.cleanInputValues(
-        step.inputs,
-        step.schema.inputs
-      )
-      // appId is always passed
-      try {
-        let tenantId = app.tenantId || DEFAULT_TENANT_ID
-        const outputs = await doInTenant(tenantId, () => {
-          return stepFn({
-            inputs: step.inputs,
-            appId: this._appId,
-            emitter: this._emitter,
-            context: this._context,
-          })
-        })
-        this._context.steps.push(outputs)
-        // if filter causes us to stop execution don't break the loop, set a var
-        // so that we can finish iterating through the steps and record that it stopped
-        if (step.stepId === FILTER_STEP_ID && !outputs.success) {
-          stopped = true
-          this.updateExecutionOutput(step.id, step.stepId, step.inputs, {
-            ...outputs,
-            ...STOPPED_STATUS,
-          })
+
+      if (loopStep) {
+        input = await processObject(loopStep.inputs, this._context)
+      }
+      let iterations = loopStep ? input.binding.length : 1
+      let iterationCount = 0
+      for (let index = 0; index < iterations; index++) {
+        let originalStepInput = cloneDeep(step.inputs)
+
+        // Handle if the user has set a max iteration count or if it reaches the max limit set by us
+        if (loopStep) {
+          // lets first of all handle the input
+          // if the input is array then use it, if it is a string then split it on every new line
+          let newInput = await processObject(
+            loopStep.inputs,
+            cloneDeep(this._context)
+          )
+          newInput = automationUtils.cleanInputValues(
+            newInput,
+            loopStep.schema.inputs
+          )
+          this._context.steps[loopStepNumber] = {
+            currentItem: newInput.binding[index],
+          }
+
+          let tempOutput = { items: loopSteps, iterations: iterationCount }
+          if (
+            (loopStep.inputs.option === "Array" &&
+              !Array.isArray(newInput.binding)) ||
+            (loopStep.inputs.option === "String" &&
+              typeof newInput.binding !== "string")
+          ) {
+            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
+              status: AutomationErrors.INCORRECT_TYPE,
+              success: false,
+            })
+            loopSteps = null
+            loopStep = null
+            break
+          }
+
+          // The "Loop" binding in the front end is "fake", so replace it here so the context can understand it
+          // Pretty hacky because we need to account for the row object
+          for (let [key, value] of Object.entries(originalStepInput)) {
+            if (typeof value === "object") {
+              for (let [innerKey, innerValue] of Object.entries(
+                originalStepInput[key]
+              )) {
+                if (typeof innerValue === "string") {
+                  originalStepInput[key][innerKey] =
+                    automationUtils.substituteLoopStep(
+                      innerValue,
+                      `steps.${loopStepNumber}`
+                    )
+                }
+              }
+            } else {
+              if (typeof value === "string") {
+                originalStepInput[key] = automationUtils.substituteLoopStep(
+                  value,
+                  `steps.${loopStepNumber}`
+                )
+              }
+            }
+          }
+
+          if (
+            index === parseInt(env.AUTOMATION_MAX_ITERATIONS) ||
+            index === loopStep.inputs.iterations
+          ) {
+            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
+              status: AutomationErrors.MAX_ITERATIONS,
+              success: true,
+            })
+            loopSteps = null
+            loopStep = null
+            break
+          }
+
+          if (
+            this._context.steps[loopStepNumber]?.currentItem ===
+            loopStep.inputs.failure
+          ) {
+            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
+              status: AutomationErrors.FAILURE_CONDITION,
+              success: false,
+            })
+            loopSteps = null
+            loopStep = null
+            break
+          }
+        }
+
+        // execution stopped, record state for that
+        if (stopped) {
+          this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS)
           continue
         }
-        this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs)
-      } catch (err) {
-        console.error(`Automation error - ${step.stepId} - ${err}`)
-        return err
+
+        // If it's a loop step, we need to manually add the bindings to the context
+        let stepFn = await this.getStepFunctionality(step.stepId)
+        let inputs = await processObject(originalStepInput, this._context)
+        inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs)
+        try {
+          // appId is always passed
+          let tenantId = app.tenantId || DEFAULT_TENANT_ID
+          const outputs = await doInTenant(tenantId, () => {
+            return stepFn({
+              inputs: inputs,
+              appId: this._appId,
+              emitter: this._emitter,
+              context: this._context,
+            })
+          })
+          this._context.steps[stepCount] = outputs
+          // if filter causes us to stop execution don't break the loop, set a var
+          // so that we can finish iterating through the steps and record that it stopped
+          if (step.stepId === FILTER_STEP_ID && !outputs.success) {
+            stopped = true
+            this.updateExecutionOutput(step.id, step.stepId, step.inputs, {
+              ...outputs,
+              ...STOPPED_STATUS,
+            })
+            continue
+          }
+          if (loopStep && loopSteps) {
+            loopSteps.push(outputs)
+          } else {
+            this.updateExecutionOutput(
+              step.id,
+              step.stepId,
+              step.inputs,
+              outputs
+            )
+          }
+        } catch (err) {
+          console.error(`Automation error - ${step.stepId} - ${err}`)
+          return err
+        }
+        if (loopStep) {
+          iterationCount++
+          if (index === iterations - 1) {
+            loopStep = null
+            this._context.steps.splice(loopStepNumber, 1)
+            break
+          }
+        }
+      }
+
+      if (loopSteps && loopSteps.length) {
+        let tempOutput = {
+          success: true,
+          items: loopSteps,
+          iterations: iterationCount,
+        }
+        this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
+          id: step.id,
+          stepId: step.stepId,
+          outputs: tempOutput,
+          inputs: step.inputs,
+        })
+
+        this._context.steps.splice(loopStepNumber, 0, tempOutput)
+        loopSteps = null
       }
     }
+
     return this.executionOutput
   }
 }

package/src/threads/{index.js → index.ts}

@@ -1,12 +1,12 @@
-const workerFarm = require("worker-farm")
-const env = require("../environment")
+import workerFarm from "worker-farm"
+import * as env from "../environment"
 
-const ThreadType = {
+export const ThreadType = {
   QUERY: "query",
   AUTOMATION: "automation",
 }
 
-function typeToFile(type) {
+function typeToFile(type: any) {
   let filename = null
   switch (type) {
     case ThreadType.QUERY:
@@ -21,8 +21,14 @@ function typeToFile(type) {
   return require.resolve(filename)
 }
 
-class Thread {
-  constructor(type, opts = { timeoutMs: null, count: 1 }) {
+export class Thread {
+  type: any
+  count: any
+  disableThreading: any
+  workers: any
+  timeoutMs: any
+
+  constructor(type: any, opts: any = { timeoutMs: null, count: 1 }) {
     this.type = type
     this.count = opts.count ? opts.count : 1
     this.disableThreading =
@@ -31,7 +37,7 @@ class Thread {
       this.count === 0 ||
       env.isInThread()
     if (!this.disableThreading) {
-      const workerOpts = {
+      const workerOpts: any = {
         autoStart: true,
         maxConcurrentWorkers: this.count,
       }
@@ -43,8 +49,7 @@ class Thread {
     }
   }
 
-  run(data) {
-    const timeoutMs = this.timeoutMs
+  run(data: any) {
     return new Promise((resolve, reject) => {
       let fncToCall
       // if in test then don't use threading
@@ -53,10 +58,12 @@ class Thread {
       } else {
         fncToCall = this.workers
       }
-      fncToCall(data, (err, response) => {
+      fncToCall(data, (err: any, response: any) => {
         if (err && err.type === "TimeoutError") {
           reject(
-            new Error(`Query response time exceeded ${timeoutMs}ms timeout.`)
+            new Error(
+              `Query response time exceeded ${this.timeoutMs}ms timeout.`
+            )
           )
         } else if (err) {
           reject(err)
@@ -67,6 +74,3 @@ class Thread {
     })
   }
 }
-
-module.exports.Thread = Thread
-module.exports.ThreadType = ThreadType

package/src/threads/utils.js

@@ -71,6 +71,7 @@ exports.hasExtraData = response => {
   return (
     typeof response === "object" &&
     !Array.isArray(response) &&
+    response &&
     response.data != null &&
     response.info != null
   )

package/src/utilities/queue/inMemoryQueue.js

@@ -66,7 +66,8 @@ class InMemoryQueue {
    * @param {object} msg A message to be transported over the queue, this should be
    * a JSON message as this is required by Bull.
    */
-  add(msg) {
+  // eslint-disable-next-line no-unused-vars
+  add(msg, repeat) {
     if (typeof msg !== "object") {
       throw "Queue only supports carrying JSON."
     }
@@ -90,6 +91,11 @@ class InMemoryQueue {
     return []
   }
 
+  // eslint-disable-next-line no-unused-vars
+  removeJobs(pattern) {
+    // no-op
+  }
+
   /**
    * Implemented for tests
    */

package/tsconfig.build.json

@@ -0,0 +1,10 @@
+{
+  // Used for building with tsc
+  "extends": "./tsconfig.json",
+  "exclude": [
+    "node_modules",
+    "**/*.json",
+    "**/*.spec.js",
+    "**/*.spec.ts"
+  ]
+}

package/tsconfig.json

@@ -19,7 +19,7 @@
   "exclude": [
     "node_modules",
     "**/*.json",
-    "**/*.spec.ts",
-    "**/*.spec.js"
+    "**/*.spec.js",
+    // "**/*.spec.ts" // don't exclude spec.ts files for editor support
   ]
 }