@budibase/server 1.0.105-alpha.3 → 1.0.105-alpha.32

package/src/middleware/{authorized.js → authorized.ts}

@@ -1,8 +1,8 @@
-const {
+import {
   getUserRoleHierarchy,
   getRequiredResourceRole,
   BUILTIN_ROLE_IDS,
-} = require("@budibase/backend-core/roles")
+} from "@budibase/backend-core/roles"
 const {
   PermissionTypes,
   PermissionLevels,
@@ -13,7 +13,7 @@ const { isWebhookEndpoint } = require("./utils")
 const { buildCsrfMiddleware } = require("@budibase/backend-core/auth")
 const { getAppId } = require("@budibase/backend-core/context")
 
-function hasResource(ctx) {
+function hasResource(ctx: any) {
   return ctx.resourceId != null
 }
 
@@ -25,7 +25,12 @@ const csrf = buildCsrfMiddleware()
  * - Builders can access all resources.
  * - Otherwise the user must have the required role.
  */
-const checkAuthorized = async (ctx, resourceRoles, permType, permLevel) => {
+const checkAuthorized = async (
+  ctx: any,
+  resourceRoles: any,
+  permType: any,
+  permLevel: any
+) => {
   // check if this is a builder api and the user is not a builder
   const isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global
   const isBuilderApi = permType === PermissionTypes.BUILDER
@@ -40,10 +45,10 @@ const checkAuthorized = async (ctx, resourceRoles, permType, permLevel) => {
 }
 
 const checkAuthorizedResource = async (
-  ctx,
-  resourceRoles,
-  permType,
-  permLevel
+  ctx: any,
+  resourceRoles: any,
+  permType: any,
+  permLevel: any
 ) => {
   // get the user's roles
   const roleId = ctx.roleId || BUILTIN_ROLE_IDS.PUBLIC
@@ -54,7 +59,9 @@ const checkAuthorizedResource = async (
   // check if the user has the required role
   if (resourceRoles.length > 0) {
     // deny access if the user doesn't have the required resource role
-    const found = userRoles.find(role => resourceRoles.indexOf(role._id) !== -1)
+    const found = userRoles.find(
+      (role: any) => resourceRoles.indexOf(role._id) !== -1
+    )
     if (!found) {
       ctx.throw(403, permError)
     }
@@ -64,9 +71,8 @@ const checkAuthorizedResource = async (
   }
 }
 
-module.exports =
-  (permType, permLevel = null, opts = { schema: false }) =>
-  async (ctx, next) => {
+export = (permType: any, permLevel: any = null, opts = { schema: false }) =>
+  async (ctx: any, next: any) => {
     // webhooks don't need authentication, each webhook unique
     // also internal requests (between services) don't need authorized
     if (isWebhookEndpoint(ctx) || ctx.internal) {
@@ -82,8 +88,8 @@ module.exports =
     await builderMiddleware(ctx, permType)
 
     // get the resource roles
-    let resourceRoles = [],
-      otherLevelRoles
+    let resourceRoles: any = []
+    let otherLevelRoles: any = []
     const otherLevel =
       permLevel === PermissionLevels.READ
         ? PermissionLevels.WRITE

package/src/middleware/currentapp.js

@@ -4,7 +4,7 @@ const {
   getCookie,
   clearCookie,
 } = require("@budibase/backend-core/utils")
-const { Cookies } = require("@budibase/backend-core/constants")
+const { Cookies, Headers } = require("@budibase/backend-core/constants")
 const { getRole } = require("@budibase/backend-core/roles")
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { generateUserMetadataID, isDevAppID } = require("../db/utils")
@@ -63,6 +63,21 @@ module.exports = async (ctx, next) => {
     appId = requestAppId
     // retrieving global user gets the right role
     roleId = globalUser.roleId || roleId
+
+    // Allow builders to specify their role via a header
+    const isBuilder =
+      globalUser && globalUser.builder && globalUser.builder.global
+    const isDevApp = appId && isDevAppID(appId)
+    const roleHeader = ctx.request && ctx.request.headers[Headers.PREVIEW_ROLE]
+    if (isBuilder && isDevApp && roleHeader) {
+      // Ensure the role is valid by ensuring a definition exists
+      try {
+        await getRole(roleHeader)
+        roleId = roleHeader
+      } catch (error) {
+        // Swallow error and do nothing
+      }
+    }
   }
 
   // nothing more to do

package/src/migrations/functions/developerQuota.ts

@@ -0,0 +1,15 @@
+const { createUserBuildersView } = require("@budibase/backend-core/db")
+import * as syncDevelopers from "./usageQuotas/syncDevelopers"
+
+/**
+ * Date:
+ * March 2022
+ *
+ * Description:
+ * Create the builder users view and sync the developer count
+ */
+
+export const run = async (db: any) => {
+  await createUserBuildersView(db)
+  await syncDevelopers.run()
+}

package/src/migrations/functions/publishedAppsQuota.ts

@@ -0,0 +1,13 @@
+import * as syncPublishedApps from "./usageQuotas/syncPublishedApps"
+
+/**
+ * Date:
+ * March 2022
+ *
+ * Description:
+ * Sync the published apps count
+ */
+
+export const run = async (db: any) => {
+  await syncPublishedApps.run()
+}

package/src/migrations/functions/tests/quotas1.spec.js

@@ -1,4 +1,3 @@
-const env = require("../../../environment")
 const TestConfig = require("../../../tests/utilities/TestConfiguration")
 
 const syncApps = jest.fn()
@@ -14,7 +13,6 @@ describe("run", () => {
 
   beforeEach(async () => {
     await config.init()
-    env._set("USE_QUOTAS", 1)
   })
 
   afterAll(config.end)  

package/src/migrations/functions/usageQuotas/index.ts

@@ -1,8 +1,3 @@
-const { useQuotas } = require("../../../utilities/usageQuota")
-
 export const runQuotaMigration = async (migration: Function) => {
-  if (!useQuotas()) {
-    return
-  }
   await migration()
 }

package/src/migrations/functions/usageQuotas/syncApps.ts

@@ -1,9 +1,8 @@
-import { getGlobalDB, getTenantId } from "@budibase/backend-core/tenancy"
+import { getTenantId } from "@budibase/backend-core/tenancy"
 import { getAllApps } from "@budibase/backend-core/db"
-import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
+import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
 
 export const run = async () => {
-  const db = getGlobalDB()
   // get app count
   // @ts-ignore
   const devApps = await getAllApps({ dev: true })
@@ -12,7 +11,5 @@ export const run = async () => {
   // sync app count
   const tenantId = getTenantId()
   console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
-  const usageDoc = await getUsageQuotaDoc(db)
-  usageDoc.usageQuota.apps = appCount
-  await db.put(usageDoc)
+  await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC)
 }

package/src/migrations/functions/usageQuotas/syncDevelopers.ts

@@ -0,0 +1,19 @@
+import { getTenantId } from "@budibase/backend-core/tenancy"
+import { utils } from "@budibase/backend-core"
+import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
+
+export const run = async () => {
+  // get developer count
+  const developerCount = await utils.getBuildersCount()
+
+  // sync developer count
+  const tenantId = getTenantId()
+  console.log(
+    `[Tenant: ${tenantId}] Syncing developer count: ${developerCount}`
+  )
+  await quotas.setUsage(
+    developerCount,
+    StaticQuotaName.DEVELOPERS,
+    QuotaUsageType.STATIC
+  )
+}

package/src/migrations/functions/usageQuotas/syncPublishedApps.ts

@@ -0,0 +1,21 @@
+import { getTenantId } from "@budibase/backend-core/tenancy"
+import { getAllApps } from "@budibase/backend-core/db"
+import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
+
+export const run = async () => {
+  // get app count
+  const opts: any = { dev: false }
+  const prodApps = await getAllApps(opts)
+  const prodAppCount = prodApps ? prodApps.length : 0
+
+  // sync app count
+  const tenantId = getTenantId()
+  console.log(
+    `[Tenant: ${tenantId}] Syncing published app count: ${prodAppCount}`
+  )
+  await quotas.setUsage(
+    prodAppCount,
+    StaticQuotaName.PUBLISHED_APPS,
+    QuotaUsageType.STATIC
+  )
+}

package/src/migrations/functions/usageQuotas/syncRows.ts

@@ -1,10 +1,9 @@
-import { getGlobalDB, getTenantId } from "@budibase/backend-core/tenancy"
+import { getTenantId } from "@budibase/backend-core/tenancy"
 import { getAllApps } from "@budibase/backend-core/db"
-import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
 import { getUniqueRows } from "../../../utilities/usageQuota/rows"
+import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
 
 export const run = async () => {
-  const db = getGlobalDB()
   // get all rows in all apps
   // @ts-ignore
   const allApps = await getAllApps({ all: true })
@@ -16,7 +15,5 @@ export const run = async () => {
   // sync row count
   const tenantId = getTenantId()
   console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
-  const usageDoc = await getUsageQuotaDoc(db)
-  usageDoc.usageQuota.rows = rowCount
-  await db.put(usageDoc)
+  await quotas.setUsage(rowCount, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
 }

package/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts

@@ -0,0 +1,32 @@
+import TestConfig from "../../../../tests/utilities/TestConfiguration"
+import * as syncApps from "../syncApps"
+import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
+
+describe("syncApps", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)
+
+  it("runs successfully", async () => {
+    // create the usage quota doc and mock usages
+    await quotas.getQuotaUsage()
+    await quotas.setUsage(3, StaticQuotaName.APPS, QuotaUsageType.STATIC)
+
+    let usageDoc = await quotas.getQuotaUsage()
+    expect(usageDoc.usageQuota.apps).toEqual(3)
+
+    // create an extra app to test the migration
+    await config.createApp("quota-test")
+
+    // migrate
+    await syncApps.run()
+
+    // assert the migration worked
+    usageDoc = await quotas.getQuotaUsage()
+    expect(usageDoc.usageQuota.apps).toEqual(2)
+  })
+})

package/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts

@@ -0,0 +1,38 @@
+import TestConfig from "../../../../tests/utilities/TestConfiguration"
+import * as syncRows from "../syncRows"
+import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
+
+describe("syncRows", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)
+
+  it("runs successfully", async () => {
+    // create the usage quota doc and mock usages
+    await quotas.getQuotaUsage()
+    await quotas.setUsage(300, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
+
+    let usageDoc = await quotas.getQuotaUsage()
+    expect(usageDoc.usageQuota.rows).toEqual(300)
+
+    // app 1
+    await config.createTable()
+    await config.createRow()
+    // app 2
+    await config.createApp("second-app")
+    await config.createTable()
+    await config.createRow()
+    await config.createRow()
+
+    // migrate
+    await syncRows.run()
+
+    // assert the migration worked
+    usageDoc = await quotas.getQuotaUsage()
+    expect(usageDoc.usageQuota.rows).toEqual(3)
+  })
+})

package/src/migrations/index.ts

@@ -8,6 +8,8 @@ const {
 import * as userEmailViewCasing from "./functions/userEmailViewCasing"
 import * as quota1 from "./functions/quotas1"
 import * as appUrls from "./functions/appUrls"
+import * as developerQuota from "./functions/developerQuota"
+import * as publishedAppsQuota from "./functions/publishedAppsQuota"
 
 export interface Migration {
   type: string
@@ -27,7 +29,7 @@ export interface Migration {
  */
 export interface MigrationOptions {
   tenantIds?: string[]
-  forced?: {
+  force?: {
     [type: string]: string[]
   }
 }
@@ -49,6 +51,16 @@ export const MIGRATIONS: Migration[] = [
     opts: { all: true },
     fn: appUrls.run,
   },
+  {
+    type: MIGRATION_TYPES.GLOBAL,
+    name: "developer_quota",
+    fn: developerQuota.run,
+  },
+  {
+    type: MIGRATION_TYPES.GLOBAL,
+    name: "published_apps_quota",
+    fn: publishedAppsQuota.run,
+  },
 ]
 
 export const migrate = async (options?: MigrationOptions) => {

package/src/module.d.ts

@@ -1,3 +1,7 @@
 declare module "@budibase/backend-core"
 declare module "@budibase/backend-core/tenancy"
 declare module "@budibase/backend-core/db"
+declare module "@budibase/backend-core/context"
+declare module "@budibase/backend-core/cache"
+declare module "@budibase/backend-core/permissions"
+declare module "@budibase/backend-core/roles"

package/src/threads/automation.js

@@ -8,10 +8,14 @@ const { DocumentTypes } = require("../db/utils")
 const { doInTenant } = require("@budibase/backend-core/tenancy")
 const { definitions: triggerDefs } = require("../automations/triggerInfo")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
-
+const { AutomationErrors } = require("../constants")
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
+const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
+
 const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: false, status: "STOPPED" }
+const { cloneDeep } = require("lodash/fp")
+const env = require("../environment")
 
 /**
  * The automation orchestrator is a class responsible for executing automations.
@@ -74,50 +78,208 @@ class Orchestrator {
     this.executionOutput.steps.push(stepObj)
   }
 
+  updateContextAndOutput(loopStepNumber, step, output, result) {
+    this.executionOutput.steps.splice(loopStepNumber, 0, {
+      id: step.id,
+      stepId: step.stepId,
+      outputs: {
+        ...output,
+        success: result.success,
+        status: result.status,
+      },
+      inputs: step.inputs,
+    })
+    this._context.steps.splice(loopStepNumber, 0, {
+      ...output,
+      success: result.success,
+      status: result.status,
+    })
+  }
+
   async execute() {
     let automation = this._automation
     const app = await this.getApp()
     let stopped = false
+    let loopStep
+
+    let stepCount = 0
+    let loopStepNumber
+    let loopSteps = []
     for (let step of automation.definition.steps) {
-      // execution stopped, record state for that
-      if (stopped) {
-        this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS)
+      stepCount++
+      let input
+      if (step.stepId === LOOP_STEP_ID) {
+        loopStep = step
+        loopStepNumber = stepCount
         continue
       }
-      let stepFn = await this.getStepFunctionality(step.stepId)
-      step.inputs = await processObject(step.inputs, this._context)
-      step.inputs = automationUtils.cleanInputValues(
-        step.inputs,
-        step.schema.inputs
-      )
-      // appId is always passed
-      try {
-        let tenantId = app.tenantId || DEFAULT_TENANT_ID
-        const outputs = await doInTenant(tenantId, () => {
-          return stepFn({
-            inputs: step.inputs,
-            appId: this._appId,
-            emitter: this._emitter,
-            context: this._context,
-          })
-        })
-        this._context.steps.push(outputs)
-        // if filter causes us to stop execution don't break the loop, set a var
-        // so that we can finish iterating through the steps and record that it stopped
-        if (step.stepId === FILTER_STEP_ID && !outputs.success) {
-          stopped = true
-          this.updateExecutionOutput(step.id, step.stepId, step.inputs, {
-            ...outputs,
-            ...STOPPED_STATUS,
-          })
+
+      if (loopStep) {
+        input = await processObject(loopStep.inputs, this._context)
+      }
+      let iterations = loopStep ? input.binding.length : 1
+      let iterationCount = 0
+      for (let index = 0; index < iterations; index++) {
+        let originalStepInput = cloneDeep(step.inputs)
+
+        // Handle if the user has set a max iteration count or if it reaches the max limit set by us
+        if (loopStep) {
+          // lets first of all handle the input
+          // if the input is array then use it, if it is a string then split it on every new line
+          let newInput = await processObject(
+            loopStep.inputs,
+            cloneDeep(this._context)
+          )
+          newInput = automationUtils.cleanInputValues(
+            newInput,
+            loopStep.schema.inputs
+          )
+          this._context.steps[loopStepNumber] = {
+            currentItem: newInput.binding[index],
+          }
+
+          let tempOutput = { items: loopSteps, iterations: iterationCount }
+          if (
+            (loopStep.inputs.option === "Array" &&
+              !Array.isArray(newInput.binding)) ||
+            (loopStep.inputs.option === "String" &&
+              typeof newInput.binding !== "string")
+          ) {
+            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
+              status: AutomationErrors.INCORRECT_TYPE,
+              success: false,
+            })
+            loopSteps = null
+            loopStep = null
+            break
+          }
+
+          // The "Loop" binding in the front end is "fake", so replace it here so the context can understand it
+          // Pretty hacky because we need to account for the row object
+          for (let [key, value] of Object.entries(originalStepInput)) {
+            if (typeof value === "object") {
+              for (let [innerKey, innerValue] of Object.entries(
+                originalStepInput[key]
+              )) {
+                if (typeof innerValue === "string") {
+                  originalStepInput[key][innerKey] =
+                    automationUtils.substituteLoopStep(
+                      innerValue,
+                      `steps.${loopStepNumber}`
+                    )
+                }
+              }
+            } else {
+              if (typeof value === "string") {
+                originalStepInput[key] = automationUtils.substituteLoopStep(
+                  value,
+                  `steps.${loopStepNumber}`
+                )
+              }
+            }
+          }
+
+          if (
+            index === parseInt(env.AUTOMATION_MAX_ITERATIONS) ||
+            index === loopStep.inputs.iterations
+          ) {
+            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
+              status: AutomationErrors.MAX_ITERATIONS,
+              success: true,
+            })
+            loopSteps = null
+            loopStep = null
+            break
+          }
+
+          if (
+            this._context.steps[loopStepNumber]?.currentItem ===
+            loopStep.inputs.failure
+          ) {
+            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
+              status: AutomationErrors.FAILURE_CONDITION,
+              success: false,
+            })
+            loopSteps = null
+            loopStep = null
+            break
+          }
+        }
+
+        // execution stopped, record state for that
+        if (stopped) {
+          this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS)
           continue
         }
-        this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs)
-      } catch (err) {
-        console.error(`Automation error - ${step.stepId} - ${err}`)
-        return err
+
+        // If it's a loop step, we need to manually add the bindings to the context
+        let stepFn = await this.getStepFunctionality(step.stepId)
+        let inputs = await processObject(originalStepInput, this._context)
+        inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs)
+        try {
+          // appId is always passed
+          let tenantId = app.tenantId || DEFAULT_TENANT_ID
+          const outputs = await doInTenant(tenantId, () => {
+            return stepFn({
+              inputs: inputs,
+              appId: this._appId,
+              emitter: this._emitter,
+              context: this._context,
+            })
+          })
+          this._context.steps[stepCount] = outputs
+          // if filter causes us to stop execution don't break the loop, set a var
+          // so that we can finish iterating through the steps and record that it stopped
+          if (step.stepId === FILTER_STEP_ID && !outputs.success) {
+            stopped = true
+            this.updateExecutionOutput(step.id, step.stepId, step.inputs, {
+              ...outputs,
+              ...STOPPED_STATUS,
+            })
+            continue
+          }
+          if (loopStep && loopSteps) {
+            loopSteps.push(outputs)
+          } else {
+            this.updateExecutionOutput(
+              step.id,
+              step.stepId,
+              step.inputs,
+              outputs
+            )
+          }
+        } catch (err) {
+          console.error(`Automation error - ${step.stepId} - ${err}`)
+          return err
+        }
+        if (loopStep) {
+          iterationCount++
+          if (index === iterations - 1) {
+            loopStep = null
+            this._context.steps.splice(loopStepNumber, 1)
+            break
+          }
+        }
+      }
+
+      if (loopSteps && loopSteps.length) {
+        let tempOutput = {
+          success: true,
+          items: loopSteps,
+          iterations: iterationCount,
+        }
+        this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
+          id: step.id,
+          stepId: step.stepId,
+          outputs: tempOutput,
+          inputs: step.inputs,
+        })
+
+        this._context.steps.splice(loopStepNumber, 0, tempOutput)
+        loopSteps = null
       }
     }
+
     return this.executionOutput
   }
 }