@budibase/server 1.0.105-alpha.3 → 1.0.105-alpha.30

package/src/api/routes/tests/utilities/{TestFunctions.js → TestFunctions.ts}

@@ -1,31 +1,38 @@
-const rowController = require("../../../controllers/row")
-const appController = require("../../../controllers/application")
-const { AppStatus } = require("../../../../db/utils")
-const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
-const { TENANT_ID } = require("../../../../tests/utilities/structures")
-const { getAppDB, doInAppContext } = require("@budibase/backend-core/context")
-const env = require("../../../../environment")
-
-function Request(appId, params) {
-  this.appId = appId
-  this.params = params
-  this.request = {}
+import * as rowController from "../../../controllers/row"
+import * as appController from "../../../controllers/application"
+import { AppStatus } from "../../../../db/utils"
+import { BUILTIN_ROLE_IDS } from "@budibase/backend-core/roles"
+import { TENANT_ID } from "../../../../tests/utilities/structures"
+import { getAppDB, doInAppContext } from "@budibase/backend-core/context"
+import * as env from "../../../../environment"
+
+class Request {
+  appId: any
+  params: any
+  request: any
+  body: any
+
+  constructor(appId: any, params: any) {
+    this.appId = appId
+    this.params = params
+    this.request = {}
+  }
 }
 
-function runRequest(appId, controlFunc, request) {
+function runRequest(appId: any, controlFunc: any, request?: any) {
   return doInAppContext(appId, async () => {
     return controlFunc(request)
   })
 }
 
-exports.getAllTableRows = async config => {
+export const getAllTableRows = async (config: any) => {
   const req = new Request(config.appId, { tableId: config.table._id })
   await runRequest(config.appId, rowController.fetch, req)
   return req.body
 }
 
-exports.clearAllApps = async (tenantId = TENANT_ID) => {
-  const req = { query: { status: AppStatus.DEV }, user: { tenantId } }
+export const clearAllApps = async (tenantId = TENANT_ID) => {
+  const req: any = { query: { status: AppStatus.DEV }, user: { tenantId } }
   await appController.fetch(req)
   const apps = req.body
   if (!apps || apps.length <= 0) {
@@ -34,11 +41,11 @@ exports.clearAllApps = async (tenantId = TENANT_ID) => {
   for (let app of apps) {
     const { appId } = app
     const req = new Request(null, { appId })
-    await runRequest(appId, appController.delete, req)
+    await runRequest(appId, appController.destroy, req)
   }
 }
 
-exports.clearAllAutomations = async config => {
+export const clearAllAutomations = async (config: any) => {
   const automations = await config.getAllAutomations()
   for (let auto of automations) {
     await doInAppContext(config.appId, async () => {
@@ -47,7 +54,12 @@ exports.clearAllAutomations = async config => {
   }
 }
 
-exports.createRequest = (request, method, url, body) => {
+export const createRequest = (
+  request: any,
+  method: any,
+  url: any,
+  body: any
+) => {
   let req
 
   if (method === "POST") req = request.post(url).send(body)
@@ -59,7 +71,12 @@ exports.createRequest = (request, method, url, body) => {
   return req
 }
 
-exports.checkBuilderEndpoint = async ({ config, method, url, body }) => {
+export const checkBuilderEndpoint = async ({
+  config,
+  method,
+  url,
+  body,
+}: any) => {
   const headers = await config.login({
     userId: "us_fail",
     builder: false,
@@ -71,14 +88,14 @@ exports.checkBuilderEndpoint = async ({ config, method, url, body }) => {
     .expect(403)
 }
 
-exports.checkPermissionsEndpoint = async ({
+export const checkPermissionsEndpoint = async ({
   config,
   method,
   url,
   body,
   passRole,
   failRole,
-}) => {
+}: any) => {
   const passHeader = await config.login({
     roleId: passRole,
     prodApp: true,
@@ -106,11 +123,11 @@ exports.checkPermissionsEndpoint = async ({
     .expect(403)
 }
 
-exports.getDB = () => {
+export const getDB = () => {
   return getAppDB()
 }
 
-exports.testAutomation = async (config, automation) => {
+export const testAutomation = async (config: any, automation: any) => {
   return runRequest(automation.appId, async () => {
     return await config.request
       .post(`/api/automations/${automation._id}/test`)
@@ -126,7 +143,7 @@ exports.testAutomation = async (config, automation) => {
   })
 }
 
-exports.runInProd = async func => {
+export const runInProd = async (func: any) => {
   const nodeEnv = env.NODE_ENV
   const workerId = env.JEST_WORKER_ID
   env._set("NODE_ENV", "PRODUCTION")

package/src/app.ts

@@ -1,6 +1,5 @@
 // need to load environment first
 import { ExtendableContext } from "koa"
-
 import * as env from "./environment"
 const CouchDB = require("./db")
 require("@budibase/backend-core").init(CouchDB)
@@ -15,7 +14,7 @@ const automations = require("./automations/index")
 const Sentry = require("@sentry/node")
 const fileSystem = require("./utilities/fileSystem")
 const bullboard = require("./automations/bullboard")
-const redis = require("./utilities/redis")
+import redis from "./utilities/redis"
 import * as migrations from "./migrations"
 
 const app = new Koa()

package/src/automations/actions.js

@@ -13,6 +13,7 @@ const integromat = require("./steps/integromat")
 let filter = require("./steps/filter")
 let delay = require("./steps/delay")
 let queryRow = require("./steps/queryRows")
+let loop = require("./steps/loop")
 const env = require("../environment")
 
 const ACTION_IMPLS = {
@@ -27,6 +28,7 @@ const ACTION_IMPLS = {
   DELAY: delay.run,
   FILTER: filter.run,
   QUERY_ROWS: queryRow.run,
+  LOOP: loop.run,
   // these used to be lowercase step IDs, maintain for backwards compat
   discord: discord.run,
   slack: slack.run,
@@ -45,6 +47,7 @@ const ACTION_DEFINITIONS = {
   DELAY: delay.definition,
   FILTER: filter.definition,
   QUERY_ROWS: queryRow.definition,
+  LOOP: loop.definition,
   // these used to be lowercase step IDs, maintain for backwards compat
   discord: discord.definition,
   slack: slack.definition,

package/src/automations/automationUtils.js

@@ -1,4 +1,5 @@
 const { getTable } = require("../api/controllers/table/utils")
+const { findHBSBlocks } = require("@budibase/string-templates")
 
 /**
  * When values are input to the system generally they will be of type string as this is required for template strings.
@@ -74,3 +75,14 @@ exports.getError = err => {
   }
   return typeof err !== "string" ? err.toString() : err
 }
+
+exports.substituteLoopStep = (hbsString, substitute) => {
+  let blocks = findHBSBlocks(hbsString)
+  for (let block of blocks) {
+    let oldBlock = block
+    block = block.replace(/loop/, substitute)
+    hbsString = hbsString.replace(new RegExp(oldBlock, "g"), block)
+  }
+
+  return hbsString
+}

package/src/automations/steps/{createRow.js → createRow.ts}

@@ -1,9 +1,8 @@
-const rowController = require("../../api/controllers/row")
-const automationUtils = require("../automationUtils")
-const usage = require("../../utilities/usageQuota")
-const { buildCtx } = require("./utils")
+import { save } from "../../api/controllers/row"
+import { cleanUpRow, getError } from "../automationUtils"
+import { buildCtx } from "./utils"
 
-exports.definition = {
+export const definition = {
   name: "Create Row",
   tagline: "Create a {{inputs.enriched.table.name}} row",
   icon: "TableRowAddBottom",
@@ -59,7 +58,7 @@ exports.definition = {
   },
 }
 
-exports.run = async function ({ inputs, appId, emitter }) {
+export async function run({ inputs, appId, emitter }: any) {
   if (inputs.row == null || inputs.row.tableId == null) {
     return {
       success: false,
@@ -69,7 +68,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
     }
   }
   // have to clean up the row, remove the table from it
-  const ctx = buildCtx(appId, emitter, {
+  const ctx: any = buildCtx(appId, emitter, {
     body: inputs.row,
     params: {
       tableId: inputs.row.tableId,
@@ -77,13 +76,8 @@ exports.run = async function ({ inputs, appId, emitter }) {
   })
 
   try {
-    inputs.row = await automationUtils.cleanUpRow(
-      inputs.row.tableId,
-      inputs.row
-    )
-    await usage.update(usage.Properties.ROW, 1, { dryRun: true })
-    await rowController.save(ctx)
-    await usage.update(usage.Properties.ROW, 1)
+    inputs.row = await cleanUpRow(inputs.row.tableId, inputs.row)
+    await save(ctx)
     return {
       row: inputs.row,
       response: ctx.body,
@@ -94,7 +88,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
   } catch (err) {
     return {
       success: false,
-      response: automationUtils.getError(err),
+      response: getError(err),
     }
   }
 }

package/src/automations/steps/{deleteRow.js → deleteRow.ts}

@@ -1,9 +1,8 @@
-const rowController = require("../../api/controllers/row")
-const usage = require("../../utilities/usageQuota")
-const { buildCtx } = require("./utils")
-const automationUtils = require("../automationUtils")
+import { destroy } from "../../api/controllers/row"
+import { buildCtx } from "./utils"
+import { getError } from "../automationUtils"
 
-exports.definition = {
+export const definition = {
   description: "Delete a row from your database",
   icon: "TableRowRemoveCenter",
   name: "Delete Row",
@@ -24,12 +23,8 @@ exports.definition = {
           type: "string",
           title: "Row ID",
         },
-        revision: {
-          type: "string",
-          title: "Row Revision",
-        },
       },
-      required: ["tableId", "id", "revision"],
+      required: ["tableId", "id"],
     },
     outputs: {
       properties: {
@@ -52,8 +47,8 @@ exports.definition = {
   },
 }
 
-exports.run = async function ({ inputs, appId, emitter }) {
-  if (inputs.id == null || inputs.revision == null) {
+export async function run({ inputs, appId, emitter }: any) {
+  if (inputs.id == null) {
     return {
       success: false,
       response: {
@@ -62,7 +57,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
     }
   }
 
-  let ctx = buildCtx(appId, emitter, {
+  let ctx: any = buildCtx(appId, emitter, {
     body: {
       _id: inputs.id,
       _rev: inputs.revision,
@@ -73,8 +68,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
   })
 
   try {
-    await usage.update(usage.Properties.ROW, -1)
-    await rowController.destroy(ctx)
+    await destroy(ctx)
     return {
       response: ctx.body,
       row: ctx.row,
@@ -83,7 +77,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
   } catch (err) {
     return {
       success: false,
-      response: automationUtils.getError(err),
+      response: getError(err),
     }
   }
 }

package/src/automations/steps/loop.js

@@ -0,0 +1,50 @@
+exports.definition = {
+  name: "Looping",
+  icon: "Reuse",
+  tagline: "Loop the block",
+  description: "Loop",
+  stepId: "LOOP",
+  internal: true,
+  inputs: {},
+  schema: {
+    inputs: {
+      properties: {
+        option: {
+          customType: "loopOption",
+          title: "Input type",
+        },
+        binding: {
+          type: "string",
+          title: "Binding / Value",
+        },
+        iterations: {
+          type: "number",
+          title: "Max loop iterations",
+        },
+        failure: {
+          type: "string",
+          title: "Failure Condition",
+        },
+      },
+      required: ["type", "value", "iterations", "failure"],
+    },
+    outputs: {
+      properties: {
+        items: {
+          customType: "item",
+          description: "The item currently being executed",
+        },
+        success: {
+          type: "boolean",
+          description: "Whether the message loop was successfully",
+        },
+        iterations: {
+          type: "number",
+          descriptions: "The amount of times the block ran",
+        },
+      },
+      required: ["success", "items", "iterations"],
+    },
+  },
+  type: "LOGIC",
+}

package/src/automations/tests/automation.spec.js

@@ -1,4 +1,3 @@
-jest.mock("../../utilities/usageQuota")
 jest.mock("../../threads/automation")
 jest.mock("../../utilities/redis", () => ({
   init: jest.fn(),

package/src/automations/tests/{createRow.spec.js → createRow.spec.ts}

@@ -1,10 +1,8 @@
-jest.mock("../../utilities/usageQuota")
-
-const usageQuota = require("../../utilities/usageQuota")
-const setup = require("./utilities")
+import * as setup from "./utilities"
 
 describe("test the create row action", () => {
-  let table, row
+  let table: any
+  let row: any
   let config = setup.getConfig()
 
   beforeEach(async () => {
@@ -36,20 +34,11 @@ describe("test the create row action", () => {
       row: {
         tableId: "invalid",
         invalid: "invalid",
-      }
+      },
     })
     expect(res.success).toEqual(false)
   })
 
-  it("check usage quota attempts", async () => {
-    await setup.runInProd(async () => {
-      await setup.runStep(setup.actions.CREATE_ROW.stepId, {
-        row
-      })
-      expect(usageQuota.update).toHaveBeenCalledWith("rows", 1)
-    })
-  })
-
   it("should check invalid inputs return an error", async () => {
     const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, {})
     expect(res.success).toEqual(false)

package/src/automations/tests/{deleteRow.spec.js → deleteRow.spec.ts}

@@ -1,10 +1,9 @@
-jest.mock("../../utilities/usageQuota")
-
-const usageQuota = require("../../utilities/usageQuota")
 const setup = require("./utilities")
 
 describe("test the delete row action", () => {
-  let table, row, inputs
+  let table: any
+  let row: any
+  let inputs: any
   let config = setup.getConfig()
 
   beforeEach(async () => {
@@ -37,7 +36,6 @@ describe("test the delete row action", () => {
   it("check usage quota attempts", async () => {
     await setup.runInProd(async () => {
       await setup.runStep(setup.actions.DELETE_ROW.stepId, inputs)
-      expect(usageQuota.update).toHaveBeenCalledWith("rows", -1)
     })
   })
 

package/src/automations/tests/utilities/index.js

@@ -18,7 +18,6 @@ exports.afterAll = () => {
 
 exports.runInProd = async fn => {
   env._set("NODE_ENV", "production")
-  env._set("USE_QUOTAS", 1)
   let error
   try {
     await fn()
@@ -26,7 +25,6 @@ exports.runInProd = async fn => {
     error = err
   }
   env._set("NODE_ENV", "jest")
-  env._set("USE_QUOTAS", null)
   if (error) {
     throw error
   }

package/src/automations/{utils.js → utils.ts}

@@ -1,26 +1,32 @@
-const { Thread, ThreadType } = require("../threads")
-const { definitions } = require("./triggerInfo")
-const webhooks = require("../api/controllers/webhook")
-const CouchDB = require("../db")
-const { queue } = require("./bullboard")
-const newid = require("../db/newid")
-const { updateEntityMetadata } = require("../utilities")
-const { MetadataTypes, WebhookType } = require("../constants")
-const { getProdAppID } = require("@budibase/backend-core/db")
-const { cloneDeep } = require("lodash/fp")
-const { getAppDB, getAppId } = require("@budibase/backend-core/context")
+import { Thread, ThreadType } from "../threads"
+import { definitions } from "./triggerInfo"
+import * as webhooks from "../api/controllers/webhook"
+import CouchDB from "../db"
+import { queue } from "./bullboard"
+import newid from "../db/newid"
+import { updateEntityMetadata } from "../utilities"
+import { MetadataTypes, WebhookType } from "../constants"
+import { getProdAppID } from "@budibase/backend-core/db"
+import { cloneDeep } from "lodash/fp"
+import { getAppDB, getAppId } from "@budibase/backend-core/context"
+import { tenancy } from "@budibase/backend-core"
+import { quotas } from "@budibase/pro"
 
 const WH_STEP_ID = definitions.WEBHOOK.stepId
 const CRON_STEP_ID = definitions.CRON.stepId
 const Runner = new Thread(ThreadType.AUTOMATION)
 
-exports.processEvent = async job => {
+export async function processEvent(job: any) {
   try {
-    // need to actually await these so that an error can be captured properly
     console.log(
       `${job.data.automation.appId} automation ${job.data.automation._id} running`
     )
-    return await Runner.run(job)
+    // need to actually await these so that an error can be captured properly
+    const tenantId = tenancy.getTenantIDFromAppID(job.data.event.appId)
+    return await tenancy.doInTenant(tenantId, async () => {
+      const runFn = () => Runner.run(job)
+      return quotas.addAutomation(runFn)
+    })
   } catch (err) {
     const errJson = JSON.stringify(err)
     console.error(
@@ -31,11 +37,15 @@ exports.processEvent = async job => {
   }
 }
 
-exports.updateTestHistory = async (appId, automation, history) => {
+export async function updateTestHistory(
+  appId: any,
+  automation: any,
+  history: any
+) {
   return updateEntityMetadata(
     MetadataTypes.AUTOMATION_TEST_HISTORY,
     automation._id,
-    metadata => {
+    (metadata: any) => {
       if (metadata && Array.isArray(metadata.history)) {
         metadata.history.push(history)
       } else {
@@ -48,7 +58,7 @@ exports.updateTestHistory = async (appId, automation, history) => {
   )
 }
 
-exports.removeDeprecated = definitions => {
+export function removeDeprecated(definitions: any) {
   const base = cloneDeep(definitions)
   for (let key of Object.keys(base)) {
     if (base[key].deprecated) {
@@ -59,13 +69,15 @@ exports.removeDeprecated = definitions => {
 }
 
 // end the repetition and the job itself
-exports.disableAllCrons = async appId => {
+export async function disableAllCrons(appId: any) {
   const promises = []
   const jobs = await queue.getRepeatableJobs()
   for (let job of jobs) {
     if (job.key.includes(`${appId}_cron`)) {
       promises.push(queue.removeRepeatableByKey(job.key))
-      promises.push(queue.removeJobs(job.id))
+      if (job.id) {
+        promises.push(queue.removeJobs(job.id))
+      }
     }
   }
   return Promise.all(promises)
@@ -76,9 +88,9 @@ exports.disableAllCrons = async appId => {
  * @param {string} appId The ID of the app in which we are checking for webhooks
  * @param {object|undefined} automation The automation object to be updated.
  */
-exports.enableCronTrigger = async (appId, automation) => {
+export async function enableCronTrigger(appId: any, automation: any) {
   const trigger = automation ? automation.definition.trigger : null
-  function isCronTrigger(auto) {
+  function isCronTrigger(auto: any) {
     return (
       auto &&
       auto.definition.trigger &&
@@ -89,7 +101,7 @@ exports.enableCronTrigger = async (appId, automation) => {
   if (isCronTrigger(automation)) {
     // make a job id rather than letting Bull decide, makes it easier to handle on way out
     const jobId = `${appId}_cron_${newid()}`
-    const job = await queue.add(
+    const job: any = await queue.add(
       {
         automation,
         event: { appId, timestamp: Date.now() },
@@ -117,13 +129,13 @@ exports.enableCronTrigger = async (appId, automation) => {
  * @returns {Promise<object|undefined>} After this is complete the new automation object may have been updated and should be
  * written to DB (this does not write to DB as it would be wasteful to repeat).
  */
-exports.checkForWebhooks = async ({ oldAuto, newAuto }) => {
+export async function checkForWebhooks({ oldAuto, newAuto }: any) {
   const appId = getAppId()
   const oldTrigger = oldAuto ? oldAuto.definition.trigger : null
   const newTrigger = newAuto ? newAuto.definition.trigger : null
   const triggerChanged =
     oldTrigger && newTrigger && oldTrigger.id !== newTrigger.id
-  function isWebhookTrigger(auto) {
+  function isWebhookTrigger(auto: any) {
     return (
       auto &&
       auto.definition.trigger &&
@@ -159,7 +171,7 @@ exports.checkForWebhooks = async ({ oldAuto, newAuto }) => {
     (!isWebhookTrigger(oldAuto) || triggerChanged) &&
     isWebhookTrigger(newAuto)
   ) {
-    const ctx = {
+    const ctx: any = {
       appId,
       request: {
         body: new webhooks.Webhook(
@@ -189,6 +201,6 @@ exports.checkForWebhooks = async ({ oldAuto, newAuto }) => {
  * @param appId {string} the app that is being removed.
  * @return {Promise<void>} clean is complete if this succeeds.
  */
-exports.cleanupAutomations = async appId => {
-  await exports.disableAllCrons(appId)
+export async function cleanupAutomations(appId: any) {
+  await disableAllCrons(appId)
 }

package/src/constants/index.js

@@ -190,5 +190,11 @@ exports.WebhookType = {
   AUTOMATION: "automation",
 }
 
+exports.AutomationErrors = {
+  INCORRECT_TYPE: "INCORRECT_TYPE",
+  MAX_ITERATIONS: "MAX_ITERATIONS_REACHED",
+  FAILURE_CONDITION: "FAILURE_CONDITION_MET",
+}
+
 // pass through the list from the auth/core lib
 exports.ObjectStoreBuckets = ObjectStoreBuckets

package/src/environment.js

@@ -38,8 +38,6 @@ module.exports = {
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
-  USE_QUOTAS: process.env.USE_QUOTAS,
-  EXCLUDE_QUOTAS_TENANTS: process.env.EXCLUDE_QUOTAS_TENANTS,
   REDIS_URL: process.env.REDIS_URL,
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
@@ -59,6 +57,7 @@ module.exports = {
   LOG_LEVEL: process.env.LOG_LEVEL,
   AUTOMATION_DIRECTORY: process.env.AUTOMATION_DIRECTORY,
   AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET,
+  AUTOMATION_MAX_ITERATIONS: process.env.AUTOMATION_MAX_ITERATIONS,
   SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
   DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
   POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,