@budibase/server 2.6.22 → 2.6.24-alpha.0

package/src/api/routes/tests/{automation.spec.js → automation.spec.ts}

@@ -1,15 +1,27 @@
-const {
+import {
   checkBuilderEndpoint,
   getAllTableRows,
   clearAllAutomations,
   testAutomation,
-} = require("./utilities/TestFunctions")
-const setup = require("./utilities")
-const { basicAutomation, newAutomation, automationTrigger, automationStep } = setup.structures
-const MAX_RETRIES = 4
-const { TRIGGER_DEFINITIONS, BUILTIN_ACTION_DEFINITIONS } = require("../../../automations")
-const { events } = require("@budibase/backend-core")
+} from "./utilities/TestFunctions"
+import * as setup from "./utilities"
+import {
+  TRIGGER_DEFINITIONS,
+  BUILTIN_ACTION_DEFINITIONS,
+} from "../../../automations"
+import { events } from "@budibase/backend-core"
+import sdk from "../../../sdk"
+import { Automation } from "@budibase/types"
+import { mocks } from "@budibase/backend-core/tests"
 
+const MAX_RETRIES = 4
+let {
+  basicAutomation,
+  newAutomation,
+  automationTrigger,
+  automationStep,
+  collectAutomation,
+} = setup.structures
 
 jest.setTimeout(30000)
 
@@ -24,6 +36,7 @@ describe("/automations", () => {
   })
 
   beforeEach(() => {
+    // @ts-ignore
     events.automation.deleted.mockClear()
   })
 
@@ -32,7 +45,7 @@ describe("/automations", () => {
       const res = await request
         .get(`/api/automations/action/list`)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
 
       expect(Object.keys(res.body).length).not.toEqual(0)
@@ -42,7 +55,7 @@ describe("/automations", () => {
       const res = await request
         .get(`/api/automations/trigger/list`)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
 
       expect(Object.keys(res.body).length).not.toEqual(0)
@@ -52,14 +65,18 @@ describe("/automations", () => {
       const res = await request
         .get(`/api/automations/definitions/list`)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
 
       let definitionsLength = Object.keys(BUILTIN_ACTION_DEFINITIONS).length
       definitionsLength-- // OUTGOING_WEBHOOK is deprecated
 
-      expect(Object.keys(res.body.action).length).toBeGreaterThanOrEqual(definitionsLength)
-      expect(Object.keys(res.body.trigger).length).toEqual(Object.keys(TRIGGER_DEFINITIONS).length)
+      expect(Object.keys(res.body.action).length).toBeGreaterThanOrEqual(
+        definitionsLength
+      )
+      expect(Object.keys(res.body.trigger).length).toEqual(
+        Object.keys(TRIGGER_DEFINITIONS).length
+      )
     })
   })
 
@@ -72,7 +89,7 @@ describe("/automations", () => {
         .post(`/api/automations`)
         .set(config.defaultHeaders())
         .send(automation)
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
 
       expect(res.body.message).toEqual("Automation created successfully")
@@ -91,7 +108,7 @@ describe("/automations", () => {
         .post(`/api/automations`)
         .set(config.defaultHeaders())
         .send(automation)
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
 
       expect(res.body.message).toEqual("Automation created successfully")
@@ -107,7 +124,7 @@ describe("/automations", () => {
         config,
         method: "POST",
         url: `/api/automations`,
-        body: automation
+        body: automation,
       })
     })
   })
@@ -118,7 +135,7 @@ describe("/automations", () => {
       const res = await request
         .get(`/api/automations/${automation._id}`)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
       expect(res.body._id).toEqual(automation._id)
       expect(res.body._rev).toEqual(automation._rev)
@@ -134,8 +151,8 @@ describe("/automations", () => {
         row: {
           name: "{{trigger.row.name}}",
           description: "{{trigger.row.description}}",
-          tableId: table._id
-        }
+          tableId: table._id,
+        },
       }
       automation.appId = config.appId
       automation = await config.createAutomation(automation)
@@ -162,23 +179,68 @@ describe("/automations", () => {
     })
   })
 
-  describe("update", () => {
+  describe("trigger", () => {
+    it("does not trigger an automation when not synchronous and in dev", async () => {
+      let automation = newAutomation()
+      automation = await config.createAutomation(automation)
+      const res = await request
+        .post(`/api/automations/${automation._id}/trigger`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(400)
+
+      expect(res.body.message).toEqual(
+        "Only apps in production support this endpoint"
+      )
+    })
+
+    it("triggers a synchronous automation", async () => {
+      mocks.licenses.useSyncAutomations()
+      let automation = collectAutomation()
+      automation = await config.createAutomation(automation)
+      const res = await request
+        .post(`/api/automations/${automation._id}/trigger`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+
+      expect(res.body.success).toEqual(true)
+      expect(res.body.value).toEqual([1, 2, 3])
+    })
 
-    const update = async (automation) => {
+    it("triggers an asynchronous automation", async () => {
+      let automation = newAutomation()
+      automation = await config.createAutomation(automation)
+      await config.publish()
+
+      const res = await request
+        .post(`/api/automations/${automation._id}/trigger`)
+        .set(config.defaultHeaders({}, true))
+        .expect("Content-Type", /json/)
+        .expect(200)
+
+      expect(res.body.message).toEqual(
+        `Automation ${automation._id} has been triggered.`
+      )
+    })
+  })
+
+  describe("update", () => {
+    const update = async (automation: Automation) => {
       return request
         .put(`/api/automations`)
         .set(config.defaultHeaders())
         .send(automation)
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
     }
 
-    const updateWithPost = async (automation) => {
+    const updateWithPost = async (automation: Automation) => {
       return request
         .post(`/api/automations`)
         .set(config.defaultHeaders())
         .send(automation)
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
     }
 
@@ -199,7 +261,9 @@ describe("/automations", () => {
       expect(automationRes._rev).not.toEqual(automation._rev)
       // content updates
       expect(automationRes.name).toEqual("Updated Name")
-      expect(message).toEqual(`Automation ${automation._id} updated successfully.`)
+      expect(message).toEqual(
+        `Automation ${automation._id} updated successfully.`
+      )
       // events
       expect(events.automation.created).not.toBeCalled()
       expect(events.automation.stepCreated).not.toBeCalled()
@@ -207,7 +271,6 @@ describe("/automations", () => {
       expect(events.automation.triggerUpdated).not.toBeCalled()
     })
 
-
     it("updates a automations name using POST request", async () => {
       let automation = newAutomation()
       await config.createAutomation(automation)
@@ -226,7 +289,9 @@ describe("/automations", () => {
       expect(automationRes._rev).not.toEqual(automation._rev)
       // content updates
       expect(automationRes.name).toEqual("Updated Name")
-      expect(message).toEqual(`Automation ${automation._id} updated successfully.`)
+      expect(message).toEqual(
+        `Automation ${automation._id} updated successfully.`
+      )
       // events
       expect(events.automation.created).not.toBeCalled()
       expect(events.automation.stepCreated).not.toBeCalled()
@@ -237,7 +302,9 @@ describe("/automations", () => {
     it("updates an automation trigger", async () => {
       let automation = newAutomation()
       automation = await config.createAutomation(automation)
-      automation.definition.trigger = automationTrigger(TRIGGER_DEFINITIONS.WEBHOOK)
+      automation.definition.trigger = automationTrigger(
+        TRIGGER_DEFINITIONS.WEBHOOK
+      )
       jest.clearAllMocks()
 
       await update(automation)
@@ -266,7 +333,6 @@ describe("/automations", () => {
       expect(events.automation.triggerUpdated).not.toBeCalled()
     })
 
-
     it("removes automation steps", async () => {
       let automation = newAutomation()
       automation.definition.steps.push(automationStep())
@@ -305,11 +371,11 @@ describe("/automations", () => {
     it("return all the automations for an instance", async () => {
       await clearAllAutomations(config)
       const autoConfig = basicAutomation()
-      automation = await config.createAutomation(autoConfig)
+      await config.createAutomation(autoConfig)
       const res = await request
         .get(`/api/automations`)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
 
       expect(res.body[0]).toEqual(expect.objectContaining(autoConfig))
@@ -330,7 +396,7 @@ describe("/automations", () => {
       const res = await request
         .delete(`/api/automations/${automation.id}/${automation.rev}`)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
 
       expect(res.body.id).toEqual(automation._id)
@@ -346,4 +412,13 @@ describe("/automations", () => {
       })
     })
   })
+
+  describe("checkForCollectStep", () => {
+    it("should return true if a collect step exists in an automation", async () => {
+      let automation = collectAutomation()
+      await config.createAutomation(automation)
+      let res = await sdk.automations.utils.checkForCollectStep(automation)
+      expect(res).toEqual(true)
+    })
+  })
 })

package/src/api/routes/tests/datasource.spec.ts

@@ -87,7 +87,7 @@ describe("/datasources", () => {
         expect(contents.rows.length).toEqual(1)
 
         // update the datasource to remove the variables
-        datasource.config.dynamicVariables = []
+        datasource.config!.dynamicVariables = []
         const res = await request
           .put(`/api/datasources/${datasource._id}`)
           .send(datasource)

package/src/api/routes/tests/view.spec.js

@@ -114,8 +114,8 @@ describe("/views", () => {
       expect(res.body.tableId).toBe(table._id)
 
       const updatedTable = await config.getTable(table._id)
-      expect(updatedTable.views).toEqual({
-        TestView: {
+      const expectedObj = expect.objectContaining({
+        TestView: expect.objectContaining({
           field: "Price",
           calculation: "stats",
           tableId: table._id,
@@ -143,8 +143,9 @@ describe("/views", () => {
               type: "string",
             },
           },
-        },
+        }),
       })
+      expect(updatedTable.views).toEqual(expectedObj)
     })
   })
 

package/src/api/routes/tests/{webhook.spec.js → webhook.spec.ts}

@@ -1,11 +1,13 @@
-const setup = require("./utilities")
-const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
-const { basicWebhook, basicAutomation } = setup.structures
+import { Webhook } from "@budibase/types"
+import * as setup from "./utilities"
+import { checkBuilderEndpoint } from "./utilities/TestFunctions"
+import { mocks } from "@budibase/backend-core/tests"
+const { basicWebhook, basicAutomation, collectAutomation } = setup.structures
 
 describe("/webhooks", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
-  let webhook
+  let webhook: Webhook
 
   afterAll(setup.afterAll)
 
@@ -13,10 +15,11 @@ describe("/webhooks", () => {
     config.modeSelf()
     await config.init()
     const autoConfig = basicAutomation()
-    autoConfig.definition.trigger = {
-      schema: { outputs: { properties: {} } },
-      inputs: {},
+    autoConfig.definition.trigger.schema = {
+      outputs: { properties: {} },
+      inputs: { properties: {} },
     }
+    autoConfig.definition.trigger.inputs = {}
     await config.createAutomation(autoConfig)
     webhook = await config.createWebhook()
   }
@@ -70,7 +73,7 @@ describe("/webhooks", () => {
 
   describe("delete", () => {
     beforeAll(setupTest)
-    
+
     it("should successfully delete", async () => {
       const res = await request
         .delete(`/api/webhooks/${webhook._id}/${webhook._rev}`)
@@ -97,7 +100,7 @@ describe("/webhooks", () => {
       const res = await request
         .post(`/api/webhooks/schema/${config.getAppId()}/${webhook._id}`)
         .send({
-          a: 1
+          a: 1,
         })
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
@@ -112,7 +115,7 @@ describe("/webhooks", () => {
       expect(fetch.body[0]).toBeDefined()
       expect(fetch.body[0].bodySchema).toEqual({
         properties: {
-          a: { type: "integer" }
+          a: { type: "integer" },
         },
         type: "object",
       })
@@ -131,4 +134,23 @@ describe("/webhooks", () => {
       expect(res.body.message).toBeDefined()
     })
   })
-})
+
+  it("should trigger a synchronous webhook call ", async () => {
+    mocks.licenses.useSyncAutomations()
+    let automation = collectAutomation()
+    let newAutomation = await config.createAutomation(automation)
+    let syncWebhook = await config.createWebhook(
+      basicWebhook(newAutomation._id)
+    )
+
+    // replicate changes before checking webhook
+    await config.publish()
+
+    const res = await request
+      .post(`/api/webhooks/trigger/${config.prodAppId}/${syncWebhook._id}`)
+      .expect("Content-Type", /json/)
+      .expect(200)
+    expect(res.body.success).toEqual(true)
+    expect(res.body.value).toEqual([1, 2, 3])
+  })
+})

package/src/app.ts

@@ -61,7 +61,6 @@ if (env.isProd()) {
 
 const server = http.createServer(app.callback())
 destroyable(server)
-initialiseWebsockets(app, server)
 
 let shuttingDown = false,
   errCode = 0

package/src/automations/actions.ts

@@ -14,6 +14,7 @@ import * as filter from "./steps/filter"
 import * as delay from "./steps/delay"
 import * as queryRow from "./steps/queryRows"
 import * as loop from "./steps/loop"
+import * as collect from "./steps/collect"
 import env from "../environment"
 import {
   AutomationStepSchema,
@@ -39,6 +40,7 @@ const ACTION_IMPLS: Record<
   DELAY: delay.run,
   FILTER: filter.run,
   QUERY_ROWS: queryRow.run,
+  COLLECT: collect.run,
   // these used to be lowercase step IDs, maintain for backwards compat
   discord: discord.run,
   slack: slack.run,
@@ -59,6 +61,7 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
     FILTER: filter.definition,
     QUERY_ROWS: queryRow.definition,
     LOOP: loop.definition,
+    COLLECT: collect.definition,
     // these used to be lowercase step IDs, maintain for backwards compat
     discord: discord.definition,
     slack: slack.definition,
@@ -71,10 +74,15 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
 // ran at all
 if (env.SELF_HOSTED) {
   const bash = require("./steps/bash")
+  const openai = require("./steps/openai")
+
   // @ts-ignore
   ACTION_IMPLS["EXECUTE_BASH"] = bash.run
   // @ts-ignore
   BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
+
+  ACTION_IMPLS.OPENAI = openai.run
+  BUILTIN_ACTION_DEFINITIONS.OPENAI = openai.definition
 }
 
 export async function getActionDefinitions() {

package/src/automations/logging/index.ts

@@ -2,6 +2,23 @@ import env from "../../environment"
 import { AutomationResults, Automation, App } from "@budibase/types"
 import { automations } from "@budibase/pro"
 import { db as dbUtils } from "@budibase/backend-core"
+import sizeof from "object-sizeof"
+
+const MAX_LOG_SIZE_MB = 5
+const MB_IN_BYTES = 1024 * 1024
+
+function sanitiseResults(results: AutomationResults) {
+  const message = `[removed] - max results size of ${MAX_LOG_SIZE_MB}MB exceeded`
+  for (let step of results.steps) {
+    step.inputs = {
+      message,
+    }
+    step.outputs = {
+      message,
+      success: step.outputs.success,
+    }
+  }
+}
 
 export async function storeLog(
   automation: Automation,
@@ -11,6 +28,10 @@ export async function storeLog(
   if (env.DISABLE_AUTOMATION_LOGS) {
     return
   }
+  const bytes = sizeof(results)
+  if (bytes / MB_IN_BYTES > MAX_LOG_SIZE_MB) {
+    sanitiseResults(results)
+  }
   await automations.logs.storeLog(automation, results)
 }
 

package/src/automations/steps/bash.ts

@@ -5,6 +5,7 @@ import environment from "../../environment"
 import {
   AutomationActionStepId,
   AutomationCustomIOType,
+  AutomationFeature,
   AutomationIOType,
   AutomationStepInput,
   AutomationStepSchema,
@@ -18,6 +19,9 @@ export const definition: AutomationStepSchema = {
   description: "Run a bash script",
   type: AutomationStepType.ACTION,
   internal: true,
+  features: {
+    [AutomationFeature.LOOPING]: true,
+  },
   stepId: AutomationActionStepId.EXECUTE_BASH,
   inputs: {},
   schema: {

package/src/automations/steps/collect.ts

@@ -0,0 +1,58 @@
+import {
+  AutomationActionStepId,
+  AutomationStepSchema,
+  AutomationStepInput,
+  AutomationStepType,
+  AutomationIOType,
+  AutomationFeature,
+} from "@budibase/types"
+
+export const definition: AutomationStepSchema = {
+  name: "Collect Data",
+  tagline: "Collect data to be sent to design",
+  icon: "Collection",
+  description:
+    "Collects specified data so it can be provided to the design section",
+  type: AutomationStepType.ACTION,
+  internal: true,
+  features: {},
+  stepId: AutomationActionStepId.COLLECT,
+  inputs: {},
+  schema: {
+    inputs: {
+      properties: {
+        collection: {
+          type: AutomationIOType.STRING,
+          title: "What to Collect",
+        },
+      },
+      required: ["collection"],
+    },
+    outputs: {
+      properties: {
+        success: {
+          type: AutomationIOType.BOOLEAN,
+          description: "Whether the action was successful",
+        },
+        value: {
+          type: AutomationIOType.STRING,
+          description: "Collected data",
+        },
+      },
+      required: ["success", "value"],
+    },
+  },
+}
+
+export async function run({ inputs }: AutomationStepInput) {
+  if (!inputs.collection) {
+    return {
+      success: false,
+    }
+  } else {
+    return {
+      success: true,
+      value: inputs.collection,
+    }
+  }
+}

package/src/automations/steps/createRow.ts

@@ -4,6 +4,7 @@ import { buildCtx } from "./utils"
 import {
   AutomationActionStepId,
   AutomationCustomIOType,
+  AutomationFeature,
   AutomationIOType,
   AutomationStepInput,
   AutomationStepSchema,
@@ -17,6 +18,9 @@ export const definition: AutomationStepSchema = {
   description: "Add a row to your database",
   type: AutomationStepType.ACTION,
   internal: true,
+  features: {
+    [AutomationFeature.LOOPING]: true,
+  },
   stepId: AutomationActionStepId.CREATE_ROW,
   inputs: {},
   schema: {

package/src/automations/steps/delay.ts

@@ -14,6 +14,7 @@ export const definition: AutomationStepSchema = {
   description: "Delay the automation until an amount of time has passed",
   stepId: AutomationActionStepId.DELAY,
   internal: true,
+  features: {},
   inputs: {},
   schema: {
     inputs: {

package/src/automations/steps/deleteRow.ts

@@ -8,6 +8,7 @@ import {
   AutomationStepType,
   AutomationIOType,
   AutomationCustomIOType,
+  AutomationFeature,
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
@@ -18,6 +19,9 @@ export const definition: AutomationStepSchema = {
   type: AutomationStepType.ACTION,
   stepId: AutomationActionStepId.DELETE_ROW,
   internal: true,
+  features: {
+    [AutomationFeature.LOOPING]: true,
+  },
   inputs: {},
   schema: {
     inputs: {

package/src/automations/steps/discord.ts

@@ -6,6 +6,7 @@ import {
   AutomationStepInput,
   AutomationStepType,
   AutomationIOType,
+  AutomationFeature,
 } from "@budibase/types"
 
 const DEFAULT_USERNAME = "Budibase Automate"
@@ -19,6 +20,9 @@ export const definition: AutomationStepSchema = {
   stepId: AutomationActionStepId.discord,
   type: AutomationStepType.ACTION,
   internal: false,
+  features: {
+    [AutomationFeature.LOOPING]: true,
+  },
   inputs: {},
   schema: {
     inputs: {

package/src/automations/steps/executeQuery.ts

@@ -4,6 +4,7 @@ import * as automationUtils from "../automationUtils"
 import {
   AutomationActionStepId,
   AutomationCustomIOType,
+  AutomationFeature,
   AutomationIOType,
   AutomationStepInput,
   AutomationStepSchema,
@@ -18,6 +19,9 @@ export const definition: AutomationStepSchema = {
   type: AutomationStepType.ACTION,
   stepId: AutomationActionStepId.EXECUTE_QUERY,
   internal: true,
+  features: {
+    [AutomationFeature.LOOPING]: true,
+  },
   inputs: {},
   schema: {
     inputs: {

package/src/automations/steps/executeScript.ts

@@ -4,6 +4,7 @@ import * as automationUtils from "../automationUtils"
 import {
   AutomationActionStepId,
   AutomationCustomIOType,
+  AutomationFeature,
   AutomationIOType,
   AutomationStepInput,
   AutomationStepSchema,
@@ -19,6 +20,9 @@ export const definition: AutomationStepSchema = {
   internal: true,
   stepId: AutomationActionStepId.EXECUTE_SCRIPT,
   inputs: {},
+  features: {
+    [AutomationFeature.LOOPING]: true,
+  },
   schema: {
     inputs: {
       properties: {

package/src/automations/steps/filter.ts

@@ -28,6 +28,7 @@ export const definition: AutomationStepSchema = {
     "Conditionally halt automations which do not meet certain conditions",
   type: AutomationStepType.LOGIC,
   internal: true,
+  features: {},
   stepId: AutomationActionStepId.FILTER,
   inputs: {
     condition: FilterConditions.EQUAL,

package/src/automations/steps/loop.ts

@@ -13,6 +13,7 @@ export const definition: AutomationStepSchema = {
   description: "Loop",
   stepId: AutomationActionStepId.LOOP,
   internal: true,
+  features: {},
   inputs: {},
   schema: {
     inputs: {