@budibase/server 2.6.19-alpha.3 → 2.6.19-alpha.31

package/src/api/controllers/view/index.ts

@@ -16,6 +16,7 @@ import {
   View,
 } from "@budibase/types"
 import { cleanExportRows } from "../row/utils"
+import { builderSocket } from "../../../websockets"
 
 const { cloneDeep, isEqual } = require("lodash")
 
@@ -48,7 +49,7 @@ export async function save(ctx: Ctx) {
   if (!view.meta.schema) {
     view.meta.schema = table.schema
   }
-  table.views[viewName] = view.meta
+  table.views[viewName] = { ...view.meta, name: viewName }
   if (originalName) {
     delete table.views[originalName]
     existingTable.views[viewName] = existingTable.views[originalName]
@@ -56,10 +57,8 @@ export async function save(ctx: Ctx) {
   await db.put(table)
   await handleViewEvents(existingTable.views[viewName], table.views[viewName])
 
-  ctx.body = {
-    ...table.views[viewToSave.name],
-    name: viewToSave.name,
-  }
+  ctx.body = table.views[viewName]
+  builderSocket.emitTableUpdate(ctx, table)
 }
 
 export async function calculationEvents(existingView: View, newView: View) {
@@ -128,6 +127,7 @@ export async function destroy(ctx: Ctx) {
   await events.view.deleted(view)
 
   ctx.body = view
+  builderSocket.emitTableUpdate(ctx, table)
 }
 
 export async function exportView(ctx: Ctx) {

package/src/api/controllers/webhook.ts

@@ -6,8 +6,11 @@ import {
   WebhookActionType,
   BBContext,
   Automation,
+  AutomationActionStepId,
 } from "@budibase/types"
 import sdk from "../../sdk"
+import * as pro from "@budibase/pro"
+
 const toJsonSchema = require("to-json-schema")
 const validate = require("jsonschema").validate
 
@@ -78,15 +81,36 @@ export async function trigger(ctx: BBContext) {
       if (webhook.action.type === WebhookActionType.AUTOMATION) {
         // trigger with both the pure request and then expand it
         // incase the user has produced a schema to bind to
-        await triggers.externalTrigger(target, {
-          body: ctx.request.body,
-          ...ctx.request.body,
-          appId: prodAppId,
-        })
-      }
-      ctx.status = 200
-      ctx.body = {
-        message: "Webhook trigger fired successfully",
+        let hasCollectStep = sdk.automations.utils.checkForCollectStep(target)
+
+        if (hasCollectStep && (await pro.features.isSyncAutomationsEnabled())) {
+          const response = await triggers.externalTrigger(
+            target,
+            {
+              body: ctx.request.body,
+              ...ctx.request.body,
+              appId: prodAppId,
+            },
+            { getResponses: true }
+          )
+
+          let collectedValue = response.steps.find(
+            (step: any) => step.stepId === AutomationActionStepId.COLLECT
+          )
+
+          ctx.status = 200
+          ctx.body = collectedValue.outputs
+        } else {
+          await triggers.externalTrigger(target, {
+            body: ctx.request.body,
+            ...ctx.request.body,
+            appId: prodAppId,
+          })
+          ctx.status = 200
+          ctx.body = {
+            message: "Webhook trigger fired successfully",
+          }
+        }
       }
     } catch (err: any) {
       if (err.status === 404) {

package/src/api/routes/application.ts

@@ -38,6 +38,11 @@ router
     authorized(permissions.BUILDER),
     controller.revertClient
   )
+  .post(
+    "/api/applications/:appId/sample",
+    authorized(permissions.BUILDER),
+    controller.addSampleData
+  )
   .post(
     "/api/applications/:appId/publish",
     authorized(permissions.BUILDER),

package/src/api/routes/automation.ts

@@ -65,7 +65,6 @@ router
   )
   .post(
     "/api/automations/:id/trigger",
-    appInfoMiddleware({ appType: AppType.PROD }),
     paramResource("id"),
     authorized(
       permissions.PermissionType.AUTOMATION,

package/src/api/routes/datasource.ts

@@ -20,6 +20,11 @@ router
     authorized(permissions.BUILDER),
     datasourceController.verify
   )
+  .get(
+    "/api/datasources/:datasourceId/info",
+    authorized(permissions.BUILDER),
+    datasourceController.information
+  )
   .get(
     "/api/datasources/:datasourceId",
     authorized(

package/src/api/routes/static.ts

@@ -5,6 +5,7 @@ import authorized from "../../middleware/authorized"
 import { permissions } from "@budibase/backend-core"
 import env from "../../environment"
 import { paramResource } from "../../middleware/resourceId"
+import { devClientLibPath } from "../../utilities/fileSystem"
 const { BUILDER, PermissionType, PermissionLevel } = permissions
 
 const router: Router = new Router()
@@ -17,7 +18,8 @@ router.param("file", async (file: any, ctx: any, next: any) => {
   }
   // test serves from require
   if (env.isTest()) {
-    ctx.devPath = require.resolve("@budibase/client").split(ctx.file)[0]
+    const path = devClientLibPath()
+    ctx.devPath = path.split(ctx.file)[0]
   } else if (env.isDev()) {
     // Serving the client library from your local dir in dev
     ctx.devPath = budibaseTempDir()

package/src/api/routes/tests/{automation.spec.js → automation.spec.ts}

@@ -1,15 +1,27 @@
-const {
+import {
   checkBuilderEndpoint,
   getAllTableRows,
   clearAllAutomations,
   testAutomation,
-} = require("./utilities/TestFunctions")
-const setup = require("./utilities")
-const { basicAutomation, newAutomation, automationTrigger, automationStep } = setup.structures
-const MAX_RETRIES = 4
-const { TRIGGER_DEFINITIONS, BUILTIN_ACTION_DEFINITIONS } = require("../../../automations")
-const { events } = require("@budibase/backend-core")
+} from "./utilities/TestFunctions"
+import * as setup from "./utilities"
+import {
+  TRIGGER_DEFINITIONS,
+  BUILTIN_ACTION_DEFINITIONS,
+} from "../../../automations"
+import { events } from "@budibase/backend-core"
+import sdk from "../../../sdk"
+import { Automation } from "@budibase/types"
+import { mocks } from "@budibase/backend-core/tests"
 
+const MAX_RETRIES = 4
+let {
+  basicAutomation,
+  newAutomation,
+  automationTrigger,
+  automationStep,
+  collectAutomation,
+} = setup.structures
 
 jest.setTimeout(30000)
 
@@ -24,6 +36,7 @@ describe("/automations", () => {
   })
 
   beforeEach(() => {
+    // @ts-ignore
     events.automation.deleted.mockClear()
   })
 
@@ -32,7 +45,7 @@ describe("/automations", () => {
       const res = await request
         .get(`/api/automations/action/list`)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
 
       expect(Object.keys(res.body).length).not.toEqual(0)
@@ -42,7 +55,7 @@ describe("/automations", () => {
       const res = await request
         .get(`/api/automations/trigger/list`)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
 
       expect(Object.keys(res.body).length).not.toEqual(0)
@@ -52,14 +65,18 @@ describe("/automations", () => {
       const res = await request
         .get(`/api/automations/definitions/list`)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
 
       let definitionsLength = Object.keys(BUILTIN_ACTION_DEFINITIONS).length
       definitionsLength-- // OUTGOING_WEBHOOK is deprecated
 
-      expect(Object.keys(res.body.action).length).toBeGreaterThanOrEqual(definitionsLength)
-      expect(Object.keys(res.body.trigger).length).toEqual(Object.keys(TRIGGER_DEFINITIONS).length)
+      expect(Object.keys(res.body.action).length).toBeGreaterThanOrEqual(
+        definitionsLength
+      )
+      expect(Object.keys(res.body.trigger).length).toEqual(
+        Object.keys(TRIGGER_DEFINITIONS).length
+      )
     })
   })
 
@@ -72,7 +89,7 @@ describe("/automations", () => {
         .post(`/api/automations`)
         .set(config.defaultHeaders())
         .send(automation)
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
 
       expect(res.body.message).toEqual("Automation created successfully")
@@ -91,7 +108,7 @@ describe("/automations", () => {
         .post(`/api/automations`)
         .set(config.defaultHeaders())
         .send(automation)
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
 
       expect(res.body.message).toEqual("Automation created successfully")
@@ -107,7 +124,7 @@ describe("/automations", () => {
         config,
         method: "POST",
         url: `/api/automations`,
-        body: automation
+        body: automation,
       })
     })
   })
@@ -118,7 +135,7 @@ describe("/automations", () => {
       const res = await request
         .get(`/api/automations/${automation._id}`)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
       expect(res.body._id).toEqual(automation._id)
       expect(res.body._rev).toEqual(automation._rev)
@@ -134,8 +151,8 @@ describe("/automations", () => {
         row: {
           name: "{{trigger.row.name}}",
           description: "{{trigger.row.description}}",
-          tableId: table._id
-        }
+          tableId: table._id,
+        },
       }
       automation.appId = config.appId
       automation = await config.createAutomation(automation)
@@ -162,23 +179,68 @@ describe("/automations", () => {
     })
   })
 
-  describe("update", () => {
+  describe("trigger", () => {
+    it("does not trigger an automation when not synchronous and in dev", async () => {
+      let automation = newAutomation()
+      automation = await config.createAutomation(automation)
+      const res = await request
+        .post(`/api/automations/${automation._id}/trigger`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(400)
+
+      expect(res.body.message).toEqual(
+        "Only apps in production support this endpoint"
+      )
+    })
+
+    it("triggers a synchronous automation", async () => {
+      mocks.licenses.useSyncAutomations()
+      let automation = collectAutomation()
+      automation = await config.createAutomation(automation)
+      const res = await request
+        .post(`/api/automations/${automation._id}/trigger`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+
+      expect(res.body.success).toEqual(true)
+      expect(res.body.value).toEqual([1, 2, 3])
+    })
 
-    const update = async (automation) => {
+    it("triggers an asynchronous automation", async () => {
+      let automation = newAutomation()
+      automation = await config.createAutomation(automation)
+      await config.publish()
+
+      const res = await request
+        .post(`/api/automations/${automation._id}/trigger`)
+        .set(config.defaultHeaders({}, true))
+        .expect("Content-Type", /json/)
+        .expect(200)
+
+      expect(res.body.message).toEqual(
+        `Automation ${automation._id} has been triggered.`
+      )
+    })
+  })
+
+  describe("update", () => {
+    const update = async (automation: Automation) => {
       return request
         .put(`/api/automations`)
         .set(config.defaultHeaders())
         .send(automation)
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
     }
 
-    const updateWithPost = async (automation) => {
+    const updateWithPost = async (automation: Automation) => {
       return request
         .post(`/api/automations`)
         .set(config.defaultHeaders())
         .send(automation)
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
     }
 
@@ -199,7 +261,9 @@ describe("/automations", () => {
       expect(automationRes._rev).not.toEqual(automation._rev)
       // content updates
       expect(automationRes.name).toEqual("Updated Name")
-      expect(message).toEqual(`Automation ${automation._id} updated successfully.`)
+      expect(message).toEqual(
+        `Automation ${automation._id} updated successfully.`
+      )
       // events
       expect(events.automation.created).not.toBeCalled()
       expect(events.automation.stepCreated).not.toBeCalled()
@@ -207,7 +271,6 @@ describe("/automations", () => {
       expect(events.automation.triggerUpdated).not.toBeCalled()
     })
 
-
     it("updates a automations name using POST request", async () => {
       let automation = newAutomation()
       await config.createAutomation(automation)
@@ -226,7 +289,9 @@ describe("/automations", () => {
       expect(automationRes._rev).not.toEqual(automation._rev)
       // content updates
       expect(automationRes.name).toEqual("Updated Name")
-      expect(message).toEqual(`Automation ${automation._id} updated successfully.`)
+      expect(message).toEqual(
+        `Automation ${automation._id} updated successfully.`
+      )
       // events
       expect(events.automation.created).not.toBeCalled()
       expect(events.automation.stepCreated).not.toBeCalled()
@@ -237,7 +302,9 @@ describe("/automations", () => {
     it("updates an automation trigger", async () => {
       let automation = newAutomation()
       automation = await config.createAutomation(automation)
-      automation.definition.trigger = automationTrigger(TRIGGER_DEFINITIONS.WEBHOOK)
+      automation.definition.trigger = automationTrigger(
+        TRIGGER_DEFINITIONS.WEBHOOK
+      )
       jest.clearAllMocks()
 
       await update(automation)
@@ -266,7 +333,6 @@ describe("/automations", () => {
       expect(events.automation.triggerUpdated).not.toBeCalled()
     })
 
-
     it("removes automation steps", async () => {
       let automation = newAutomation()
       automation.definition.steps.push(automationStep())
@@ -305,11 +371,11 @@ describe("/automations", () => {
     it("return all the automations for an instance", async () => {
       await clearAllAutomations(config)
       const autoConfig = basicAutomation()
-      automation = await config.createAutomation(autoConfig)
+      await config.createAutomation(autoConfig)
       const res = await request
         .get(`/api/automations`)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
 
       expect(res.body[0]).toEqual(expect.objectContaining(autoConfig))
@@ -330,7 +396,7 @@ describe("/automations", () => {
       const res = await request
         .delete(`/api/automations/${automation.id}/${automation.rev}`)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
 
       expect(res.body.id).toEqual(automation._id)
@@ -346,4 +412,13 @@ describe("/automations", () => {
       })
     })
   })
+
+  describe("checkForCollectStep", () => {
+    it("should return true if a collect step exists in an automation", async () => {
+      let automation = collectAutomation()
+      await config.createAutomation(automation)
+      let res = await sdk.automations.utils.checkForCollectStep(automation)
+      expect(res).toEqual(true)
+    })
+  })
 })

package/src/api/routes/tests/datasource.spec.ts

@@ -87,7 +87,7 @@ describe("/datasources", () => {
         expect(contents.rows.length).toEqual(1)
 
         // update the datasource to remove the variables
-        datasource.config.dynamicVariables = []
+        datasource.config!.dynamicVariables = []
         const res = await request
           .put(`/api/datasources/${datasource._id}`)
           .send(datasource)

package/src/api/routes/tests/{webhook.spec.js → webhook.spec.ts}

@@ -1,11 +1,13 @@
-const setup = require("./utilities")
-const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
-const { basicWebhook, basicAutomation } = setup.structures
+import { Webhook } from "@budibase/types"
+import * as setup from "./utilities"
+import { checkBuilderEndpoint } from "./utilities/TestFunctions"
+import { mocks } from "@budibase/backend-core/tests"
+const { basicWebhook, basicAutomation, collectAutomation } = setup.structures
 
 describe("/webhooks", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
-  let webhook
+  let webhook: Webhook
 
   afterAll(setup.afterAll)
 
@@ -13,10 +15,11 @@ describe("/webhooks", () => {
     config.modeSelf()
     await config.init()
     const autoConfig = basicAutomation()
-    autoConfig.definition.trigger = {
-      schema: { outputs: { properties: {} } },
-      inputs: {},
+    autoConfig.definition.trigger.schema = {
+      outputs: { properties: {} },
+      inputs: { properties: {} },
     }
+    autoConfig.definition.trigger.inputs = {}
     await config.createAutomation(autoConfig)
     webhook = await config.createWebhook()
   }
@@ -70,7 +73,7 @@ describe("/webhooks", () => {
 
   describe("delete", () => {
     beforeAll(setupTest)
-    
+
     it("should successfully delete", async () => {
       const res = await request
         .delete(`/api/webhooks/${webhook._id}/${webhook._rev}`)
@@ -97,7 +100,7 @@ describe("/webhooks", () => {
       const res = await request
         .post(`/api/webhooks/schema/${config.getAppId()}/${webhook._id}`)
         .send({
-          a: 1
+          a: 1,
         })
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
@@ -112,7 +115,7 @@ describe("/webhooks", () => {
       expect(fetch.body[0]).toBeDefined()
       expect(fetch.body[0].bodySchema).toEqual({
         properties: {
-          a: { type: "integer" }
+          a: { type: "integer" },
         },
         type: "object",
       })
@@ -131,4 +134,23 @@ describe("/webhooks", () => {
       expect(res.body.message).toBeDefined()
     })
   })
-})
+
+  it("should trigger a synchronous webhook call ", async () => {
+    mocks.licenses.useSyncAutomations()
+    let automation = collectAutomation()
+    let newAutomation = await config.createAutomation(automation)
+    let syncWebhook = await config.createWebhook(
+      basicWebhook(newAutomation._id)
+    )
+
+    // replicate changes before checking webhook
+    await config.publish()
+
+    const res = await request
+      .post(`/api/webhooks/trigger/${config.prodAppId}/${syncWebhook._id}`)
+      .expect("Content-Type", /json/)
+      .expect(200)
+    expect(res.body.success).toEqual(true)
+    expect(res.body.value).toEqual([1, 2, 3])
+  })
+})

package/src/app.ts

@@ -61,7 +61,6 @@ if (env.isProd()) {
 
 const server = http.createServer(app.callback())
 destroyable(server)
-initialiseWebsockets(app, server)
 
 let shuttingDown = false,
   errCode = 0

package/src/automations/actions.ts

@@ -14,6 +14,7 @@ import * as filter from "./steps/filter"
 import * as delay from "./steps/delay"
 import * as queryRow from "./steps/queryRows"
 import * as loop from "./steps/loop"
+import * as collect from "./steps/collect"
 import env from "../environment"
 import {
   AutomationStepSchema,
@@ -39,6 +40,7 @@ const ACTION_IMPLS: Record<
   DELAY: delay.run,
   FILTER: filter.run,
   QUERY_ROWS: queryRow.run,
+  COLLECT: collect.run,
   // these used to be lowercase step IDs, maintain for backwards compat
   discord: discord.run,
   slack: slack.run,
@@ -59,6 +61,7 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
     FILTER: filter.definition,
     QUERY_ROWS: queryRow.definition,
     LOOP: loop.definition,
+    COLLECT: collect.definition,
     // these used to be lowercase step IDs, maintain for backwards compat
     discord: discord.definition,
     slack: slack.definition,
@@ -71,10 +74,15 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
 // ran at all
 if (env.SELF_HOSTED) {
   const bash = require("./steps/bash")
+  const openai = require("./steps/openai")
+
   // @ts-ignore
   ACTION_IMPLS["EXECUTE_BASH"] = bash.run
   // @ts-ignore
   BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
+
+  ACTION_IMPLS.OPENAI = openai.run
+  BUILTIN_ACTION_DEFINITIONS.OPENAI = openai.definition
 }
 
 export async function getActionDefinitions() {

package/src/automations/logging/index.ts

@@ -2,6 +2,23 @@ import env from "../../environment"
 import { AutomationResults, Automation, App } from "@budibase/types"
 import { automations } from "@budibase/pro"
 import { db as dbUtils } from "@budibase/backend-core"
+import sizeof from "object-sizeof"
+
+const MAX_LOG_SIZE_MB = 5
+const MB_IN_BYTES = 1024 * 1024
+
+function sanitiseResults(results: AutomationResults) {
+  const message = `[removed] - max results size of ${MAX_LOG_SIZE_MB}MB exceeded`
+  for (let step of results.steps) {
+    step.inputs = {
+      message,
+    }
+    step.outputs = {
+      message,
+      success: step.outputs.success,
+    }
+  }
+}
 
 export async function storeLog(
   automation: Automation,
@@ -11,6 +28,10 @@ export async function storeLog(
   if (env.DISABLE_AUTOMATION_LOGS) {
     return
   }
+  const bytes = sizeof(results)
+  if (bytes / MB_IN_BYTES > MAX_LOG_SIZE_MB) {
+    sanitiseResults(results)
+  }
   await automations.logs.storeLog(automation, results)
 }
 

package/src/automations/steps/bash.ts

@@ -5,6 +5,7 @@ import environment from "../../environment"
 import {
   AutomationActionStepId,
   AutomationCustomIOType,
+  AutomationFeature,
   AutomationIOType,
   AutomationStepInput,
   AutomationStepSchema,
@@ -18,6 +19,9 @@ export const definition: AutomationStepSchema = {
   description: "Run a bash script",
   type: AutomationStepType.ACTION,
   internal: true,
+  features: {
+    [AutomationFeature.LOOPING]: true,
+  },
   stepId: AutomationActionStepId.EXECUTE_BASH,
   inputs: {},
   schema: {