@budibase/server 2.5.6-alpha.9 → 2.5.7

package/src/api/controllers/row/internal.ts

@@ -30,6 +30,7 @@ import { finaliseRow, updateRelatedFormula } from "./staticFormula"
 import { csv, json, jsonWithSchema, Format } from "../view/exporters"
 import { apiFileReturn } from "../../../utilities/fileSystem"
 import {
+  Ctx,
   UserCtx,
   Database,
   LinkDocumentValue,
@@ -71,7 +72,7 @@ async function getView(db: Database, viewName: string) {
   return viewInfo
 }
 
-async function getRawTableData(ctx: UserCtx, db: Database, tableId: string) {
+async function getRawTableData(ctx: Ctx, db: Database, tableId: string) {
   let rows
   if (tableId === InternalTables.USER_METADATA) {
     await userController.fetchMetadata(ctx)
@@ -187,7 +188,7 @@ export async function save(ctx: UserCtx) {
   })
 }
 
-export async function fetchView(ctx: UserCtx) {
+export async function fetchView(ctx: Ctx) {
   const viewName = decodeURIComponent(ctx.params.viewName)
 
   // if this is a table view being looked for just transfer to that
@@ -254,7 +255,7 @@ export async function fetchView(ctx: UserCtx) {
   return rows
 }
 
-export async function fetch(ctx: UserCtx) {
+export async function fetch(ctx: Ctx) {
   const db = context.getAppDB()
 
   const tableId = ctx.params.tableId
@@ -263,7 +264,7 @@ export async function fetch(ctx: UserCtx) {
   return outputProcessing(table, rows)
 }
 
-export async function find(ctx: UserCtx) {
+export async function find(ctx: Ctx) {
   const db = dbCore.getDB(ctx.appId)
   const table = await db.get(ctx.params.tableId)
   let row = await utils.findRow(ctx, ctx.params.tableId, ctx.params.rowId)
@@ -271,7 +272,7 @@ export async function find(ctx: UserCtx) {
   return row
 }
 
-export async function destroy(ctx: UserCtx) {
+export async function destroy(ctx: Ctx) {
   const db = context.getAppDB()
   const { _id } = ctx.request.body
   let row = await db.get(_id)
@@ -307,7 +308,7 @@ export async function destroy(ctx: UserCtx) {
   return { response, row }
 }
 
-export async function bulkDestroy(ctx: UserCtx) {
+export async function bulkDestroy(ctx: Ctx) {
   const db = context.getAppDB()
   const tableId = ctx.params.tableId
   const table = await db.get(tableId)
@@ -346,7 +347,7 @@ export async function bulkDestroy(ctx: UserCtx) {
   return { response: { ok: true }, rows: processedRows }
 }
 
-export async function search(ctx: UserCtx) {
+export async function search(ctx: Ctx) {
   // Fetch the whole table when running in cypress, as search doesn't work
   if (!env.COUCH_DB_URL && env.isCypress()) {
     return { rows: await fetch(ctx) }
@@ -386,7 +387,7 @@ export async function search(ctx: UserCtx) {
   return response
 }
 
-export async function exportRows(ctx: UserCtx) {
+export async function exportRows(ctx: Ctx) {
   const db = context.getAppDB()
   const table = await db.get(ctx.params.tableId)
   const rowIds = ctx.request.body.rows
@@ -438,7 +439,7 @@ export async function exportRows(ctx: UserCtx) {
   }
 }
 
-export async function fetchEnrichedRow(ctx: UserCtx) {
+export async function fetchEnrichedRow(ctx: Ctx) {
   const db = context.getAppDB()
   const tableId = ctx.params.tableId
   const rowId = ctx.params.rowId

package/src/api/controllers/row/utils.ts

@@ -5,7 +5,7 @@ import { context } from "@budibase/backend-core"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { Row, Table } from "@budibase/types"
 import { Format } from "../view/exporters"
-import { UserCtx } from "@budibase/types"
+import { Ctx } from "@budibase/types"
 import sdk from "../../../sdk"
 const validateJs = require("validate.js")
 const { cloneDeep } = require("lodash/fp")
@@ -26,7 +26,7 @@ export async function getDatasourceAndQuery(json: any) {
   return makeExternalQuery(datasource, json)
 }
 
-export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
+export async function findRow(ctx: Ctx, tableId: string, rowId: string) {
   const db = context.getAppDB()
   let row
   // TODO remove special user case in future

package/src/api/controllers/table/utils.ts

@@ -124,8 +124,7 @@ export function importToRows(
     for (const [fieldName, schema] of Object.entries(table.schema)) {
       // check whether the options need to be updated for inclusion as part of the data import
       if (
-        (schema.type === FieldTypes.OPTIONS ||
-          schema.type === FieldTypes.ARRAY) &&
+        schema.type === FieldTypes.OPTIONS &&
         row[fieldName] &&
         (!schema.constraints!.inclusion ||
           schema.constraints!.inclusion.indexOf(row[fieldName]) === -1)

package/src/api/controllers/user.ts

@@ -1,12 +1,98 @@
 import { generateUserMetadataID, generateUserFlagID } from "../../db/utils"
 import { InternalTables } from "../../db/utils"
-import { getGlobalUsers } from "../../utilities/global"
+import { getGlobalUsers, getRawGlobalUser } from "../../utilities/global"
 import { getFullUser } from "../../utilities/users"
-import { context } from "@budibase/backend-core"
-import { UserCtx } from "@budibase/types"
+import {
+  context,
+  roles as rolesCore,
+  db as dbCore,
+} from "@budibase/backend-core"
+import { BBContext, Ctx, SyncUserRequest, User } from "@budibase/types"
 import sdk from "../../sdk"
 
-export async function fetchMetadata(ctx: UserCtx) {
+export async function syncUser(ctx: Ctx<SyncUserRequest>) {
+  let deleting = false,
+    user: User | any
+  const userId = ctx.params.id
+
+  const previousUser = ctx.request.body?.previousUser
+
+  try {
+    user = (await getRawGlobalUser(userId)) as User
+  } catch (err: any) {
+    if (err && err.status === 404) {
+      user = {}
+      deleting = true
+    } else {
+      throw err
+    }
+  }
+
+  let previousApps = previousUser
+    ? Object.keys(previousUser.roles).map(appId => appId)
+    : []
+
+  const roles = deleting ? {} : user.roles
+  // remove props which aren't useful to metadata
+  delete user.password
+  delete user.forceResetPassword
+  delete user.roles
+  // run through all production appIDs in the users roles
+  let prodAppIds
+  // if they are a builder then get all production app IDs
+  if ((user.builder && user.builder.global) || deleting) {
+    prodAppIds = await dbCore.getProdAppIDs()
+  } else {
+    prodAppIds = Object.entries(roles)
+      .filter(entry => entry[1] !== rolesCore.BUILTIN_ROLE_IDS.PUBLIC)
+      .map(([appId]) => appId)
+  }
+  for (let prodAppId of new Set([...prodAppIds, ...previousApps])) {
+    const roleId = roles[prodAppId]
+    const deleteFromApp = !roleId
+    const devAppId = dbCore.getDevelopmentAppID(prodAppId)
+    for (let appId of [prodAppId, devAppId]) {
+      if (!(await dbCore.dbExists(appId))) {
+        continue
+      }
+      await context.doInAppContext(appId, async () => {
+        const db = context.getAppDB()
+        const metadataId = generateUserMetadataID(userId)
+        let metadata
+        try {
+          metadata = await db.get(metadataId)
+        } catch (err) {
+          if (deleteFromApp) {
+            return
+          }
+          metadata = {
+            tableId: InternalTables.USER_METADATA,
+          }
+        }
+
+        if (deleteFromApp) {
+          await db.remove(metadata)
+          return
+        }
+
+        // assign the roleId for the metadata doc
+        if (roleId) {
+          metadata.roleId = roleId
+        }
+        let combined = sdk.users.combineMetadataAndUser(user, metadata)
+        // if its null then there was no updates required
+        if (combined) {
+          await db.put(combined)
+        }
+      })
+    }
+  }
+  ctx.body = {
+    message: "User synced.",
+  }
+}
+
+export async function fetchMetadata(ctx: BBContext) {
   const global = await getGlobalUsers()
   const metadata = await sdk.users.rawUserMetadata()
   const users = []
@@ -25,7 +111,7 @@ export async function fetchMetadata(ctx: UserCtx) {
   ctx.body = users
 }
 
-export async function updateSelfMetadata(ctx: UserCtx) {
+export async function updateSelfMetadata(ctx: BBContext) {
   // overwrite the ID with current users
   ctx.request.body._id = ctx.user?._id
   // make sure no stale rev
@@ -35,7 +121,7 @@ export async function updateSelfMetadata(ctx: UserCtx) {
   await updateMetadata(ctx)
 }
 
-export async function updateMetadata(ctx: UserCtx) {
+export async function updateMetadata(ctx: BBContext) {
   const db = context.getAppDB()
   const user = ctx.request.body
   // this isn't applicable to the user
@@ -47,7 +133,7 @@ export async function updateMetadata(ctx: UserCtx) {
   ctx.body = await db.put(metadata)
 }
 
-export async function destroyMetadata(ctx: UserCtx) {
+export async function destroyMetadata(ctx: BBContext) {
   const db = context.getAppDB()
   try {
     const dbUser = await db.get(ctx.params.id)
@@ -60,11 +146,11 @@ export async function destroyMetadata(ctx: UserCtx) {
   }
 }
 
-export async function findMetadata(ctx: UserCtx) {
+export async function findMetadata(ctx: BBContext) {
   ctx.body = await getFullUser(ctx, ctx.params.id)
 }
 
-export async function setFlag(ctx: UserCtx) {
+export async function setFlag(ctx: BBContext) {
   const userId = ctx.user?._id
   const { flag, value } = ctx.request.body
   if (!flag) {
@@ -83,7 +169,7 @@ export async function setFlag(ctx: UserCtx) {
   ctx.body = { message: "Flag set successfully" }
 }
 
-export async function getFlags(ctx: UserCtx) {
+export async function getFlags(ctx: BBContext) {
   const userId = ctx.user?._id
   const docId = generateUserFlagID(userId!)
   const db = context.getAppDB()

package/src/api/routes/index.ts

@@ -25,7 +25,6 @@ import devRoutes from "./dev"
 import cloudRoutes from "./cloud"
 import migrationRoutes from "./migrations"
 import pluginRoutes from "./plugin"
-import opsRoutes from "./ops"
 import Router from "@koa/router"
 import { api as pro } from "@budibase/pro"
 
@@ -64,7 +63,6 @@ export const mainRoutes: Router[] = [
   rowRoutes,
   migrationRoutes,
   pluginRoutes,
-  opsRoutes,
   scheduleRoutes,
   environmentVariableRoutes,
   // these need to be handled last as they still use /api/:tableId

package/src/api/routes/tests/automation.spec.js

@@ -7,7 +7,7 @@ const {
 const setup = require("./utilities")
 const { basicAutomation, newAutomation, automationTrigger, automationStep } = setup.structures
 const MAX_RETRIES = 4
-const { TRIGGER_DEFINITIONS, BUILTIN_ACTION_DEFINITIONS } = require("../../../automations")
+const { TRIGGER_DEFINITIONS, ACTION_DEFINITIONS } = require("../../../automations")
 const { events } = require("@budibase/backend-core")
 
 
@@ -19,14 +19,11 @@ describe("/automations", () => {
 
   afterAll(setup.afterAll)
 
-  beforeAll(async () => {
+  // For some reason this cannot be a beforeAll or the test "tests the automation successfully" fail
+  beforeEach(async () => {
     await config.init()
   })
 
-  beforeEach(() => {
-    events.automation.deleted.mockClear()
-  })
-
   describe("get definitions", () => {
     it("returns a list of definitions for actions", async () => {
       const res = await request
@@ -55,7 +52,7 @@ describe("/automations", () => {
         .expect('Content-Type', /json/)
         .expect(200)
 
-      let definitionsLength = Object.keys(BUILTIN_ACTION_DEFINITIONS).length
+      let definitionsLength = Object.keys(ACTION_DEFINITIONS).length
       definitionsLength-- // OUTGOING_WEBHOOK is deprecated
 
       expect(Object.keys(res.body.action).length).toBeGreaterThanOrEqual(definitionsLength)

package/src/api/routes/tests/user.spec.js

@@ -57,7 +57,6 @@ describe("/users", () => {
     it("should be able to update the user", async () => {
       const user = await config.createUser({ id: `us_update${utils.newid()}` })
       user.roleId = BUILTIN_ROLE_IDS.BASIC
-      delete user._rev
       const res = await request
         .put(`/api/users/metadata`)
         .set(config.defaultHeaders())
@@ -66,46 +65,6 @@ describe("/users", () => {
         .expect("Content-Type", /json/)
       expect(res.body.ok).toEqual(true)
     })
-
-    it("should be able to update the user multiple times", async () => {
-      const user = await config.createUser()
-      delete user._rev
-
-      const res1 = await request
-        .put(`/api/users/metadata`)
-        .set(config.defaultHeaders())
-        .send({ ...user, roleId: BUILTIN_ROLE_IDS.BASIC })
-        .expect(200)
-        .expect("Content-Type", /json/)
-
-      const res = await request
-        .put(`/api/users/metadata`)
-        .set(config.defaultHeaders())
-        .send({ ...user, _rev: res1.body.rev, roleId: BUILTIN_ROLE_IDS.POWER })
-        .expect(200)
-        .expect("Content-Type", /json/)
-
-      expect(res.body.ok).toEqual(true)
-    })
-
-    it("should require the _rev field for multiple updates", async () => {
-      const user = await config.createUser()
-      delete user._rev
-
-      await request
-        .put(`/api/users/metadata`)
-        .set(config.defaultHeaders())
-        .send({ ...user, roleId: BUILTIN_ROLE_IDS.BASIC })
-        .expect(200)
-        .expect("Content-Type", /json/)
-
-      await request
-        .put(`/api/users/metadata`)
-        .set(config.defaultHeaders())
-        .send({ ...user, roleId: BUILTIN_ROLE_IDS.POWER })
-        .expect(409)
-        .expect("Content-Type", /json/)
-    })
   })
 
   describe("destroy", () => {
@@ -133,7 +92,6 @@ describe("/users", () => {
       expect(res.body.tableId).toBeDefined()
     })
   })
-
   describe("setFlag", () => {
     it("should throw an error if a flag is not provided", async () => {
       await config.createUser()
@@ -143,9 +101,8 @@ describe("/users", () => {
         .send({ value: "test" })
         .expect(400)
         .expect("Content-Type", /json/)
-      expect(res.body.message).toEqual(
-        "Must supply a 'flag' field in request body."
-      )
+      expect(res.body.message).toEqual("Must supply a 'flag' field in request body.")
+
     })
 
     it("should be able to set a flag on the user", async () => {
@@ -189,9 +146,8 @@ describe("/users", () => {
         .send({ value: "test" })
         .expect(400)
         .expect("Content-Type", /json/)
-      expect(res.body.message).toEqual(
-        "Must supply a 'flag' field in request body."
-      )
+      expect(res.body.message).toEqual("Must supply a 'flag' field in request body.")
+
     })
 
     it("should be able to set a flag on the user", async () => {
@@ -205,4 +161,37 @@ describe("/users", () => {
       expect(res.body.message).toEqual("Flag set successfully")
     })
   })
+
+  describe("syncUser", () => {
+    it("should sync the user", async () => {
+      let user = await config.createUser()
+      await config.createApp('New App')
+      let res = await request
+        .post(`/api/users/metadata/sync/${user._id}`)
+        .set(config.defaultHeaders())
+        .expect(200)
+        .expect("Content-Type", /json/)
+      expect(res.body.message).toEqual('User synced.')
+    })
+
+
+    it("should sync the user when a previous user is specified", async () => {
+      const app1 = await config.createApp('App 1')
+      const app2 = await config.createApp('App 2')
+
+      let user = await config.createUser({
+        builder: false,
+        admin: true,
+      roles: { [app1.appId]: 'ADMIN' }
+    })
+      let res = await request
+        .post(`/api/users/metadata/sync/${user._id}`)
+        .set(config.defaultHeaders())
+        .send({ previousUser: { ...user, roles: { ...user.roles, [app2.appId]: 'BASIC' } } })
+        .expect(200)
+        .expect("Content-Type", /json/)
+
+      expect(res.body.message).toEqual('User synced.')
+    })
+  })
 })

package/src/api/routes/user.ts

@@ -32,6 +32,11 @@ router
     authorized(PermissionType.USER, PermissionLevel.WRITE),
     controller.destroyMetadata
   )
+  .post(
+    "/api/users/metadata/sync/:id",
+    authorized(PermissionType.USER, PermissionLevel.WRITE),
+    controller.syncUser
+  )
   .post(
     "/api/users/flags",
     authorized(PermissionType.USER, PermissionLevel.WRITE),

package/src/app.ts

@@ -2,9 +2,21 @@ if (process.env.DD_APM_ENABLED) {
   require("./ddApm")
 }
 
+if (process.env.ELASTIC_APM_ENABLED) {
+  require("./elasticApm")
+}
+
 // need to load environment first
 import env from "./environment"
 
+// enable APM if configured
+if (process.env.ELASTIC_APM_ENABLED) {
+  const apm = require("elastic-apm-node").start({
+    serviceName: process.env.SERVICE,
+    environment: process.env.BUDIBASE_ENVIRONMENT,
+  })
+}
+
 import { ExtendableContext } from "koa"
 import * as db from "./db"
 db.init()
@@ -15,8 +27,8 @@ import * as api from "./api"
 import * as automations from "./automations"
 import { Thread } from "./threads"
 import * as redis from "./utilities/redis"
-import { initialise as initialiseWebsockets } from "./websockets"
 import { events, logging, middleware, timers } from "@budibase/backend-core"
+import { initialise as initialiseWebsockets } from "./websocket"
 import { startup } from "./startup"
 const Sentry = require("@sentry/node")
 const destroyable = require("server-destroy")
@@ -41,8 +53,7 @@ app.use(
   })
 )
 
-app.use(middleware.correlation)
-app.use(middleware.pino)
+app.use(middleware.logging)
 app.use(userAgent)
 
 if (env.isProd()) {
@@ -61,7 +72,7 @@ if (env.isProd()) {
 
 const server = http.createServer(app.callback())
 destroyable(server)
-initialiseWebsockets(app, server)
+initialiseWebsockets(server)
 
 let shuttingDown = false,
   errCode = 0

package/src/automations/actions.ts

@@ -15,14 +15,7 @@ import * as delay from "./steps/delay"
 import * as queryRow from "./steps/queryRows"
 import * as loop from "./steps/loop"
 import env from "../environment"
-import {
-  AutomationStepSchema,
-  AutomationStepInput,
-  PluginType,
-  AutomationStep,
-} from "@budibase/types"
-import sdk from "../sdk"
-import { getAutomationPlugin } from "../utilities/fileSystem"
+import { AutomationStepSchema, AutomationStepInput } from "@budibase/types"
 
 const ACTION_IMPLS: Record<
   string,
@@ -45,26 +38,25 @@ const ACTION_IMPLS: Record<
   zapier: zapier.run,
   integromat: integromat.run,
 }
-export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
-  {
-    SEND_EMAIL_SMTP: sendSmtpEmail.definition,
-    CREATE_ROW: createRow.definition,
-    UPDATE_ROW: updateRow.definition,
-    DELETE_ROW: deleteRow.definition,
-    OUTGOING_WEBHOOK: outgoingWebhook.definition,
-    EXECUTE_SCRIPT: executeScript.definition,
-    EXECUTE_QUERY: executeQuery.definition,
-    SERVER_LOG: serverLog.definition,
-    DELAY: delay.definition,
-    FILTER: filter.definition,
-    QUERY_ROWS: queryRow.definition,
-    LOOP: loop.definition,
-    // these used to be lowercase step IDs, maintain for backwards compat
-    discord: discord.definition,
-    slack: slack.definition,
-    zapier: zapier.definition,
-    integromat: integromat.definition,
-  }
+export const ACTION_DEFINITIONS: Record<string, AutomationStepSchema> = {
+  SEND_EMAIL_SMTP: sendSmtpEmail.definition,
+  CREATE_ROW: createRow.definition,
+  UPDATE_ROW: updateRow.definition,
+  DELETE_ROW: deleteRow.definition,
+  OUTGOING_WEBHOOK: outgoingWebhook.definition,
+  EXECUTE_SCRIPT: executeScript.definition,
+  EXECUTE_QUERY: executeQuery.definition,
+  SERVER_LOG: serverLog.definition,
+  DELAY: delay.definition,
+  FILTER: filter.definition,
+  QUERY_ROWS: queryRow.definition,
+  LOOP: loop.definition,
+  // these used to be lowercase step IDs, maintain for backwards compat
+  discord: discord.definition,
+  slack: slack.definition,
+  zapier: zapier.definition,
+  integromat: integromat.definition,
+}
 
 // don't add the bash script/definitions unless in self host
 // the fact this isn't included in any definitions means it cannot be
@@ -74,36 +66,12 @@ if (env.SELF_HOSTED) {
   // @ts-ignore
   ACTION_IMPLS["EXECUTE_BASH"] = bash.run
   // @ts-ignore
-  BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
-}
-
-export async function getActionDefinitions() {
-  const actionDefinitions = BUILTIN_ACTION_DEFINITIONS
-  if (env.SELF_HOSTED) {
-    const plugins = await sdk.plugins.fetch(PluginType.AUTOMATION)
-    for (let plugin of plugins) {
-      const schema = plugin.schema.schema as AutomationStep
-      actionDefinitions[schema.stepId] = {
-        ...schema,
-        custom: true,
-      }
-    }
-  }
-  return actionDefinitions
+  ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
 }
 
 /* istanbul ignore next */
-export async function getAction(stepId: string) {
-  if (ACTION_IMPLS[stepId] != null) {
-    return ACTION_IMPLS[stepId]
-  }
-  // must be a plugin
-  if (env.SELF_HOSTED) {
-    const plugins = await sdk.plugins.fetch(PluginType.AUTOMATION)
-    const found = plugins.find(plugin => plugin.schema.schema.stepId === stepId)
-    if (!found) {
-      throw new Error(`Unable to find action implementation for "${stepId}"`)
-    }
-    return (await getAutomationPlugin(found)).action
+export async function getAction(actionName: string) {
+  if (ACTION_IMPLS[actionName] != null) {
+    return ACTION_IMPLS[actionName]
   }
 }

package/src/automations/index.ts

@@ -6,7 +6,7 @@ import BullQueue from "bull"
 export { automationQueue } from "./bullboard"
 export { shutdown } from "./bullboard"
 export { TRIGGER_DEFINITIONS } from "./triggers"
-export { BUILTIN_ACTION_DEFINITIONS, getActionDefinitions } from "./actions"
+export { ACTION_DEFINITIONS } from "./actions"
 
 /**
  * This module is built purely to kick off the worker farm and manage the inputs/outputs

package/src/automations/steps/bash.ts

@@ -4,11 +4,8 @@ import * as automationUtils from "../automationUtils"
 import environment from "../../environment"
 import {
   AutomationActionStepId,
-  AutomationCustomIOType,
-  AutomationIOType,
-  AutomationStepInput,
   AutomationStepSchema,
-  AutomationStepType,
+  AutomationStepInput,
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
@@ -16,7 +13,7 @@ export const definition: AutomationStepSchema = {
   tagline: "Execute a bash command",
   icon: "JourneyEvent",
   description: "Run a bash script",
-  type: AutomationStepType.ACTION,
+  type: "ACTION",
   internal: true,
   stepId: AutomationActionStepId.EXECUTE_BASH,
   inputs: {},
@@ -24,8 +21,8 @@ export const definition: AutomationStepSchema = {
     inputs: {
       properties: {
         code: {
-          type: AutomationIOType.STRING,
-          customType: AutomationCustomIOType.CODE,
+          type: "string",
+          customType: "code",
           title: "Code",
         },
       },
@@ -34,16 +31,16 @@ export const definition: AutomationStepSchema = {
     outputs: {
       properties: {
         stdout: {
-          type: AutomationIOType.STRING,
+          type: "string",
           description: "Standard output of your bash command or script",
         },
         success: {
-          type: AutomationIOType.BOOLEAN,
+          type: "boolean",
           description: "Whether the command was successful",
         },
       },
-      required: ["stdout"],
     },
+    required: ["stdout"],
   },
 }