@budibase/server 2.5.6-alpha.1 → 2.5.6-alpha.10

package/src/sdk/app/applications/tests/sync.spec.ts

@@ -0,0 +1,137 @@
+import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
+import { events, context, roles, constants } from "@budibase/backend-core"
+import { init } from "../../../../events"
+import { rawUserMetadata } from "../../../users/utils"
+import EventEmitter from "events"
+import { UserGroup, UserMetadata, UserRoles, User } from "@budibase/types"
+
+const config = new TestConfiguration()
+let app, group: UserGroup, groupUser: User
+const ROLE_ID = roles.BUILTIN_ROLE_IDS.BASIC
+
+const emitter = new EventEmitter()
+
+function updateCb(docId: string) {
+  const isGroup = docId.startsWith(constants.DocumentType.GROUP)
+  if (isGroup) {
+    emitter.emit("update-group")
+  } else {
+    emitter.emit("update-user")
+  }
+}
+
+init(updateCb)
+
+function waitForUpdate(opts: { group?: boolean }) {
+  return new Promise<void>((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      reject()
+    }, 5000)
+    const event = opts?.group ? "update-group" : "update-user"
+    emitter.on(event, () => {
+      clearTimeout(timeout)
+      resolve()
+    })
+  })
+}
+
+beforeAll(async () => {
+  app = await config.init("syncApp")
+})
+
+async function createUser(email: string, roles: UserRoles, builder?: boolean) {
+  const user = await config.createUser({
+    email,
+    roles,
+    builder: builder || false,
+    admin: false,
+  })
+  await context.doInContext(config.appId!, async () => {
+    await events.user.created(user)
+  })
+  return user
+}
+
+async function removeUserRole(user: User) {
+  const final = await config.globalUser({
+    ...user,
+    id: user._id,
+    roles: {},
+    builder: false,
+    admin: false,
+  })
+  await context.doInContext(config.appId!, async () => {
+    await events.user.updated(final)
+  })
+}
+
+async function createGroupAndUser(email: string) {
+  groupUser = await config.createUser({
+    email,
+    roles: {},
+    builder: false,
+    admin: false,
+  })
+  group = await config.createGroup()
+  await config.addUserToGroup(group._id!, groupUser._id!)
+}
+
+async function removeUserFromGroup() {
+  await config.removeUserFromGroup(group._id!, groupUser._id!)
+  return context.doInContext(config.appId!, async () => {
+    await events.user.updated(groupUser)
+  })
+}
+
+async function getUserMetadata(): Promise<UserMetadata[]> {
+  return context.doInContext(config.appId!, async () => {
+    return await rawUserMetadata()
+  })
+}
+
+function buildRoles() {
+  return { [config.prodAppId!]: ROLE_ID }
+}
+
+describe("app user/group sync", () => {
+  const groupEmail = "test2@test.com",
+    normalEmail = "test@test.com"
+  async function checkEmail(
+    email: string,
+    opts?: { group?: boolean; notFound?: boolean }
+  ) {
+    await waitForUpdate(opts || {})
+    const metadata = await getUserMetadata()
+    const found = metadata.find(data => data.email === email)
+    if (opts?.notFound) {
+      expect(found).toBeUndefined()
+    } else {
+      expect(found).toBeDefined()
+    }
+  }
+
+  it("should be able to sync a new user, add then remove", async () => {
+    const user = await createUser(normalEmail, buildRoles())
+    await checkEmail(normalEmail)
+    await removeUserRole(user)
+    await checkEmail(normalEmail, { notFound: true })
+  })
+
+  it("should be able to sync a group", async () => {
+    await createGroupAndUser(groupEmail)
+    await checkEmail(groupEmail, { group: true })
+  })
+
+  it("should be able to remove user from group", async () => {
+    if (!group) {
+      await createGroupAndUser(groupEmail)
+    }
+    await removeUserFromGroup()
+    await checkEmail(groupEmail, { notFound: true })
+  })
+
+  it("should be able to handle builder users", async () => {
+    await createUser("test3@test.com", {}, true)
+    await checkEmail("test3@test.com")
+  })
+})

package/src/sdk/index.ts

@@ -6,6 +6,7 @@ import { default as datasources } from "./app/datasources"
 import { default as queries } from "./app/queries"
 import { default as rows } from "./app/rows"
 import { default as users } from "./users"
+import { default as plugins } from "./plugins"
 
 const sdk = {
   backups,
@@ -16,6 +17,7 @@ const sdk = {
   users,
   datasources,
   queries,
+  plugins,
 }
 
 // default export for TS

package/src/sdk/plugins/index.ts

@@ -0,0 +1,5 @@
+import * as plugins from "./plugins"
+
+export default {
+  ...plugins,
+}

package/src/sdk/plugins/plugins.ts

@@ -0,0 +1,41 @@
+import { FileType, Plugin, PluginSource, PluginType } from "@budibase/types"
+import {
+  db as dbCore,
+  objectStore,
+  plugins as pluginCore,
+  tenancy,
+} from "@budibase/backend-core"
+import { fileUpload } from "../../api/controllers/plugin/file"
+import env from "../../environment"
+import { clientAppSocket } from "../../websockets"
+import { sdk as pro } from "@budibase/pro"
+
+export async function fetch(type?: PluginType) {
+  const db = tenancy.getGlobalDB()
+  const response = await db.allDocs(
+    dbCore.getPluginParams(null, {
+      include_docs: true,
+    })
+  )
+  let plugins = response.rows.map((row: any) => row.doc) as Plugin[]
+  plugins = objectStore.enrichPluginURLs(plugins)
+  if (type) {
+    return plugins.filter((plugin: Plugin) => plugin.schema?.type === type)
+  } else {
+    return plugins
+  }
+}
+
+export async function processUploaded(plugin: FileType, source?: PluginSource) {
+  const { metadata, directory } = await fileUpload(plugin)
+  pluginCore.validate(metadata?.schema)
+
+  // Only allow components in cloud
+  if (!env.SELF_HOSTED && metadata?.schema?.type !== PluginType.COMPONENT) {
+    throw new Error("Only component plugins are supported outside of self-host")
+  }
+
+  const doc = await pro.plugins.storePlugin(metadata, directory, source)
+  clientAppSocket.emit("plugin-update", { name: doc.name, hash: doc.hash })
+  return doc
+}

package/src/sdk/users/tests/utils.spec.ts

@@ -121,38 +121,7 @@ describe("syncGlobalUsers", () => {
         await syncGlobalUsers()
 
         const metadata = await rawUserMetadata()
-        expect(metadata).toHaveLength(1)
-      })
-    })
-  })
-
-  it("app users are removed when app is removed from user group", async () => {
-    await config.doInTenant(async () => {
-      const group = await proSdk.groups.save(structures.userGroups.userGroup())
-      const user1 = await config.createUser({ admin: false, builder: false })
-      const user2 = await config.createUser({ admin: false, builder: false })
-      await proSdk.groups.updateGroupApps(group.id, {
-        appsToAdd: [
-          { appId: config.prodAppId!, roleId: roles.BUILTIN_ROLE_IDS.BASIC },
-        ],
-      })
-      await proSdk.groups.addUsers(group.id, [user1._id, user2._id])
-
-      await config.doInContext(config.appId, async () => {
-        await syncGlobalUsers()
-        expect(await rawUserMetadata()).toHaveLength(3)
-
-        await proSdk.groups.removeUsers(group.id, [user1._id])
-        await syncGlobalUsers()
-
-        const metadata = await rawUserMetadata()
-        expect(metadata).toHaveLength(2)
-
-        expect(metadata).not.toContainEqual(
-          expect.objectContaining({
-            _id: db.generateUserMetadataID(user1._id),
-          })
-        )
+        expect(metadata).toHaveLength(0)
       })
     })
   })

package/src/sdk/users/utils.ts

@@ -1,12 +1,13 @@
 import { getGlobalUsers } from "../../utilities/global"
 import { context, roles as rolesCore } from "@budibase/backend-core"
 import {
+  getGlobalIDFromUserMetadataID,
   generateUserMetadataID,
   getUserMetadataParams,
   InternalTables,
 } from "../../db/utils"
 import { isEqual } from "lodash"
-import { ContextUser, UserMetadata } from "@budibase/types"
+import { ContextUser, UserMetadata, User } from "@budibase/types"
 
 export function combineMetadataAndUser(
   user: ContextUser,
@@ -37,6 +38,10 @@ export function combineMetadataAndUser(
   if (found) {
     newDoc._rev = found._rev
   }
+  // clear fields that shouldn't be in metadata
+  delete newDoc.password
+  delete newDoc.forceResetPassword
+  delete newDoc.roles
   if (found == null || !isEqual(newDoc, found)) {
     return {
       ...found,
@@ -60,10 +65,9 @@ export async function rawUserMetadata() {
 export async function syncGlobalUsers() {
   // sync user metadata
   const db = context.getAppDB()
-  const [users, metadata] = await Promise.all([
-    getGlobalUsers(),
-    rawUserMetadata(),
-  ])
+  const resp = await Promise.all([getGlobalUsers(), rawUserMetadata()])
+  const users = resp[0] as User[]
+  const metadata = resp[1] as UserMetadata[]
   const toWrite = []
   for (let user of users) {
     const combined = combineMetadataAndUser(user, metadata)
@@ -71,5 +75,19 @@ export async function syncGlobalUsers() {
       toWrite.push(combined)
     }
   }
+  let foundEmails: string[] = []
+  for (let data of metadata) {
+    if (!data._id) {
+      continue
+    }
+    const alreadyExisting = data.email && foundEmails.indexOf(data.email) !== -1
+    const globalId = getGlobalIDFromUserMetadataID(data._id)
+    if (!users.find(user => user._id === globalId) || alreadyExisting) {
+      toWrite.push({ ...data, _deleted: true })
+    }
+    if (data.email) {
+      foundEmails.push(data.email)
+    }
+  }
   await db.bulkDocs(toWrite)
 }

package/src/startup.ts

@@ -10,7 +10,7 @@ import fs from "fs"
 import { watch } from "./watch"
 import * as automations from "./automations"
 import * as fileSystem from "./utilities/fileSystem"
-import eventEmitter from "./events"
+import { default as eventEmitter, init as eventInit } from "./events"
 import * as migrations from "./migrations"
 import * as bullboard from "./automations/bullboard"
 import * as pro from "@budibase/pro"
@@ -63,6 +63,7 @@ export async function startup(app?: any, server?: any) {
   eventEmitter.emitPort(env.PORT)
   fileSystem.init()
   await redis.init()
+  eventInit()
 
   // run migrations on startup if not done via http
   // not recommended in a clustered environment

package/src/tests/utilities/TestConfiguration.ts

@@ -49,6 +49,7 @@ import {
   SearchFilters,
   UserRoles,
 } from "@budibase/types"
+import { BUILTIN_ROLE_IDS } from "@budibase/backend-core/src/security/roles"
 
 type DefaultUserValues = {
   globalUserId: string
@@ -306,6 +307,33 @@ class TestConfiguration {
     }
   }
 
+  async createGroup(roleId: string = BUILTIN_ROLE_IDS.BASIC) {
+    return context.doInTenant(this.tenantId!, async () => {
+      const baseGroup = structures.userGroups.userGroup()
+      baseGroup.roles = {
+        [this.prodAppId]: roleId,
+      }
+      const { id, rev } = await pro.sdk.groups.save(baseGroup)
+      return {
+        _id: id,
+        _rev: rev,
+        ...baseGroup,
+      }
+    })
+  }
+
+  async addUserToGroup(groupId: string, userId: string) {
+    return context.doInTenant(this.tenantId!, async () => {
+      await pro.sdk.groups.addUsers(groupId, [userId])
+    })
+  }
+
+  async removeUserFromGroup(groupId: string, userId: string) {
+    return context.doInTenant(this.tenantId!, async () => {
+      await pro.sdk.groups.removeUsers(groupId, [userId])
+    })
+  }
+
   async login({ roleId, userId, builder, prodApp = false }: any = {}) {
     const appId = prodApp ? this.prodAppId : this.appId
     return context.doInAppContext(appId, async () => {

package/src/tests/utilities/structures.ts

@@ -1,18 +1,22 @@
-import { permissions, roles } from "@budibase/backend-core"
+import { permissions, roles, utils } from "@budibase/backend-core"
 import { createHomeScreen } from "../../constants/screens"
 import { EMPTY_LAYOUT } from "../../constants/layouts"
 import { cloneDeep } from "lodash/fp"
-import { ACTION_DEFINITIONS, TRIGGER_DEFINITIONS } from "../../automations"
+import {
+  BUILTIN_ACTION_DEFINITIONS,
+  TRIGGER_DEFINITIONS,
+} from "../../automations"
 import {
   Automation,
   AutomationActionStepId,
+  AutomationStep,
+  AutomationStepType,
+  AutomationTrigger,
   AutomationTriggerStepId,
   Datasource,
   SourceName,
 } from "@budibase/types"
 
-const { v4: uuidv4 } = require("uuid")
-
 export function basicTable() {
   return {
     name: "TestTable",
@@ -71,19 +75,19 @@ export function view(tableId: string) {
 }
 
 export function automationStep(
-  actionDefinition = ACTION_DEFINITIONS.CREATE_ROW
-) {
+  actionDefinition = BUILTIN_ACTION_DEFINITIONS.CREATE_ROW
+): AutomationStep {
   return {
-    id: uuidv4(),
+    id: utils.newid(),
     ...actionDefinition,
   }
 }
 
 export function automationTrigger(
   triggerDefinition = TRIGGER_DEFINITIONS.ROW_SAVED
-) {
+): AutomationTrigger {
   return {
-    id: uuidv4(),
+    id: utils.newid(),
     ...triggerDefinition,
   }
 }
@@ -106,7 +110,7 @@ export function newAutomation({ steps, trigger }: any = {}) {
   return automation
 }
 
-export function basicAutomation(appId?: string) {
+export function basicAutomation(appId?: string): Automation {
   return {
     name: "My Automation",
     screenId: "kasdkfldsafkl",
@@ -119,18 +123,22 @@ export function basicAutomation(appId?: string) {
         tagline: "test",
         icon: "test",
         description: "test",
-        type: "trigger",
+        type: AutomationStepType.TRIGGER,
         id: "test",
         inputs: {},
         schema: {
-          inputs: {},
-          outputs: {},
+          inputs: {
+            properties: {},
+          },
+          outputs: {
+            properties: {},
+          },
         },
       },
       steps: [],
     },
     type: "automation",
-    appId,
+    appId: appId!,
   }
 }
 
@@ -154,7 +162,7 @@ export function loopAutomation(tableId: string, loopOpts?: any): Automation {
           inputs: {
             tableId,
           },
-          schema: ACTION_DEFINITIONS.QUERY_ROWS.schema,
+          schema: BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS.schema,
         },
         {
           id: "c",
@@ -163,7 +171,7 @@ export function loopAutomation(tableId: string, loopOpts?: any): Automation {
           internal: true,
           inputs: loopOpts,
           blockToLoop: "d",
-          schema: ACTION_DEFINITIONS.LOOP.schema,
+          schema: BUILTIN_ACTION_DEFINITIONS.LOOP.schema,
         },
         {
           id: "d",
@@ -173,7 +181,7 @@ export function loopAutomation(tableId: string, loopOpts?: any): Automation {
           inputs: {
             text: "log statement",
           },
-          schema: ACTION_DEFINITIONS.SERVER_LOG.schema,
+          schema: BUILTIN_ACTION_DEFINITIONS.SERVER_LOG.schema,
         },
       ],
       trigger: {

package/src/threads/automation.ts

@@ -27,8 +27,8 @@ import { processObject } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
 import * as sdkUtils from "../sdk/utils"
 import env from "../environment"
-const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
-const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
+const FILTER_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.FILTER.stepId
+const LOOP_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.LOOP.stepId
 const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED }
 

package/src/utilities/fileSystem/plugin.ts

@@ -5,6 +5,7 @@ import { join } from "path"
 import { objectStore } from "@budibase/backend-core"
 
 const DATASOURCE_PATH = join(budibaseTempDir(), "datasource")
+const AUTOMATION_PATH = join(budibaseTempDir(), "automation")
 
 export const getPluginMetadata = async (path: string) => {
   let metadata: any = {}
@@ -33,12 +34,12 @@ export const getPluginMetadata = async (path: string) => {
   return { metadata, directory: path }
 }
 
-export const getDatasourcePlugin = async (plugin: Plugin) => {
+async function getPluginImpl(path: string, plugin: Plugin) {
   const hash = plugin.schema?.hash
-  if (!fs.existsSync(DATASOURCE_PATH)) {
-    fs.mkdirSync(DATASOURCE_PATH)
+  if (!fs.existsSync(path)) {
+    fs.mkdirSync(path)
   }
-  const filename = join(DATASOURCE_PATH, plugin.name)
+  const filename = join(path, plugin.name)
   const metadataName = `${filename}.bbmetadata`
   if (fs.existsSync(filename)) {
     const currentHash = fs.readFileSync(metadataName, "utf8")
@@ -62,3 +63,11 @@ export const getDatasourcePlugin = async (plugin: Plugin) => {
 
   return require(filename)
 }
+
+export const getDatasourcePlugin = async (plugin: Plugin) => {
+  return getPluginImpl(DATASOURCE_PATH, plugin)
+}
+
+export const getAutomationPlugin = async (plugin: Plugin) => {
+  return getPluginImpl(AUTOMATION_PATH, plugin)
+}

package/src/utilities/global.ts

@@ -9,6 +9,7 @@ import {
 import env from "../environment"
 import { groups } from "@budibase/pro"
 import { UserCtx, ContextUser, User, UserGroup } from "@budibase/types"
+import { global } from "yargs"
 
 export function updateAppRole(
   user: ContextUser,
@@ -16,7 +17,7 @@ export function updateAppRole(
 ) {
   appId = appId || context.getAppId()
 
-  if (!user || !user.roles) {
+  if (!user || (!user.roles && !user.userGroups)) {
     return user
   }
   // if in an multi-tenancy environment make sure roles are never updated
@@ -27,7 +28,7 @@ export function updateAppRole(
     return user
   }
   // always use the deployed app
-  if (appId) {
+  if (appId && user.roles) {
     user.roleId = user.roles[dbCore.getProdAppID(appId)]
   }
   // if a role wasn't found then either set as admin (builder) or public (everyone else)
@@ -60,7 +61,7 @@ async function checkGroupRoles(
   return user
 }
 
-async function processUser(
+export async function processUser(
   user: ContextUser,
   opts: { appId?: string; groups?: UserGroup[] } = {}
 ) {
@@ -94,16 +95,15 @@ export async function getGlobalUser(userId: string) {
   return processUser(user, { appId })
 }
 
-export async function getGlobalUsers(users?: ContextUser[]) {
+export async function getGlobalUsers(
+  userIds?: string[],
+  opts?: { noProcessing?: boolean }
+) {
   const appId = context.getAppId()
   const db = tenancy.getGlobalDB()
-  const allGroups = await groups.fetch()
   let globalUsers
-  if (users) {
-    const globalIds = users.map(user =>
-      getGlobalIDFromUserMetadataID(user._id!)
-    )
-    globalUsers = (await db.allDocs(getMultiIDParams(globalIds))).rows.map(
+  if (userIds) {
+    globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map(
       row => row.doc
     )
   } else {
@@ -126,15 +126,20 @@ export async function getGlobalUsers(users?: ContextUser[]) {
     return globalUsers
   }
 
-  // pass in the groups, meaning we don't actually need to retrieve them for
-  // each user individually
-  return Promise.all(
-    globalUsers.map(user => processUser(user, { groups: allGroups }))
-  )
+  if (opts?.noProcessing) {
+    return globalUsers
+  } else {
+    // pass in the groups, meaning we don't actually need to retrieve them for
+    // each user individually
+    const allGroups = await groups.fetch()
+    return Promise.all(
+      globalUsers.map(user => processUser(user, { groups: allGroups }))
+    )
+  }
 }
 
 export async function getGlobalUsersFromMetadata(users: ContextUser[]) {
-  const globalUsers = await getGlobalUsers(users)
+  const globalUsers = await getGlobalUsers(users.map(user => user._id!))
   return users.map(user => {
     const globalUser = globalUsers.find(
       globalUser => globalUser && user._id?.includes(globalUser._id)

package/src/watch.ts

@@ -3,7 +3,7 @@ import env from "./environment"
 import chokidar from "chokidar"
 import fs from "fs"
 import { constants, tenancy } from "@budibase/backend-core"
-import { processUploadedPlugin } from "./api/controllers/plugin"
+import pluginsSdk from "./sdk/plugins"
 
 export function watch() {
   const watchPath = path.join(env.PLUGINS_DIR, "./**/*.tar.gz")
@@ -27,7 +27,7 @@ export function watch() {
           const split = path.split("/")
           const name = split[split.length - 1]
           console.log("Importing plugin:", path)
-          await processUploadedPlugin({ name, path })
+          await pluginsSdk.processUploaded({ name, path })
         } catch (err: any) {
           const message = err?.message ? err?.message : err
           console.error("Failed to import plugin:", message)

package/src/websockets/client.ts

@@ -0,0 +1,11 @@
+import Socket from "./websocket"
+import authorized from "../middleware/authorized"
+import http from "http"
+import Koa from "koa"
+import { permissions } from "@budibase/backend-core"
+
+export default class ClientAppWebsocket extends Socket {
+  constructor(app: Koa, server: http.Server) {
+    super(app, server, "/socket/client", [authorized(permissions.BUILDER)])
+  }
+}