@budibase/worker 2.5.9 → 2.5.10-alpha.1

package/src/api/routes/global/tests/users.spec.ts

@@ -48,7 +48,7 @@ describe("/api/global/users", () => {
         400
       )
 
-      expect(res.body.message).toBe("Unavailable")
+      expect(res.body.message).toBe(`Unavailable`)
       expect(sendMailMock).toHaveBeenCalledTimes(0)
       expect(code).toBeUndefined()
       expect(events.user.invited).toBeCalledTimes(0)
@@ -225,7 +225,9 @@ describe("/api/global/users", () => {
 
       const response = await config.api.users.saveUser(user, 400)
 
-      expect(response.body.message).toBe(`Unavailable`)
+      expect(response.body.message).toBe(
+        `Email already in use: '${user.email}'`
+      )
       expect(events.user.created).toBeCalledTimes(0)
     })
 
@@ -237,7 +239,9 @@ describe("/api/global/users", () => {
         delete user._id
         const response = await config.api.users.saveUser(user, 400)
 
-        expect(response.body.message).toBe(`Unavailable`)
+        expect(response.body.message).toBe(
+          `Email already in use: '${user.email}'`
+        )
         expect(events.user.created).toBeCalledTimes(0)
       })
     })
@@ -249,7 +253,9 @@ describe("/api/global/users", () => {
 
       const response = await config.api.users.saveUser(user, 400)
 
-      expect(response.body.message).toBe(`Unavailable`)
+      expect(response.body.message).toBe(
+        `Email already in use: '${user.email}'`
+      )
       expect(events.user.created).toBeCalledTimes(0)
     })
 

package/src/api/routes/index.ts

@@ -1,5 +1,5 @@
 import Router from "@koa/router"
-import { api } from "@budibase/pro"
+import { api as pro } from "@budibase/pro"
 import userRoutes from "./global/users"
 import configRoutes from "./global/configs"
 import workspaceRoutes from "./global/workspaces"
@@ -17,9 +17,6 @@ import migrationRoutes from "./system/migrations"
 import accountRoutes from "./system/accounts"
 import restoreRoutes from "./system/restore"
 
-let userGroupRoutes = api.groups
-let auditLogRoutes = api.auditLogs
-
 export const routes: Router[] = [
   configRoutes,
   userRoutes,
@@ -33,10 +30,11 @@ export const routes: Router[] = [
   statusRoutes,
   selfRoutes,
   licenseRoutes,
-  userGroupRoutes,
-  auditLogRoutes,
+  pro.groups,
+  pro.auditLogs,
   migrationRoutes,
   accountRoutes,
   restoreRoutes,
   eventRoutes,
+  pro.scim,
 ]

package/src/constants/templates/core.hbs

@@ -16,7 +16,8 @@
               cellspacing="0"
             >
               <img
-                width="32px"
+                width="32"
+                height="32"
                 style="margin-right:16px; vertical-align: middle;"
                 alt="Budibase Logo"
                 src="https://i.imgur.com/Xhdt1YP.png"

package/src/environment.ts

@@ -47,7 +47,6 @@ const environment = {
   // flags
   NODE_ENV: process.env.NODE_ENV,
   SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED || ""),
-  LOG_LEVEL: process.env.LOG_LEVEL,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
   SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED,

package/src/index.ts

@@ -2,10 +2,6 @@ if (process.env.DD_APM_ENABLED) {
   require("./ddApm")
 }
 
-if (process.env.ELASTIC_APM_ENABLED) {
-  require("./elasticApm")
-}
-
 // need to load environment first
 import env from "./environment"
 import { Scope } from "@sentry/node"
@@ -31,10 +27,11 @@ import api from "./api"
 import * as redis from "./utilities/redis"
 const Sentry = require("@sentry/node")
 const koaSession = require("koa-session")
-const logger = require("koa-pino-logger")
 const { userAgent } = require("koa-useragent")
 
 import destroyable from "server-destroy"
+import { initPro } from "./initPro"
+import { handleScimBody } from "./middleware/handleScimBody"
 
 // configure events to use the pro audit log write
 // can't integrate directly into backend-core due to cyclic issues
@@ -54,10 +51,12 @@ const app: Application = new Koa()
 app.keys = ["secret", "key"]
 
 // set up top level koa middleware
+app.use(handleScimBody)
 app.use(koaBody({ multipart: true }))
+
 app.use(koaSession(app))
-app.use(middleware.logging)
-app.use(logger(logging.pinoSettings()))
+app.use(middleware.correlation)
+app.use(middleware.pino)
 app.use(userAgent)
 
 // authentication
@@ -108,6 +107,7 @@ const shutdown = () => {
 
 export default server.listen(parseInt(env.PORT || "4002"), async () => {
   console.log(`Worker running on ${JSON.stringify(server.address())}`)
+  await initPro()
   await redis.init()
 })
 

package/src/initPro.ts

@@ -0,0 +1,13 @@
+import { sdk as proSdk } from "@budibase/pro"
+import * as userSdk from "./sdk/users"
+
+export const initPro = async () => {
+  await proSdk.init({
+    scimUserServiceConfig: {
+      functions: {
+        saveUser: userSdk.save,
+        removeUser: (id: string) => userSdk.destroy(id),
+      },
+    },
+  })
+}

package/src/middleware/handleScimBody.ts

@@ -0,0 +1,12 @@
+import { Ctx } from "@budibase/types"
+
+export const handleScimBody = (ctx: Ctx, next: any) => {
+  var type = ctx.req.headers["content-type"] || ""
+  type = type.split(";")[0]
+
+  if (type === "application/scim+json") {
+    ctx.req.headers["content-type"] = "application/json"
+  }
+
+  return next()
+}

package/src/sdk/accounts/metadata.ts

@@ -18,8 +18,17 @@ export const saveMetadata = async (
     if (existing) {
       metadata._rev = existing._rev
     }
-    const res = await db.put(metadata)
-    metadata._rev = res.rev
+    try {
+      const res = await db.put(metadata)
+      metadata._rev = res.rev
+    } catch (e: any) {
+      // account can be updated frequently
+      // ignore 409
+      if (e.status !== 409) {
+        throw e
+      }
+    }
+
     return metadata
   })
 }

package/src/sdk/users/users.ts

@@ -1,5 +1,4 @@
 import env from "../../environment"
-import * as apps from "../../utilities/appService"
 import * as eventHelpers from "./events"
 import {
   accounts,
@@ -15,11 +14,12 @@ import {
   utils,
   ViewName,
   env as coreEnv,
+  context,
+  EmailUnavailableError,
 } from "@budibase/backend-core"
 import {
   AccountMetadata,
   AllDocsResponse,
-  BulkUserResponse,
   CloudAccount,
   InviteUsersRequest,
   InviteUsersResponse,
@@ -28,9 +28,10 @@ import {
   PlatformUser,
   PlatformUserByEmail,
   RowResponse,
-  SearchUsersRequest,
   User,
   SaveUserOpts,
+  BulkUserCreated,
+  BulkUserDeleted,
   Account,
 } from "@budibase/types"
 import { sendEmail } from "../../utilities/email"
@@ -38,8 +39,6 @@ import { EmailTemplatePurpose } from "../../constants"
 import * as pro from "@budibase/pro"
 import * as accountSdk from "../accounts"
 
-const PAGE_LIMIT = 8
-
 export const allUsers = async () => {
   const db = tenancy.getGlobalDB()
   const response = await db.allDocs(
@@ -69,43 +68,6 @@ export const getUsersByAppAccess = async (appId?: string) => {
   return response
 }
 
-export const paginatedUsers = async ({
-  page,
-  email,
-  appId,
-}: SearchUsersRequest = {}) => {
-  const db = tenancy.getGlobalDB()
-  // get one extra document, to have the next page
-  const opts: any = {
-    include_docs: true,
-    limit: PAGE_LIMIT + 1,
-  }
-  // add a startkey if the page was specified (anchor)
-  if (page) {
-    opts.startkey = page
-  }
-  // property specifies what to use for the page/anchor
-  let userList,
-    property = "_id",
-    getKey
-  if (appId) {
-    userList = await usersCore.searchGlobalUsersByApp(appId, opts)
-    getKey = (doc: any) => usersCore.getGlobalUserByAppPage(appId, doc)
-  } else if (email) {
-    userList = await usersCore.searchGlobalUsersByEmail(email, opts)
-    property = "email"
-  } else {
-    // no search, query allDocs
-    const response = await db.allDocs(dbUtils.getGlobalUserParams(null, opts))
-    userList = response.rows.map((row: any) => row.doc)
-  }
-  return dbUtils.pagination(userList, PAGE_LIMIT, {
-    paginate: true,
-    property,
-    getKey,
-  })
-}
-
 export async function getUserByEmail(email: string) {
   return usersCore.getGlobalUserByEmail(email)
 }
@@ -198,7 +160,7 @@ const validateUniqueUser = async (email: string, tenantId: string) => {
   if (env.MULTI_TENANCY) {
     const tenantUser = await getPlatformUser(email)
     if (tenantUser != null && tenantUser.tenantId !== tenantId) {
-      throw `Unavailable`
+      throw new EmailUnavailableError(email)
     }
   }
 
@@ -206,7 +168,7 @@ const validateUniqueUser = async (email: string, tenantId: string) => {
   if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
     const account = await accounts.getAccount(email)
     if (account && account.verified && account.tenantId !== tenantId) {
-      throw `Unavailable`
+      throw new EmailUnavailableError(email)
     }
   }
 }
@@ -235,6 +197,8 @@ export async function isPreventPasswordActions(user: User, account?: Account) {
   return !!(account && account.email === user.email && isSSOAccount(account))
 }
 
+// TODO: The single save should re-use the bulk insert with a single
+// user so that we don't need to duplicate logic
 export const save = async (
   user: User,
   opts: SaveUserOpts = {}
@@ -277,60 +241,60 @@ export const save = async (
     // no id was specified - load from email instead
     dbUser = await usersCore.getGlobalUserByEmail(email)
     if (dbUser && dbUser._id !== _id) {
-      throw `Unavailable`
+      throw new EmailUnavailableError(email)
     }
   }
 
-  await validateUniqueUser(email, tenantId)
+  const change = dbUser ? 0 : 1 // no change if there is existing user
+  return pro.quotas.addUsers(change, async () => {
+    await validateUniqueUser(email, tenantId)
 
-  let builtUser = await buildUser(user, opts, tenantId, dbUser)
-  // don't allow a user to update its own roles/perms
-  if (opts.currentUserId && opts.currentUserId === dbUser?._id) {
-    builtUser.builder = dbUser.builder
-    builtUser.admin = dbUser.admin
-    builtUser.roles = dbUser.roles
-  }
+    let builtUser = await buildUser(user, opts, tenantId, dbUser)
+    // don't allow a user to update its own roles/perms
+    if (opts.currentUserId && opts.currentUserId === dbUser?._id) {
+      builtUser.builder = dbUser.builder
+      builtUser.admin = dbUser.admin
+      builtUser.roles = dbUser.roles
+    }
 
-  if (!dbUser && roles?.length) {
-    builtUser.roles = { ...roles }
-  }
+    if (!dbUser && roles?.length) {
+      builtUser.roles = { ...roles }
+    }
 
-  // make sure we set the _id field for a new user
-  // Also if this is a new user, associate groups with them
-  let groupPromises = []
-  if (!_id) {
-    _id = builtUser._id!
+    // make sure we set the _id field for a new user
+    // Also if this is a new user, associate groups with them
+    let groupPromises = []
+    if (!_id) {
+      _id = builtUser._id!
 
-    if (userGroups.length > 0) {
-      for (let groupId of userGroups) {
-        groupPromises.push(pro.groups.addUsers(groupId, [_id]))
+      if (userGroups.length > 0) {
+        for (let groupId of userGroups) {
+          groupPromises.push(pro.groups.addUsers(groupId, [_id]))
+        }
       }
     }
-  }
-
-  try {
-    // save the user to db
-    let response = await db.put(builtUser)
-    builtUser._rev = response.rev
 
-    await eventHelpers.handleSaveEvents(builtUser, dbUser)
-    await platform.users.addUser(tenantId, builtUser._id!, builtUser.email)
-    await cache.user.invalidateUser(response.id)
+    try {
+      // save the user to db
+      let response = await db.put(builtUser)
+      builtUser._rev = response.rev
 
-    // let server know to sync user
-    await apps.syncUserInApps(_id, dbUser)
+      await eventHelpers.handleSaveEvents(builtUser, dbUser)
+      await platform.users.addUser(tenantId, builtUser._id!, builtUser.email)
+      await cache.user.invalidateUser(response.id)
 
-    await Promise.all(groupPromises)
+      await Promise.all(groupPromises)
 
-    // finally returned the saved user from the db
-    return db.get(builtUser._id!)
-  } catch (err: any) {
-    if (err.status === 409) {
-      throw "User exists already"
-    } else {
-      throw err
+      // finally returned the saved user from the db
+      return db.get(builtUser._id!)
+    } catch (err: any) {
+      if (err.status === 409) {
+        throw "User exists already"
+      } else {
+        throw err
+      }
     }
-  }
+  })
 }
 
 const getExistingTenantUsers = async (emails: string[]): Promise<User[]> => {
@@ -416,7 +380,7 @@ const searchExistingEmails = async (emails: string[]) => {
 export const bulkCreate = async (
   newUsersRequested: User[],
   groups: string[]
-): Promise<BulkUserResponse["created"]> => {
+): Promise<BulkUserCreated> => {
   const tenantId = tenancy.getTenantId()
 
   let usersToSave: any[] = []
@@ -444,55 +408,56 @@ export const bulkCreate = async (
   }
 
   const account = await accountSdk.api.getAccountByTenantId(tenantId)
-  // create the promises array that will be called by bulkDocs
-  newUsers.forEach((user: any) => {
-    usersToSave.push(
-      buildUser(
-        user,
-        {
-          hashPassword: true,
-          requirePassword: user.requirePassword,
-        },
-        tenantId,
-        undefined, // no dbUser
-        account
+  return pro.quotas.addUsers(newUsers.length, async () => {
+    // create the promises array that will be called by bulkDocs
+    newUsers.forEach((user: any) => {
+      usersToSave.push(
+        buildUser(
+          user,
+          {
+            hashPassword: true,
+            requirePassword: user.requirePassword,
+          },
+          tenantId,
+          undefined, // no dbUser
+          account
+        )
       )
-    )
-  })
+    })
 
-  const usersToBulkSave = await Promise.all(usersToSave)
-  await usersCore.bulkUpdateGlobalUsers(usersToBulkSave)
+    const usersToBulkSave = await Promise.all(usersToSave)
+    await usersCore.bulkUpdateGlobalUsers(usersToBulkSave)
 
-  // Post-processing of bulk added users, e.g. events and cache operations
-  for (const user of usersToBulkSave) {
-    // TODO: Refactor to bulk insert users into the info db
-    // instead of relying on looping tenant creation
-    await platform.users.addUser(tenantId, user._id, user.email)
-    await eventHelpers.handleSaveEvents(user, undefined)
-    await apps.syncUserInApps(user._id)
-  }
-
-  const saved = usersToBulkSave.map(user => {
-    return {
-      _id: user._id,
-      email: user.email,
+    // Post-processing of bulk added users, e.g. events and cache operations
+    for (const user of usersToBulkSave) {
+      // TODO: Refactor to bulk insert users into the info db
+      // instead of relying on looping tenant creation
+      await platform.users.addUser(tenantId, user._id, user.email)
+      await eventHelpers.handleSaveEvents(user, undefined)
     }
-  })
 
-  // now update the groups
-  if (Array.isArray(saved) && groups) {
-    const groupPromises = []
-    const createdUserIds = saved.map(user => user._id)
-    for (let groupId of groups) {
-      groupPromises.push(pro.groups.addUsers(groupId, createdUserIds))
+    const saved = usersToBulkSave.map(user => {
+      return {
+        _id: user._id,
+        email: user.email,
+      }
+    })
+
+    // now update the groups
+    if (Array.isArray(saved) && groups) {
+      const groupPromises = []
+      const createdUserIds = saved.map(user => user._id)
+      for (let groupId of groups) {
+        groupPromises.push(pro.groups.addUsers(groupId, createdUserIds))
+      }
+      await Promise.all(groupPromises)
     }
-    await Promise.all(groupPromises)
-  }
 
-  return {
-    successful: saved,
-    unsuccessful,
-  }
+    return {
+      successful: saved,
+      unsuccessful,
+    }
+  })
 }
 
 /**
@@ -517,10 +482,10 @@ const getAccountHolderFromUserIds = async (
 
 export const bulkDelete = async (
   userIds: string[]
-): Promise<BulkUserResponse["deleted"]> => {
+): Promise<BulkUserDeleted> => {
   const db = tenancy.getGlobalDB()
 
-  const response: BulkUserResponse["deleted"] = {
+  const response: BulkUserDeleted = {
     successful: [],
     unsuccessful: [],
   }
@@ -554,6 +519,8 @@ export const bulkDelete = async (
     _deleted: true,
   }))
   const dbResponse = await usersCore.bulkUpdateGlobalUsers(toDelete)
+
+  await pro.quotas.removeUsers(toDelete.length)
   for (let user of usersToDelete) {
     await bulkDeleteProcessing(user)
   }
@@ -583,7 +550,9 @@ export const bulkDelete = async (
   return response
 }
 
-export const destroy = async (id: string, currentUser: any) => {
+// TODO: The single delete should re-use the bulk delete with a single
+// user so that we don't need to duplicate logic
+export const destroy = async (id: string) => {
   const db = tenancy.getGlobalDB()
   const dbUser = (await db.get(id)) as User
   const userId = dbUser._id as string
@@ -593,7 +562,7 @@ export const destroy = async (id: string, currentUser: any) => {
     const email = dbUser.email
     const account = await accounts.getAccount(email)
     if (account) {
-      if (email === currentUser.email) {
+      if (dbUser.userId === context.getIdentity()!._id) {
         throw new HTTPError('Please visit "Account" to delete this user', 400)
       } else {
         throw new HTTPError("Account holder cannot be deleted", 400)
@@ -605,11 +574,10 @@ export const destroy = async (id: string, currentUser: any) => {
 
   await db.remove(userId, dbUser._rev)
 
+  await pro.quotas.removeUsers(1)
   await eventHelpers.handleDeleteEvents(dbUser)
   await cache.user.invalidateUser(userId)
   await sessions.invalidateSessions(userId, { reason: "deletion" })
-  // let server know to sync user
-  await apps.syncUserInApps(userId, dbUser)
 }
 
 const bulkDeleteProcessing = async (dbUser: User) => {
@@ -618,8 +586,6 @@ const bulkDeleteProcessing = async (dbUser: User) => {
   await eventHelpers.handleDeleteEvents(dbUser)
   await cache.user.invalidateUser(userId)
   await sessions.invalidateSessions(userId, { reason: "bulk-deletion" })
-  // let server know to sync user
-  await apps.syncUserInApps(userId, dbUser)
 }
 
 export const invite = async (

package/src/tests/TestConfiguration.ts

@@ -20,9 +20,18 @@ import {
   auth,
   constants,
   env as coreEnv,
+  db as dbCore,
+  encryption,
+  utils,
 } from "@budibase/backend-core"
 import structures, { CSRF_TOKEN } from "./structures"
-import { SaveUserResponse, User, AuthToken } from "@budibase/types"
+import {
+  SaveUserResponse,
+  User,
+  AuthToken,
+  SCIMConfig,
+  ConfigType,
+} from "@budibase/types"
 import API from "./api"
 
 class TestConfiguration {
@@ -31,6 +40,7 @@ class TestConfiguration {
   api: API
   tenantId: string
   user?: User
+  apiKey?: string
   userPassword = "test"
 
   constructor(opts: { openServer: boolean } = { openServer: true }) {
@@ -49,6 +59,12 @@ class TestConfiguration {
     this.api = new API(this)
   }
 
+  async useNewTenant() {
+    this.tenantId = structures.tenant.id()
+
+    await this.beforeAll()
+  }
+
   getRequest() {
     return this.request
   }
@@ -201,6 +217,12 @@ class TestConfiguration {
     return { [constants.Header.API_KEY]: coreEnv.INTERNAL_API_KEY }
   }
 
+  bearerAPIHeaders() {
+    return {
+      [constants.Header.AUTHORIZATION]: `Bearer ${this.apiKey}`,
+    }
+  }
+
   adminOnlyResponse = () => {
     return { message: "Admin user only endpoint.", status: 403 }
   }
@@ -213,6 +235,20 @@ class TestConfiguration {
     })
     await context.doInTenant(this.tenantId!, async () => {
       this.user = await this.createUser(user)
+
+      const db = context.getGlobalDB()
+
+      const id = dbCore.generateDevInfoID(this.user._id)
+      // TODO: dry
+      this.apiKey = encryption.encrypt(
+        `${this.tenantId}${dbCore.SEPARATOR}${utils.newid()}`
+      )
+      const devInfo = {
+        _id: id,
+        userId: this.user._id,
+        apiKey: this.apiKey,
+      }
+      await db.put(devInfo)
     })
   }
 
@@ -305,6 +341,19 @@ class TestConfiguration {
       controllers.config.save
     )
   }
+
+  // CONFIGS - SCIM
+
+  async setSCIMConfig(enabled: boolean) {
+    await this.deleteConfig(Config.SCIM)
+    const config: SCIMConfig = {
+      type: ConfigType.SCIM,
+      config: { enabled },
+    }
+
+    await this._req(config, null, controllers.config.save)
+    return config
+  }
 }
 
 export default TestConfiguration

package/src/tests/api/index.ts

@@ -15,6 +15,9 @@ import { RolesAPI } from "./roles"
 import { TemplatesAPI } from "./templates"
 import { LicenseAPI } from "./license"
 import { AuditLogAPI } from "./auditLogs"
+import { ScimUsersAPI } from "./scim/users"
+import { ScimGroupsAPI } from "./scim/groups"
+
 export default class API {
   accounts: AccountAPI
   auth: AuthAPI
@@ -32,6 +35,8 @@ export default class API {
   templates: TemplatesAPI
   license: LicenseAPI
   auditLogs: AuditLogAPI
+  scimUsersAPI: ScimUsersAPI
+  scimGroupsAPI: ScimGroupsAPI
 
   constructor(config: TestConfiguration) {
     this.accounts = new AccountAPI(config)
@@ -50,5 +55,7 @@ export default class API {
     this.templates = new TemplatesAPI(config)
     this.license = new LicenseAPI(config)
     this.auditLogs = new AuditLogAPI(config)
+    this.scimUsersAPI = new ScimUsersAPI(config)
+    this.scimGroupsAPI = new ScimGroupsAPI(config)
   }
 }