@budibase/worker 2.3.20 → 2.3.21-alpha.0

package/src/api/routes/global/tests/self.spec.ts

@@ -30,7 +30,7 @@ describe("/api/global/self", () => {
       user.dayPassRecordedAt = mocks.date.MOCK_DATE.toISOString()
       expect(res.body._id).toBe(user._id)
       expect(events.user.updated).toBeCalledTimes(1)
-      expect(events.user.updated).toBeCalledWith(user)
+      expect(events.user.updated).toBeCalledWith(dbUser)
       expect(events.user.passwordUpdated).not.toBeCalled()
     })
 
@@ -44,12 +44,11 @@ describe("/api/global/self", () => {
       const dbUser = await config.getUser(user.email)
       user._rev = dbUser._rev
       user.dayPassRecordedAt = mocks.date.MOCK_DATE.toISOString()
-      delete user.password
       expect(res.body._id).toBe(user._id)
       expect(events.user.updated).toBeCalledTimes(1)
-      expect(events.user.updated).toBeCalledWith(user)
+      expect(events.user.updated).toBeCalledWith(dbUser)
       expect(events.user.passwordUpdated).toBeCalledTimes(1)
-      expect(events.user.passwordUpdated).toBeCalledWith(user)
+      expect(events.user.passwordUpdated).toBeCalledWith(dbUser)
     })
   })
 })

package/src/api/routes/global/tests/users.spec.ts

@@ -1,14 +1,11 @@
 import { InviteUsersResponse, User } from "@budibase/types"
 
 jest.mock("nodemailer")
-import {
-  TestConfiguration,
-  mocks,
-  structures,
-  TENANT_1,
-} from "../../../../tests"
+import { TestConfiguration, mocks, structures } from "../../../../tests"
 const sendMailMock = mocks.email.mock()
-import { events, tenancy } from "@budibase/backend-core"
+import { events, tenancy, accounts as _accounts } from "@budibase/backend-core"
+
+const accounts = jest.mocked(_accounts)
 
 describe("/api/global/users", () => {
   const config = new TestConfiguration()
@@ -25,7 +22,7 @@ describe("/api/global/users", () => {
     jest.clearAllMocks()
   })
 
-  describe("invite", () => {
+  describe("POST /api/global/users/invite", () => {
     it("should be able to generate an invitation", async () => {
       const email = structures.users.newEmail()
       const { code, res } = await config.api.users.sendUserInvite(
@@ -33,7 +30,11 @@ describe("/api/global/users", () => {
         email
       )
 
-      expect(res.body).toEqual({ message: "Invitation has been sent." })
+      expect(res.body?.message).toBe("Invitation has been sent.")
+      expect(res.body?.unsuccessful.length).toBe(0)
+      expect(res.body?.successful.length).toBe(1)
+      expect(res.body?.successful[0].email).toBe(email)
+
       expect(sendMailMock).toHaveBeenCalled()
       expect(code).toBeDefined()
       expect(events.user.invited).toBeCalledTimes(1)
@@ -42,7 +43,7 @@ describe("/api/global/users", () => {
     it("should not be able to generate an invitation for existing user", async () => {
       const { code, res } = await config.api.users.sendUserInvite(
         sendMailMock,
-        config.defaultUser!.email,
+        config.user!.email,
         400
       )
 
@@ -70,7 +71,7 @@ describe("/api/global/users", () => {
     })
   })
 
-  describe("inviteMultiple", () => {
+  describe("POST /api/global/users/multi/invite", () => {
     it("should be able to generate an invitation", async () => {
       const newUserInvite = () => ({
         email: structures.users.newEmail(),
@@ -88,7 +89,7 @@ describe("/api/global/users", () => {
     })
 
     it("should not be able to generate an invitation for existing user", async () => {
-      const request = [{ email: config.defaultUser!.email, userInfo: {} }]
+      const request = [{ email: config.user!.email, userInfo: {} }]
 
       const res = await config.api.users.sendMultiUserInvite(request)
 
@@ -101,7 +102,7 @@ describe("/api/global/users", () => {
     })
   })
 
-  describe("bulk (create)", () => {
+  describe("POST /api/global/users/bulk", () => {
     it("should ignore users existing in the same tenant", async () => {
       const user = await config.createUser()
       jest.clearAllMocks()
@@ -118,7 +119,7 @@ describe("/api/global/users", () => {
       const user = await config.createUser()
       jest.clearAllMocks()
 
-      await tenancy.doInTenant(TENANT_1, async () => {
+      await tenancy.doInTenant(config.getTenantId(), async () => {
         const response = await config.api.users.bulkCreateUsers([user])
 
         expect(response.created?.successful.length).toBe(0)
@@ -164,7 +165,7 @@ describe("/api/global/users", () => {
     })
   })
 
-  describe("create", () => {
+  describe("POST /api/global/users", () => {
     it("should be able to create a basic user", async () => {
       const user = structures.users.user()
 
@@ -231,7 +232,7 @@ describe("/api/global/users", () => {
       const user = await config.createUser()
       jest.clearAllMocks()
 
-      await tenancy.doInTenant(TENANT_1, async () => {
+      await tenancy.doInTenant(config.getTenantId(), async () => {
         delete user._id
         const response = await config.api.users.saveUser(user, 400)
 
@@ -243,7 +244,7 @@ describe("/api/global/users", () => {
     it("should not be able to create user with the same email as an account", async () => {
       const user = structures.users.user()
       const account = structures.accounts.cloudAccount()
-      mocks.accounts.getAccount.mockReturnValueOnce(account)
+      accounts.getAccount.mockReturnValueOnce(Promise.resolve(account))
 
       const response = await config.api.users.saveUser(user, 400)
 
@@ -284,7 +285,7 @@ describe("/api/global/users", () => {
     })
   })
 
-  describe("update", () => {
+  describe("POST /api/global/users (update)", () => {
     it("should be able to update a basic user", async () => {
       const user = await config.createUser()
       jest.clearAllMocks()
@@ -299,7 +300,7 @@ describe("/api/global/users", () => {
     })
 
     it("should not allow a user to update their own admin/builder status", async () => {
-      const user = (await config.api.users.getUser(config.defaultUser?._id!))
+      const user = (await config.api.users.getUser(config.user?._id!))
         .body as User
       await config.api.users.saveUser({
         ...user,
@@ -444,7 +445,7 @@ describe("/api/global/users", () => {
     })
 
     it("should not be able to update email address", async () => {
-      const email = "email@test.com"
+      const email = structures.email()
       const user = await config.createUser(structures.users.user({ email }))
       user.email = "new@test.com"
 
@@ -469,9 +470,9 @@ describe("/api/global/users", () => {
     })
   })
 
-  describe("bulk (delete)", () => {
+  describe("POST /api/global/users/bulk (delete)", () => {
     it("should not be able to bulk delete current user", async () => {
-      const user = await config.defaultUser!
+      const user = await config.user!
 
       const response = await config.api.users.bulkDeleteUsers([user._id!], 400)
 
@@ -483,7 +484,7 @@ describe("/api/global/users", () => {
       const user = await config.createUser()
       const account = structures.accounts.cloudAccount()
       account.budibaseUserId = user._id!
-      mocks.accounts.getAccountByTenantId.mockReturnValue(account)
+      accounts.getAccountByTenantId.mockReturnValue(Promise.resolve(account))
 
       const response = await config.api.users.bulkDeleteUsers([user._id!])
 
@@ -498,7 +499,7 @@ describe("/api/global/users", () => {
 
     it("should be able to bulk delete users", async () => {
       const account = structures.accounts.cloudAccount()
-      mocks.accounts.getAccountByTenantId.mockReturnValue(account)
+      accounts.getAccountByTenantId.mockReturnValue(Promise.resolve(account))
 
       const builder = structures.users.builderUser()
       const admin = structures.users.adminUser()
@@ -522,7 +523,7 @@ describe("/api/global/users", () => {
     })
   })
 
-  describe("destroy", () => {
+  describe("DELETE /api/global/users/:userId", () => {
     it("should be able to destroy a basic user", async () => {
       const user = await config.createUser()
       jest.clearAllMocks()
@@ -559,7 +560,7 @@ describe("/api/global/users", () => {
     it("should not be able to destroy account owner", async () => {
       const user = await config.createUser()
       const account = structures.accounts.cloudAccount()
-      mocks.accounts.getAccount.mockReturnValueOnce(account)
+      accounts.getAccount.mockReturnValueOnce(Promise.resolve(account))
 
       const response = await config.api.users.deleteUser(user._id!, 400)
 
@@ -567,10 +568,10 @@ describe("/api/global/users", () => {
     })
 
     it("should not be able to destroy account owner as account owner", async () => {
-      const user = await config.defaultUser!
+      const user = await config.user!
       const account = structures.accounts.cloudAccount()
       account.email = user.email
-      mocks.accounts.getAccount.mockReturnValueOnce(account)
+      accounts.getAccount.mockReturnValueOnce(Promise.resolve(account))
 
       const response = await config.api.users.deleteUser(user._id!, 400)
 

package/src/api/routes/global/users.ts

@@ -38,13 +38,6 @@ function buildInviteMultipleValidation() {
   ))
 }
 
-function buildInviteLookupValidation() {
-  // prettier-ignore
-  return auth.joiValidator.params(Joi.object({
-    code: Joi.string().required()
-  }).unknown(true))
-}
-
 const createUserAdminOnly = (ctx: any, next: any) => {
   if (!ctx.request.body._id) {
     return auth.adminOnly(ctx, next)
@@ -57,8 +50,8 @@ function buildInviteAcceptValidation() {
   // prettier-ignore
   return auth.joiValidator.body(Joi.object({
     inviteCode: Joi.string().required(),
-    password: Joi.string().required(),
-    firstName: Joi.string().required(),
+    password: Joi.string().optional(),
+    firstName: Joi.string().optional(),
     lastName: Joi.string().optional(),
   }).required().unknown(true))
 }
@@ -88,22 +81,34 @@ router
   .get("/api/global/roles/:appId")
   .post(
     "/api/global/users/invite",
-    auth.adminOnly,
+    auth.builderOrAdmin,
     buildInviteValidation(),
     controller.invite
   )
+  .post(
+    "/api/global/users/onboard",
+    auth.builderOrAdmin,
+    buildInviteMultipleValidation(),
+    controller.onboardUsers
+  )
   .post(
     "/api/global/users/multi/invite",
-    auth.adminOnly,
+    auth.builderOrAdmin,
     buildInviteMultipleValidation(),
     controller.inviteMultiple
   )
 
   // non-global endpoints
+  .get("/api/global/users/invite/:code", controller.checkInvite)
+  .post(
+    "/api/global/users/invite/update/:code",
+    auth.builderOrAdmin,
+    controller.updateInvite
+  )
   .get(
-    "/api/global/users/invite/:code",
-    buildInviteLookupValidation(),
-    controller.checkInvite
+    "/api/global/users/invites",
+    auth.builderOrAdmin,
+    controller.getUserInvites
   )
   .post(
     "/api/global/users/invite/accept",

package/src/api/routes/index.ts

@@ -18,6 +18,8 @@ import accountRoutes from "./system/accounts"
 import restoreRoutes from "./system/restore"
 
 let userGroupRoutes = api.groups
+let auditLogRoutes = api.auditLogs
+
 export const routes: Router[] = [
   configRoutes,
   userRoutes,
@@ -32,6 +34,7 @@ export const routes: Router[] = [
   selfRoutes,
   licenseRoutes,
   userGroupRoutes,
+  auditLogRoutes,
   migrationRoutes,
   accountRoutes,
   restoreRoutes,

package/src/api/routes/system/tenants.ts

@@ -7,7 +7,7 @@ const router: Router = new Router()
 router.delete(
   "/api/system/tenants/:tenantId",
   middleware.adminOnly,
-  controller.delete
+  controller.destroy
 )
 
 export default router

package/src/api/routes/system/tests/accounts.spec.ts

@@ -1,4 +1,4 @@
-import sdk from "../../../../sdk"
+import * as accounts from "../../../../sdk/accounts"
 import { TestConfiguration, structures } from "../../../../tests"
 import { v4 as uuid } from "uuid"
 
@@ -24,8 +24,8 @@ describe("accounts", () => {
 
         const response = await config.api.accounts.saveMetadata(account)
 
-        const id = sdk.accounts.formatAccountMetadataId(account.accountId)
-        const metadata = await sdk.accounts.getMetadata(id)
+        const id = accounts.metadata.formatAccountMetadataId(account.accountId)
+        const metadata = await accounts.metadata.getMetadata(id)
         expect(response).toStrictEqual(metadata)
       })
     })
@@ -37,7 +37,7 @@ describe("accounts", () => {
 
         await config.api.accounts.destroyMetadata(account.accountId)
 
-        const deleted = await sdk.accounts.getMetadata(account.accountId)
+        const deleted = await accounts.metadata.getMetadata(account.accountId)
         expect(deleted).toBe(undefined)
       })
 

package/src/api/routes/system/tests/migrations.spec.ts

@@ -30,7 +30,7 @@ describe("/api/system/migrations", () => {
         headers: {},
         status: 403,
       })
-      expect(res.text).toBe("Unauthorized - no public worker access")
+      expect(res.body).toEqual({ message: "Unauthorized", status: 403 })
       expect(migrateFn).toBeCalledTimes(0)
     })
 
@@ -47,7 +47,7 @@ describe("/api/system/migrations", () => {
         headers: {},
         status: 403,
       })
-      expect(res.text).toBe("Unauthorized - no public worker access")
+      expect(res.body).toEqual({ message: "Unauthorized", status: 403 })
     })
 
     it("returns definitions", async () => {

package/src/api/routes/system/tests/restore.spec.ts

@@ -25,12 +25,12 @@ describe("/api/system/restore", () => {
     })
 
     it("restores in self host", async () => {
-      config.modeSelf()
+      config.selfHosted()
       const res = await config.api.restore.restored()
       expect(res.body).toEqual({
         message: "System prepared after restore.",
       })
-      config.modeCloud()
+      config.cloudHosted()
     })
   })
 })

package/src/api/routes/system/tests/status.spec.ts

@@ -1,6 +1,6 @@
 import { TestConfiguration } from "../../../../tests"
-import { accounts } from "@budibase/backend-core"
-import { mocks } from "@budibase/backend-core/tests"
+import { accounts as _accounts } from "@budibase/backend-core"
+const accounts = jest.mocked(_accounts)
 
 describe("/api/system/status", () => {
   const config = new TestConfiguration()
@@ -19,7 +19,7 @@ describe("/api/system/status", () => {
 
   describe("GET /api/system/status", () => {
     it("returns status in self host", async () => {
-      config.modeSelf()
+      config.selfHosted()
       const res = await config.api.status.getStatus()
       expect(res.body).toEqual({
         health: {
@@ -27,7 +27,7 @@ describe("/api/system/status", () => {
         },
       })
       expect(accounts.getStatus).toBeCalledTimes(0)
-      config.modeCloud()
+      config.cloudHosted()
     })
 
     it("returns status in cloud", async () => {
@@ -37,7 +37,7 @@ describe("/api/system/status", () => {
         },
       }
 
-      mocks.accounts.getStatus.mockReturnValueOnce(value)
+      accounts.getStatus.mockReturnValueOnce(Promise.resolve(value))
 
       const res = await config.api.status.getStatus()
 

package/src/db/index.ts

@@ -1,10 +1,16 @@
 import * as core from "@budibase/backend-core"
 import env from "../environment"
 
-export const init = () => {
-  const dbConfig: any = {}
+export function init() {
+  const dbConfig: any = {
+    replication: true,
+    find: true,
+  }
+
   if (env.isTest() && !env.COUCH_DB_URL) {
     dbConfig.inMemory = true
+    dbConfig.allDbs = true
   }
+
   core.init({ db: dbConfig })
 }

package/src/ddApm.ts

@@ -0,0 +1,7 @@
+import apm from "dd-trace"
+
+// enable APM if configured
+if (process.env.DD_APM_ENABLED) {
+  console.log("Starting dd-trace")
+  apm.init()
+}

package/src/elasticApm.ts

@@ -0,0 +1,10 @@
+import apm from "elastic-apm-node"
+
+// enable APM if configured
+if (process.env.ELASTIC_APM_ENABLED) {
+  console.log("Starting elastic-apm-node")
+  apm.start({
+    serviceName: process.env.SERVICE,
+    environment: process.env.BUDIBASE_ENVIRONMENT,
+  })
+}

package/src/environment.ts

@@ -65,6 +65,10 @@ const environment = {
   CHECKLIST_CACHE_TTL: parseIntSafe(process.env.CHECKLIST_CACHE_TTL) || 3600,
   SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD,
   ENCRYPTED_TEST_PUBLIC_API_KEY: process.env.ENCRYPTED_TEST_PUBLIC_API_KEY,
+  /**
+   * Mock the email service in use - links to ethereal hosted emails are logged instead.
+   */
+  ENABLE_EMAIL_TEST_MODE: process.env.ENABLE_EMAIL_TEST_MODE,
   _set(key: any, value: any) {
     process.env[key] = value
     // @ts-ignore

package/src/index.ts

@@ -1,20 +1,27 @@
-// need to load environment first
-import env from "./environment"
+if (process.env.DD_APM_ENABLED) {
+  require("./ddApm")
+}
 
-// enable APM if configured
 if (process.env.ELASTIC_APM_ENABLED) {
-  const apm = require("elastic-apm-node").start({
-    serviceName: process.env.SERVICE,
-    environment: process.env.BUDIBASE_ENVIRONMENT,
-  })
+  require("./elasticApm")
 }
 
+// need to load environment first
+import env from "./environment"
 import { Scope } from "@sentry/node"
 import { Event } from "@sentry/types/dist/event"
 import Application from "koa"
 import { bootstrap } from "global-agent"
 import * as db from "./db"
-import { auth, logging, events, middleware } from "@budibase/backend-core"
+import { sdk as proSdk } from "@budibase/pro"
+import {
+  auth,
+  logging,
+  events,
+  middleware,
+  queue,
+  env as coreEnv,
+} from "@budibase/backend-core"
 db.init()
 import Koa from "koa"
 import koaBody from "koa-body"
@@ -24,8 +31,20 @@ import * as redis from "./utilities/redis"
 const Sentry = require("@sentry/node")
 const koaSession = require("koa-session")
 const logger = require("koa-pino-logger")
+const { userAgent } = require("koa-useragent")
+
 import destroyable from "server-destroy"
 
+// configure events to use the pro audit log write
+// can't integrate directly into backend-core due to cyclic issues
+events.processors.init(proSdk.auditLogs.write)
+
+if (coreEnv.ENABLE_SSO_MAINTENANCE_MODE) {
+  console.warn(
+    "Warning: ENABLE_SSO_MAINTENANCE_MODE is set. It is recommended this flag is disabled if maintenance is not in progress"
+  )
+}
+
 // this will setup http and https proxies form env variables
 bootstrap()
 
@@ -38,6 +57,7 @@ app.use(koaBody({ multipart: true }))
 app.use(koaSession(app))
 app.use(middleware.logging)
 app.use(logger(logging.pinoSettings()))
+app.use(userAgent)
 
 // authentication
 app.use(auth.passport.initialize())
@@ -73,6 +93,7 @@ server.on("close", async () => {
   console.log("Server Closed")
   await redis.shutdown()
   await events.shutdown()
+  await queue.shutdown()
   if (!env.isTest()) {
     process.exit(errCode)
   }

package/src/middleware/tests/tenancy.spec.ts

@@ -24,7 +24,7 @@ describe("tenancy middleware", () => {
   })
 
   it("should get tenant id from header", async () => {
-    const tenantId = structures.uuid()
+    const tenantId = structures.tenant.id()
     const headers = {
       [constants.Header.TENANT_ID]: tenantId,
     }
@@ -35,7 +35,7 @@ describe("tenancy middleware", () => {
   })
 
   it("should get tenant id from query param", async () => {
-    const tenantId = structures.uuid()
+    const tenantId = structures.tenant.id()
     const res = await config.request.get(
       `/api/global/configs/checklist?tenantId=${tenantId}`
     )
@@ -43,7 +43,7 @@ describe("tenancy middleware", () => {
   })
 
   it("should get tenant id from subdomain", async () => {
-    const tenantId = structures.uuid()
+    const tenantId = structures.tenant.id()
     const headers = {
       host: `${tenantId}.localhost:10000`,
     }
@@ -67,7 +67,7 @@ describe("tenancy middleware", () => {
   it("should throw when no tenant id is found", async () => {
     const res = await config.request.get(`/api/global/configs/checklist`)
     expect(res.status).toBe(403)
-    expect(res.text).toBe("Tenant id not set")
+    expect(res.body).toEqual({ message: "Tenant id not set", status: 403 })
     expect(res.headers[constants.Header.TENANT_ID]).toBe(undefined)
   })
 })

package/src/migrations/functions/globalInfoSyncUsers.ts

@@ -1,5 +1,6 @@
 import { User } from "@budibase/types"
-import sdk from "../../sdk"
+import * as usersSdk from "../../sdk/users"
+import { platform } from "@budibase/backend-core"
 
 /**
  * Date:
@@ -9,11 +10,11 @@ import sdk from "../../sdk"
  * Re-sync the global-db users to the global-info db users
  */
 export const run = async (globalDb: any) => {
-  const users = (await sdk.users.allUsers()) as User[]
+  const users = (await usersSdk.allUsers()) as User[]
   const promises = []
   for (let user of users) {
     promises.push(
-      sdk.users.addTenant(user.tenantId, user._id as string, user.email)
+      platform.users.addUser(user.tenantId, user._id as string, user.email)
     )
   }
   await Promise.all(promises)

package/src/sdk/accounts/index.ts

@@ -1 +1,2 @@
-export * from "./accounts"
+export * as metadata from "./metadata"
+export { accounts as api } from "@budibase/backend-core"

package/src/sdk/accounts/{accounts.ts → metadata.ts}

@@ -2,7 +2,6 @@ import { AccountMetadata } from "@budibase/types"
 import {
   db,
   StaticDatabases,
-  HTTPError,
   DocumentType,
   SEPARATOR,
 } from "@budibase/backend-core"