@budibase/worker 2.3.17 → 2.3.18-alpha.0

package/src/api/routes/global/tests/configs.spec.ts

@@ -7,7 +7,7 @@ import { Config, events } from "@budibase/backend-core"
 describe("configs", () => {
   const config = new TestConfiguration()
 
-  beforeAll(async () => {
+  beforeEach(async () => {
     await config.beforeAll()
   })
 
@@ -221,7 +221,7 @@ describe("configs", () => {
         })
 
         it("should create settings config with non-default settings", async () => {
-          config.modeSelf()
+          config.selfHosted()
           await config.deleteConfig(Config.SETTINGS)
           const conf = {
             company: "acme",
@@ -234,13 +234,13 @@ describe("configs", () => {
           expect(events.org.nameUpdated).toBeCalledTimes(1)
           expect(events.org.logoUpdated).toBeCalledTimes(1)
           expect(events.org.platformURLUpdated).toBeCalledTimes(1)
-          config.modeCloud()
+          config.cloudHosted()
         })
       })
 
       describe("update", () => {
         it("should update settings config", async () => {
-          config.modeSelf()
+          config.selfHosted()
           await config.deleteConfig(Config.SETTINGS)
           const settingsConfig = await saveSettingsConfig()
           settingsConfig.config.company = "acme"
@@ -256,7 +256,7 @@ describe("configs", () => {
           expect(events.org.nameUpdated).toBeCalledTimes(1)
           expect(events.org.logoUpdated).toBeCalledTimes(1)
           expect(events.org.platformURLUpdated).toBeCalledTimes(1)
-          config.modeCloud()
+          config.cloudHosted()
         })
       })
     })

package/src/api/routes/global/tests/roles.spec.ts

@@ -1,6 +1,7 @@
 import { structures, TestConfiguration } from "../../../../tests"
 import { context, db, permissions, roles } from "@budibase/backend-core"
 import { Mock } from "jest-mock"
+import { Database } from "@budibase/types"
 
 jest.mock("@budibase/backend-core", () => {
   const core = jest.requireActual("@budibase/backend-core")
@@ -16,14 +17,13 @@ jest.mock("@budibase/backend-core", () => {
   }
 })
 
-const appDb = db.getDB("app_test")
-const mockAppDB = context.getAppDB as Mock
-mockAppDB.mockReturnValue(appDb)
+let appId: string
+let appDb: Database
 
 async function addAppMetadata() {
   await appDb.put({
     _id: "app_metadata",
-    appId: "app_test",
+    appId: appId,
     name: "New App",
     version: "version",
     url: "url",
@@ -32,6 +32,7 @@ async function addAppMetadata() {
 
 describe("/api/global/roles", () => {
   const config = new TestConfiguration()
+
   const role = new roles.Role(
     db.generateRoleID("newRole"),
     roles.BUILTIN_ROLE_IDS.BASIC,
@@ -39,12 +40,19 @@ describe("/api/global/roles", () => {
   )
 
   beforeAll(async () => {
-    console.debug(role)
-    appDb.put(role)
-    await addAppMetadata()
     await config.beforeAll()
   })
 
+  beforeEach(async () => {
+    appId = db.generateAppID(config.tenantId)
+    appDb = db.getDB(appId)
+    const mockAppDB = context.getAppDB as Mock
+    mockAppDB.mockReturnValue(appDb)
+
+    await addAppMetadata()
+    await appDb.put(role)
+  })
+
   afterAll(async () => {
     await config.afterAll()
   })
@@ -57,16 +65,14 @@ describe("/api/global/roles", () => {
     it("retrieves roles", async () => {
       const res = await config.api.roles.get()
       expect(res.body).toBeDefined()
-      expect(res.body["app_test"].roles.length).toEqual(5)
-      expect(res.body["app_test"].roles.map((r: any) => r._id)).toContain(
-        role._id
-      )
+      expect(res.body[appId].roles.length).toEqual(5)
+      expect(res.body[appId].roles.map((r: any) => r._id)).toContain(role._id)
     })
   })
 
   describe("GET api/global/roles/:appId", () => {
     it("finds a role by appId", async () => {
-      const res = await config.api.roles.find("app_test")
+      const res = await config.api.roles.find(appId)
       expect(res.body).toBeDefined()
       expect(res.body.name).toEqual("New App")
     })
@@ -79,9 +85,9 @@ describe("/api/global/roles", () => {
         app_test: "role1",
       }
       const userResponse = await config.createUser(user)
-      const res = await config.api.roles.remove("app_test")
+      const res = await config.api.roles.remove(appId)
       const updatedUser = await config.api.users.getUser(userResponse._id!)
-      expect(updatedUser.body.roles).not.toHaveProperty("app_test")
+      expect(updatedUser.body.roles).not.toHaveProperty(appId)
       expect(res.body.message).toEqual("App role removed from all users")
     })
   })

package/src/api/routes/global/tests/self.spec.ts

@@ -30,7 +30,7 @@ describe("/api/global/self", () => {
       user.dayPassRecordedAt = mocks.date.MOCK_DATE.toISOString()
       expect(res.body._id).toBe(user._id)
       expect(events.user.updated).toBeCalledTimes(1)
-      expect(events.user.updated).toBeCalledWith(user)
+      expect(events.user.updated).toBeCalledWith(dbUser)
       expect(events.user.passwordUpdated).not.toBeCalled()
     })
 
@@ -44,12 +44,11 @@ describe("/api/global/self", () => {
       const dbUser = await config.getUser(user.email)
       user._rev = dbUser._rev
       user.dayPassRecordedAt = mocks.date.MOCK_DATE.toISOString()
-      delete user.password
       expect(res.body._id).toBe(user._id)
       expect(events.user.updated).toBeCalledTimes(1)
-      expect(events.user.updated).toBeCalledWith(user)
+      expect(events.user.updated).toBeCalledWith(dbUser)
       expect(events.user.passwordUpdated).toBeCalledTimes(1)
-      expect(events.user.passwordUpdated).toBeCalledWith(user)
+      expect(events.user.passwordUpdated).toBeCalledWith(dbUser)
     })
   })
 })

package/src/api/routes/global/tests/users.spec.ts

@@ -1,14 +1,11 @@
 import { InviteUsersResponse, User } from "@budibase/types"
 
 jest.mock("nodemailer")
-import {
-  TestConfiguration,
-  mocks,
-  structures,
-  TENANT_1,
-} from "../../../../tests"
+import { TestConfiguration, mocks, structures } from "../../../../tests"
 const sendMailMock = mocks.email.mock()
-import { events, tenancy } from "@budibase/backend-core"
+import { events, tenancy, accounts as _accounts } from "@budibase/backend-core"
+
+const accounts = jest.mocked(_accounts)
 
 describe("/api/global/users", () => {
   const config = new TestConfiguration()
@@ -25,7 +22,7 @@ describe("/api/global/users", () => {
     jest.clearAllMocks()
   })
 
-  describe("invite", () => {
+  describe("POST /api/global/users/invite", () => {
     it("should be able to generate an invitation", async () => {
       const email = structures.users.newEmail()
       const { code, res } = await config.api.users.sendUserInvite(
@@ -42,7 +39,7 @@ describe("/api/global/users", () => {
     it("should not be able to generate an invitation for existing user", async () => {
       const { code, res } = await config.api.users.sendUserInvite(
         sendMailMock,
-        config.defaultUser!.email,
+        config.user!.email,
         400
       )
 
@@ -70,7 +67,7 @@ describe("/api/global/users", () => {
     })
   })
 
-  describe("inviteMultiple", () => {
+  describe("POST /api/global/users/multi/invite", () => {
     it("should be able to generate an invitation", async () => {
       const newUserInvite = () => ({
         email: structures.users.newEmail(),
@@ -88,7 +85,7 @@ describe("/api/global/users", () => {
     })
 
     it("should not be able to generate an invitation for existing user", async () => {
-      const request = [{ email: config.defaultUser!.email, userInfo: {} }]
+      const request = [{ email: config.user!.email, userInfo: {} }]
 
       const res = await config.api.users.sendMultiUserInvite(request)
 
@@ -101,7 +98,7 @@ describe("/api/global/users", () => {
     })
   })
 
-  describe("bulk (create)", () => {
+  describe("POST /api/global/users/bulk", () => {
     it("should ignore users existing in the same tenant", async () => {
       const user = await config.createUser()
       jest.clearAllMocks()
@@ -118,7 +115,7 @@ describe("/api/global/users", () => {
       const user = await config.createUser()
       jest.clearAllMocks()
 
-      await tenancy.doInTenant(TENANT_1, async () => {
+      await tenancy.doInTenant(config.getTenantId(), async () => {
         const response = await config.api.users.bulkCreateUsers([user])
 
         expect(response.created?.successful.length).toBe(0)
@@ -164,7 +161,7 @@ describe("/api/global/users", () => {
     })
   })
 
-  describe("create", () => {
+  describe("POST /api/global/users", () => {
     it("should be able to create a basic user", async () => {
       const user = structures.users.user()
 
@@ -231,7 +228,7 @@ describe("/api/global/users", () => {
       const user = await config.createUser()
       jest.clearAllMocks()
 
-      await tenancy.doInTenant(TENANT_1, async () => {
+      await tenancy.doInTenant(config.getTenantId(), async () => {
         delete user._id
         const response = await config.api.users.saveUser(user, 400)
 
@@ -243,7 +240,7 @@ describe("/api/global/users", () => {
     it("should not be able to create user with the same email as an account", async () => {
       const user = structures.users.user()
       const account = structures.accounts.cloudAccount()
-      mocks.accounts.getAccount.mockReturnValueOnce(account)
+      accounts.getAccount.mockReturnValueOnce(Promise.resolve(account))
 
       const response = await config.api.users.saveUser(user, 400)
 
@@ -284,7 +281,7 @@ describe("/api/global/users", () => {
     })
   })
 
-  describe("update", () => {
+  describe("POST /api/global/users (update)", () => {
     it("should be able to update a basic user", async () => {
       const user = await config.createUser()
       jest.clearAllMocks()
@@ -299,7 +296,7 @@ describe("/api/global/users", () => {
     })
 
     it("should not allow a user to update their own admin/builder status", async () => {
-      const user = (await config.api.users.getUser(config.defaultUser?._id!))
+      const user = (await config.api.users.getUser(config.user?._id!))
         .body as User
       await config.api.users.saveUser({
         ...user,
@@ -444,7 +441,7 @@ describe("/api/global/users", () => {
     })
 
     it("should not be able to update email address", async () => {
-      const email = "email@test.com"
+      const email = structures.email()
       const user = await config.createUser(structures.users.user({ email }))
       user.email = "new@test.com"
 
@@ -469,9 +466,9 @@ describe("/api/global/users", () => {
     })
   })
 
-  describe("bulk (delete)", () => {
+  describe("POST /api/global/users/bulk (delete)", () => {
     it("should not be able to bulk delete current user", async () => {
-      const user = await config.defaultUser!
+      const user = await config.user!
 
       const response = await config.api.users.bulkDeleteUsers([user._id!], 400)
 
@@ -483,7 +480,7 @@ describe("/api/global/users", () => {
       const user = await config.createUser()
       const account = structures.accounts.cloudAccount()
       account.budibaseUserId = user._id!
-      mocks.accounts.getAccountByTenantId.mockReturnValue(account)
+      accounts.getAccountByTenantId.mockReturnValue(Promise.resolve(account))
 
       const response = await config.api.users.bulkDeleteUsers([user._id!])
 
@@ -498,7 +495,7 @@ describe("/api/global/users", () => {
 
     it("should be able to bulk delete users", async () => {
       const account = structures.accounts.cloudAccount()
-      mocks.accounts.getAccountByTenantId.mockReturnValue(account)
+      accounts.getAccountByTenantId.mockReturnValue(Promise.resolve(account))
 
       const builder = structures.users.builderUser()
       const admin = structures.users.adminUser()
@@ -522,7 +519,7 @@ describe("/api/global/users", () => {
     })
   })
 
-  describe("destroy", () => {
+  describe("DELETE /api/global/users/:userId", () => {
     it("should be able to destroy a basic user", async () => {
       const user = await config.createUser()
       jest.clearAllMocks()
@@ -559,7 +556,7 @@ describe("/api/global/users", () => {
     it("should not be able to destroy account owner", async () => {
       const user = await config.createUser()
       const account = structures.accounts.cloudAccount()
-      mocks.accounts.getAccount.mockReturnValueOnce(account)
+      accounts.getAccount.mockReturnValueOnce(Promise.resolve(account))
 
       const response = await config.api.users.deleteUser(user._id!, 400)
 
@@ -567,10 +564,10 @@ describe("/api/global/users", () => {
     })
 
     it("should not be able to destroy account owner as account owner", async () => {
-      const user = await config.defaultUser!
+      const user = await config.user!
       const account = structures.accounts.cloudAccount()
       account.email = user.email
-      mocks.accounts.getAccount.mockReturnValueOnce(account)
+      accounts.getAccount.mockReturnValueOnce(Promise.resolve(account))
 
       const response = await config.api.users.deleteUser(user._id!, 400)
 

package/src/api/routes/system/tenants.ts

@@ -7,7 +7,7 @@ const router: Router = new Router()
 router.delete(
   "/api/system/tenants/:tenantId",
   middleware.adminOnly,
-  controller.delete
+  controller.destroy
 )
 
 export default router

package/src/api/routes/system/tests/accounts.spec.ts

@@ -1,4 +1,4 @@
-import sdk from "../../../../sdk"
+import * as accounts from "../../../../sdk/accounts"
 import { TestConfiguration, structures } from "../../../../tests"
 import { v4 as uuid } from "uuid"
 
@@ -24,8 +24,8 @@ describe("accounts", () => {
 
         const response = await config.api.accounts.saveMetadata(account)
 
-        const id = sdk.accounts.formatAccountMetadataId(account.accountId)
-        const metadata = await sdk.accounts.getMetadata(id)
+        const id = accounts.metadata.formatAccountMetadataId(account.accountId)
+        const metadata = await accounts.metadata.getMetadata(id)
         expect(response).toStrictEqual(metadata)
       })
     })
@@ -37,7 +37,7 @@ describe("accounts", () => {
 
         await config.api.accounts.destroyMetadata(account.accountId)
 
-        const deleted = await sdk.accounts.getMetadata(account.accountId)
+        const deleted = await accounts.metadata.getMetadata(account.accountId)
         expect(deleted).toBe(undefined)
       })
 

package/src/api/routes/system/tests/migrations.spec.ts

@@ -30,7 +30,7 @@ describe("/api/system/migrations", () => {
         headers: {},
         status: 403,
       })
-      expect(res.text).toBe("Unauthorized - no public worker access")
+      expect(res.body).toEqual({ message: "Unauthorized", status: 403 })
       expect(migrateFn).toBeCalledTimes(0)
     })
 
@@ -47,7 +47,7 @@ describe("/api/system/migrations", () => {
         headers: {},
         status: 403,
       })
-      expect(res.text).toBe("Unauthorized - no public worker access")
+      expect(res.body).toEqual({ message: "Unauthorized", status: 403 })
     })
 
     it("returns definitions", async () => {

package/src/api/routes/system/tests/restore.spec.ts

@@ -25,12 +25,12 @@ describe("/api/system/restore", () => {
     })
 
     it("restores in self host", async () => {
-      config.modeSelf()
+      config.selfHosted()
       const res = await config.api.restore.restored()
       expect(res.body).toEqual({
         message: "System prepared after restore.",
       })
-      config.modeCloud()
+      config.cloudHosted()
     })
   })
 })

package/src/api/routes/system/tests/status.spec.ts

@@ -1,6 +1,6 @@
 import { TestConfiguration } from "../../../../tests"
-import { accounts } from "@budibase/backend-core"
-import { mocks } from "@budibase/backend-core/tests"
+import { accounts as _accounts } from "@budibase/backend-core"
+const accounts = jest.mocked(_accounts)
 
 describe("/api/system/status", () => {
   const config = new TestConfiguration()
@@ -19,7 +19,7 @@ describe("/api/system/status", () => {
 
   describe("GET /api/system/status", () => {
     it("returns status in self host", async () => {
-      config.modeSelf()
+      config.selfHosted()
       const res = await config.api.status.getStatus()
       expect(res.body).toEqual({
         health: {
@@ -27,7 +27,7 @@ describe("/api/system/status", () => {
         },
       })
       expect(accounts.getStatus).toBeCalledTimes(0)
-      config.modeCloud()
+      config.cloudHosted()
     })
 
     it("returns status in cloud", async () => {
@@ -37,7 +37,7 @@ describe("/api/system/status", () => {
         },
       }
 
-      mocks.accounts.getStatus.mockReturnValueOnce(value)
+      accounts.getStatus.mockReturnValueOnce(Promise.resolve(value))
 
       const res = await config.api.status.getStatus()
 

package/src/ddApm.ts

@@ -0,0 +1,7 @@
+import apm from "dd-trace"
+
+// enable APM if configured
+if (process.env.DD_APM_ENABLED) {
+  console.log("Starting dd-trace")
+  apm.init()
+}

package/src/elasticApm.ts

@@ -0,0 +1,10 @@
+import apm from "elastic-apm-node"
+
+// enable APM if configured
+if (process.env.ELASTIC_APM_ENABLED) {
+  console.log("Starting elastic-apm-node")
+  apm.start({
+    serviceName: process.env.SERVICE,
+    environment: process.env.BUDIBASE_ENVIRONMENT,
+  })
+}

package/src/environment.ts

@@ -26,6 +26,8 @@ function parseIntSafe(number: any) {
   }
 }
 
+const selfHosted = !!parseInt(process.env.SELF_HOSTED || "")
+
 const environment = {
   // auth
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
@@ -49,7 +51,7 @@ const environment = {
   CLUSTER_PORT: process.env.CLUSTER_PORT,
   // flags
   NODE_ENV: process.env.NODE_ENV,
-  SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED || ""),
+  SELF_HOSTED: selfHosted,
   LOG_LEVEL: process.env.LOG_LEVEL,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
@@ -65,6 +67,18 @@ const environment = {
   CHECKLIST_CACHE_TTL: parseIntSafe(process.env.CHECKLIST_CACHE_TTL) || 3600,
   SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD,
   ENCRYPTED_TEST_PUBLIC_API_KEY: process.env.ENCRYPTED_TEST_PUBLIC_API_KEY,
+  /**
+   * Mock the email service in use - links to ethereal hosted emails are logged instead.
+   */
+  ENABLE_EMAIL_TEST_MODE: process.env.ENABLE_EMAIL_TEST_MODE,
+  /**
+   * Enable to allow an admin user to login using a password.
+   * This can be useful to prevent lockout when configuring SSO.
+   * However, this should be turned OFF by default for security purposes.
+   */
+  ENABLE_SSO_MAINTENANCE_MODE: selfHosted
+    ? process.env.ENABLE_SSO_MAINTENANCE_MODE
+    : false,
   _set(key: any, value: any) {
     process.env[key] = value
     // @ts-ignore

package/src/index.ts

@@ -1,14 +1,13 @@
-// need to load environment first
-import env from "./environment"
+if (process.env.DD_APM_ENABLED) {
+  require("./ddApm")
+}
 
-// enable APM if configured
 if (process.env.ELASTIC_APM_ENABLED) {
-  const apm = require("elastic-apm-node").start({
-    serviceName: process.env.SERVICE,
-    environment: process.env.BUDIBASE_ENVIRONMENT,
-  })
+  require("./elasticApm")
 }
 
+// need to load environment first
+import env from "./environment"
 import { Scope } from "@sentry/node"
 import { Event } from "@sentry/types/dist/event"
 import Application from "koa"
@@ -26,6 +25,12 @@ const koaSession = require("koa-session")
 const logger = require("koa-pino-logger")
 import destroyable from "server-destroy"
 
+if (env.ENABLE_SSO_MAINTENANCE_MODE) {
+  console.warn(
+    "Warning: ENABLE_SSO_MAINTENANCE_MODE is set. It is recommended this flag is disabled if maintenance is not in progress"
+  )
+}
+
 // this will setup http and https proxies form env variables
 bootstrap()
 

package/src/middleware/tests/tenancy.spec.ts

@@ -24,7 +24,7 @@ describe("tenancy middleware", () => {
   })
 
   it("should get tenant id from header", async () => {
-    const tenantId = structures.uuid()
+    const tenantId = structures.tenant.id()
     const headers = {
       [constants.Header.TENANT_ID]: tenantId,
     }
@@ -35,7 +35,7 @@ describe("tenancy middleware", () => {
   })
 
   it("should get tenant id from query param", async () => {
-    const tenantId = structures.uuid()
+    const tenantId = structures.tenant.id()
     const res = await config.request.get(
       `/api/global/configs/checklist?tenantId=${tenantId}`
     )
@@ -43,7 +43,7 @@ describe("tenancy middleware", () => {
   })
 
   it("should get tenant id from subdomain", async () => {
-    const tenantId = structures.uuid()
+    const tenantId = structures.tenant.id()
     const headers = {
       host: `${tenantId}.localhost:10000`,
     }
@@ -67,7 +67,7 @@ describe("tenancy middleware", () => {
   it("should throw when no tenant id is found", async () => {
     const res = await config.request.get(`/api/global/configs/checklist`)
     expect(res.status).toBe(403)
-    expect(res.text).toBe("Tenant id not set")
+    expect(res.body).toEqual({ message: "Tenant id not set", status: 403 })
     expect(res.headers[constants.Header.TENANT_ID]).toBe(undefined)
   })
 })

package/src/migrations/functions/globalInfoSyncUsers.ts

@@ -1,5 +1,6 @@
 import { User } from "@budibase/types"
-import sdk from "../../sdk"
+import * as usersSdk from "../../sdk/users"
+import { platform } from "@budibase/backend-core"
 
 /**
  * Date:
@@ -9,11 +10,11 @@ import sdk from "../../sdk"
  * Re-sync the global-db users to the global-info db users
  */
 export const run = async (globalDb: any) => {
-  const users = (await sdk.users.allUsers()) as User[]
+  const users = (await usersSdk.allUsers()) as User[]
   const promises = []
   for (let user of users) {
     promises.push(
-      sdk.users.addTenant(user.tenantId, user._id as string, user.email)
+      platform.users.addUser(user.tenantId, user._id as string, user.email)
     )
   }
   await Promise.all(promises)

package/src/sdk/accounts/index.ts

@@ -1 +1,2 @@
-export * from "./accounts"
+export * as metadata from "./metadata"
+export { accounts as api } from "@budibase/backend-core"

package/src/sdk/accounts/{accounts.ts → metadata.ts}

@@ -2,7 +2,6 @@ import { AccountMetadata } from "@budibase/types"
 import {
   db,
   StaticDatabases,
-  HTTPError,
   DocumentType,
   SEPARATOR,
 } from "@budibase/backend-core"