@budibase/worker 2.8.29 → 2.8.31-alpha.0

package/jest.config.ts

@@ -1,9 +1,7 @@
 import { Config } from "@jest/types"
 import * as fs from "fs"
-const preset = require("ts-jest/jest-preset")
 
 const config: Config.InitialOptions = {
-  ...preset,
   preset: "@trendyol/jest-testcontainers",
   setupFiles: ["./src/tests/jestEnv.ts"],
   setupFilesAfterEnv: ["./src/tests/jestSetup.ts"],

package/nodemon.json

@@ -1,10 +1,13 @@
 {
-  "watch": ["src", "../backend-core", "../pro"],
-  "ext": "js,ts,json",
-  "ignore": [
-    "src/**/*.spec.ts",
-    "src/**/*.spec.js",
-    "../backend-core/dist/**/*"
+  "watch": [
+    "src",
+    "../backend-core",
+    "../pro",
+    "../types",
+    "../shared-core",
+    "../string-templates"
   ],
+  "ext": "js,ts,json",
+  "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../*/dist/**/*"],
   "exec": "yarn build && node dist/index.js"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "2.8.29",
+  "version": "2.8.31-alpha.0",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -14,7 +14,7 @@
   "scripts": {
     "prebuild": "rimraf dist/",
     "build": "node ../../scripts/build.js",
-    "check:types": "tsc -p tsconfig.build.json --noEmit",
+    "check:types": "tsc -p tsconfig.json --noEmit --paths null",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "run:docker": "node dist/index.js",
     "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js",
@@ -38,10 +38,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "2.8.29",
-    "@budibase/pro": "2.8.29",
-    "@budibase/string-templates": "2.8.29",
-    "@budibase/types": "2.8.29",
+    "@budibase/backend-core": "2.8.31-alpha.0",
+    "@budibase/pro": "2.8.31-alpha.0",
+    "@budibase/string-templates": "2.8.31-alpha.0",
+    "@budibase/types": "2.8.31-alpha.0",
     "@koa/router": "8.0.8",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "0.3.2",
@@ -74,10 +74,10 @@
     "server-destroy": "1.0.1"
   },
   "devDependencies": {
-    "@swc/core": "^1.3.25",
-    "@swc/jest": "^0.2.24",
-    "@trendyol/jest-testcontainers": "^2.1.1",
-    "@types/jest": "28.1.1",
+    "@swc/core": "1.3.71",
+    "@swc/jest": "0.2.27",
+    "@trendyol/jest-testcontainers": "2.1.1",
+    "@types/jest": "29.5.3",
     "@types/jsonwebtoken": "8.5.1",
     "@types/koa": "2.13.4",
     "@types/koa__router": "8.0.8",
@@ -91,32 +91,17 @@
     "@typescript-eslint/parser": "5.45.0",
     "copyfiles": "2.4.1",
     "eslint": "6.8.0",
-    "jest": "28.1.1",
+    "jest": "29.6.2",
     "lodash": "4.17.21",
     "nodemon": "2.0.15",
     "pouchdb-adapter-memory": "7.2.2",
     "rimraf": "3.0.2",
     "supertest": "6.2.2",
     "timekeeper": "2.2.0",
-    "ts-jest": "28.0.4",
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
     "typescript": "4.7.3",
     "update-dotenv": "1.1.1"
   },
-  "nx": {
-    "targets": {
-      "dev:builder": {
-        "dependsOn": [
-          {
-            "projects": [
-              "@budibase/backend-core"
-            ],
-            "target": "build"
-          }
-        ]
-      }
-    }
-  },
-  "gitHead": "c5b487659606add604ff3211f422c26c4924bc35"
+  "gitHead": "38e2c52eb34c0fae42da6e7f346996c379cd281f"
 }

package/scripts/test.sh

@@ -4,8 +4,8 @@ set -e
 if [[ -n $CI ]]
 then
   # --runInBand performs better in ci where resources are limited
-  echo "jest --coverage --runInBand --forceExit"
-  jest --coverage --runInBand --forceExit
+  echo "jest --coverage --runInBand --forceExit --bail"
+  jest --coverage --runInBand --forceExit --bail
 else
   # --maxWorkers performs better in development
   echo "jest --coverage --maxWorkers=2 --forceExit"

package/src/api/controllers/global/auth.ts

@@ -55,8 +55,8 @@ async function passportCallback(
 export const login = async (ctx: Ctx<LoginRequest>, next: any) => {
   const email = ctx.request.body.username
 
-  const user = await userSdk.getUserByEmail(email)
-  if (user && (await userSdk.isPreventPasswordActions(user))) {
+  const user = await userSdk.db.getUserByEmail(email)
+  if (user && (await userSdk.db.isPreventPasswordActions(user))) {
     ctx.throw(403, "Invalid credentials")
   }
 
@@ -174,7 +174,7 @@ export const googlePreAuth = async (ctx: any, next: any) => {
   const strategy = await google.strategyFactory(
     config,
     callbackUrl,
-    userSdk.save
+    userSdk.db.save
   )
 
   return passport.authenticate(strategy, {
@@ -193,7 +193,7 @@ export const googleCallback = async (ctx: any, next: any) => {
   const strategy = await google.strategyFactory(
     config,
     callbackUrl,
-    userSdk.save
+    userSdk.db.save
   )
 
   return passport.authenticate(
@@ -228,7 +228,7 @@ export const oidcStrategyFactory = async (ctx: any, configId: any) => {
 
   //Remote Config
   const enrichedConfig = await oidc.fetchStrategyConfig(config, callbackUrl)
-  return oidc.strategyFactory(enrichedConfig, userSdk.save)
+  return oidc.strategyFactory(enrichedConfig, userSdk.db.save)
 }
 
 /**

package/src/api/controllers/global/configs.ts

@@ -507,17 +507,17 @@ export async function configChecklist(ctx: Ctx) {
           smtp: {
             checked: !!smtpConfig,
             label: "Set up email",
-            link: "/builder/portal/manage/email",
+            link: "/builder/portal/settings/email",
           },
           adminUser: {
             checked: userExists,
             label: "Create your first user",
-            link: "/builder/portal/manage/users",
+            link: "/builder/portal/users/users",
           },
           sso: {
             checked: !!googleConfig || !!oidcConfig,
             label: "Set up single sign-on",
-            link: "/builder/portal/manage/auth",
+            link: "/builder/portal/settings/auth",
           },
         }
       }

package/src/api/controllers/global/email.ts

@@ -1,6 +1,6 @@
 import { sendEmail as sendEmailFn } from "../../../utilities/email"
 import { tenancy } from "@budibase/backend-core"
-import { BBContext } from "@budibase/types"
+import { BBContext, User } from "@budibase/types"
 
 export async function sendEmail(ctx: BBContext) {
   let {
@@ -16,10 +16,10 @@ export async function sendEmail(ctx: BBContext) {
     automation,
     invite,
   } = ctx.request.body
-  let user
+  let user: any
   if (userId) {
     const db = tenancy.getGlobalDB()
-    user = await db.get(userId)
+    user = await db.get<User>(userId)
   }
   const response = await sendEmailFn(email, purpose, {
     workspaceId,

package/src/api/controllers/global/roles.ts

@@ -5,10 +5,10 @@ import {
   cache,
   tenancy,
 } from "@budibase/backend-core"
-import { BBContext, App } from "@budibase/types"
-import { allUsers } from "../../../sdk/users"
+import sdk from "../../../sdk"
+import { Ctx, App } from "@budibase/types"
 
-export async function fetch(ctx: BBContext) {
+export async function fetch(ctx: Ctx) {
   const tenantId = ctx.user!.tenantId
   // always use the dev apps as they'll be most up to date (true)
   const apps = (await dbCore.getAllApps({ tenantId, all: true })) as App[]
@@ -31,11 +31,11 @@ export async function fetch(ctx: BBContext) {
   ctx.body = response
 }
 
-export async function find(ctx: BBContext) {
+export async function find(ctx: Ctx) {
   const appId = ctx.params.appId
   await context.doInAppContext(dbCore.getDevAppID(appId), async () => {
     const db = context.getAppDB()
-    const app = await db.get(dbCore.DocumentType.APP_METADATA)
+    const app = await db.get<App>(dbCore.DocumentType.APP_METADATA)
     ctx.body = {
       roles: await roles.getAllRoles(),
       name: app.name,
@@ -45,10 +45,10 @@ export async function find(ctx: BBContext) {
   })
 }
 
-export async function removeAppRole(ctx: BBContext) {
+export async function removeAppRole(ctx: Ctx) {
   const { appId } = ctx.params
   const db = tenancy.getGlobalDB()
-  const users = await allUsers()
+  const users = await sdk.users.db.allUsers()
   const bulk = []
   const cacheInvalidations = []
   for (let user of users) {

package/src/api/controllers/global/self.ts

@@ -44,7 +44,7 @@ export async function generateAPIKey(ctx: any) {
   const id = dbCore.generateDevInfoID(userId)
   let devInfo
   try {
-    devInfo = await db.get(id)
+    devInfo = await db.get<any>(id)
   } catch (err) {
     devInfo = { _id: id, userId }
   }
@@ -91,7 +91,7 @@ export async function getSelf(ctx: any) {
   }
 
   // get the main body of the user
-  const user = await userSdk.getUser(userId)
+  const user = await userSdk.db.getUser(userId)
   ctx.body = await groups.enrichUserRolesFromGroups(user)
 
   // add the feature flags for this tenant
@@ -106,12 +106,12 @@ export async function updateSelf(
 ) {
   const update = ctx.request.body
 
-  let user = await userSdk.getUser(ctx.user._id!)
+  let user = await userSdk.db.getUser(ctx.user._id!)
   user = {
     ...user,
     ...update,
   }
-  user = await userSdk.save(user, { requirePassword: false })
+  user = await userSdk.db.save(user, { requirePassword: false })
 
   if (update.password) {
     // Log all other sessions out apart from the current one

package/src/api/controllers/global/users.ts

@@ -41,7 +41,7 @@ export const save = async (ctx: UserCtx<User, SaveUserResponse>) => {
     const currentUserId = ctx.user?._id
     const requestUser = ctx.request.body
 
-    const user = await userSdk.save(requestUser, { currentUserId })
+    const user = await userSdk.db.save(requestUser, { currentUserId })
 
     ctx.body = {
       _id: user._id!,
@@ -57,7 +57,7 @@ const bulkDelete = async (userIds: string[], currentUserId: string) => {
   if (userIds?.indexOf(currentUserId) !== -1) {
     throw new Error("Unable to delete self.")
   }
-  return await userSdk.bulkDelete(userIds)
+  return await userSdk.db.bulkDelete(userIds)
 }
 
 const bulkCreate = async (users: User[], groupIds: string[]) => {
@@ -66,7 +66,7 @@ const bulkCreate = async (users: User[], groupIds: string[]) => {
       "Max limit for upload is 1000 users. Please reduce file size and try again."
     )
   }
-  return await userSdk.bulkCreate(users, groupIds)
+  return await userSdk.db.bulkCreate(users, groupIds)
 }
 
 export const bulkUpdate = async (
@@ -141,7 +141,7 @@ export const adminUser = async (
       // always bust checklist beforehand, if an error occurs but can proceed, don't get
       // stuck in a cycle
       await cache.bustCache(cache.CacheKey.CHECKLIST)
-      const finalUser = await userSdk.save(user, {
+      const finalUser = await userSdk.db.save(user, {
         hashPassword,
         requirePassword,
       })
@@ -167,7 +167,7 @@ export const adminUser = async (
 export const countByApp = async (ctx: any) => {
   const appId = ctx.params.appId
   try {
-    ctx.body = await userSdk.countUsersByApp(appId)
+    ctx.body = await userSdk.db.countUsersByApp(appId)
   } catch (err: any) {
     ctx.throw(err.status || 400, err)
   }
@@ -179,7 +179,7 @@ export const destroy = async (ctx: any) => {
     ctx.throw(400, "Unable to delete self.")
   }
 
-  await userSdk.destroy(id)
+  await userSdk.db.destroy(id)
 
   ctx.body = {
     message: `User ${id} deleted.`,
@@ -188,7 +188,7 @@ export const destroy = async (ctx: any) => {
 
 export const getAppUsers = async (ctx: Ctx<SearchUsersRequest>) => {
   const body = ctx.request.body
-  const users = await userSdk.getUsersByAppAccess(body?.appId)
+  const users = await userSdk.db.getUsersByAppAccess(body?.appId)
 
   ctx.body = { data: users }
 }
@@ -212,7 +212,7 @@ export const search = async (ctx: Ctx<SearchUsersRequest>) => {
 
 // called internally by app server user fetch
 export const fetch = async (ctx: any) => {
-  const all = await userSdk.allUsers()
+  const all = await userSdk.db.allUsers()
   // user hashed password shouldn't ever be returned
   for (let user of all) {
     if (user) {
@@ -224,12 +224,12 @@ export const fetch = async (ctx: any) => {
 
 // called internally by app server user find
 export const find = async (ctx: any) => {
-  ctx.body = await userSdk.getUser(ctx.params.id)
+  ctx.body = await userSdk.db.getUser(ctx.params.id)
 }
 
 export const tenantUserLookup = async (ctx: any) => {
   const id = ctx.params.id
-  const user = await userSdk.getPlatformUser(id)
+  const user = await userSdk.core.getPlatformUser(id)
   if (user) {
     ctx.body = user
   } else {
@@ -252,7 +252,7 @@ export const onboardUsers = async (ctx: Ctx<InviteUsersRequest>) => {
     // @ts-ignore
     const { users, groups, roles } = request.create
     const assignUsers = users.map((user: User) => (user.roles = roles))
-    onboardingResponse = await userSdk.bulkCreate(assignUsers, groups)
+    onboardingResponse = await userSdk.db.bulkCreate(assignUsers, groups)
     ctx.body = onboardingResponse
   } else if (emailConfigured) {
     onboardingResponse = await inviteMultiple(ctx)
@@ -277,7 +277,7 @@ export const onboardUsers = async (ctx: Ctx<InviteUsersRequest>) => {
         tenantId: tenancy.getTenantId(),
       }
     })
-    let bulkCreateReponse = await userSdk.bulkCreate(users, [])
+    let bulkCreateReponse = await userSdk.db.bulkCreate(users, [])
 
     // Apply temporary credentials
     let createWithCredentials = {
@@ -410,9 +410,9 @@ export const inviteAccept = async (
         ...info,
       }
 
-      const saved = await userSdk.save(request)
+      const saved = await userSdk.db.save(request)
       const db = tenancy.getGlobalDB()
-      const user = await db.get(saved._id)
+      const user = await db.get<User>(saved._id)
       await events.user.inviteAccepted(user)
       return saved
     })

package/src/api/controllers/system/logs.ts

@@ -0,0 +1,13 @@
+import { UserCtx } from "@budibase/types"
+import { installation, logging } from "@budibase/backend-core"
+
+export async function getLogs(ctx: UserCtx) {
+  const logReadStream = logging.system.getLogReadStream()
+
+  const { installId } = await installation.getInstall()
+
+  const fileName = `${installId}-${Date.now()}.log`
+
+  ctx.set("content-disposition", `attachment; filename=${fileName}`)
+  ctx.body = logReadStream
+}

package/src/api/routes/global/tests/appBuilder.spec.ts

@@ -0,0 +1,62 @@
+import { TestConfiguration, structures } from "../../../../tests"
+import { mocks } from "@budibase/backend-core/tests"
+import { User } from "@budibase/types"
+
+const MOCK_APP_ID = "app_a"
+
+describe("/api/global/users/:userId/app/builder", () => {
+  const config = new TestConfiguration()
+
+  beforeAll(async () => {
+    await config.beforeAll()
+  })
+
+  afterAll(async () => {
+    await config.afterAll()
+  })
+
+  async function newUser() {
+    const base = structures.users.user()
+    return await config.createUser(base)
+  }
+
+  async function getUser(userId: string) {
+    const response = await config.api.users.getUser(userId)
+    return response.body as User
+  }
+
+  describe("Confirm pro license", () => {
+    it("should 400 with licensing", async () => {
+      const user = await newUser()
+      const resp = await config.api.users.grantBuilderToApp(
+        user._id!,
+        MOCK_APP_ID,
+        400
+      )
+      expect(resp.body.message).toContain("Feature not enabled")
+    })
+  })
+
+  describe("PATCH /api/global/users/:userId/app/:appId/builder", () => {
+    it("should be able to grant a user access to a particular app", async () => {
+      mocks.licenses.useAppBuilders()
+      const user = await newUser()
+      await config.api.users.grantBuilderToApp(user._id!, MOCK_APP_ID)
+      const updated = await getUser(user._id!)
+      expect(updated.builder?.apps![0]).toBe(MOCK_APP_ID)
+    })
+  })
+
+  describe("DELETE /api/global/users/:userId/app/:appId/builder", () => {
+    it("should allow revoking access", async () => {
+      mocks.licenses.useAppBuilders()
+      const user = await newUser()
+      await config.api.users.grantBuilderToApp(user._id!, MOCK_APP_ID)
+      let updated = await getUser(user._id!)
+      expect(updated.builder?.apps![0]).toBe(MOCK_APP_ID)
+      await config.api.users.revokeBuilderFromApp(user._id!, MOCK_APP_ID)
+      updated = await getUser(user._id!)
+      expect(updated.builder?.apps!.length).toBe(0)
+    })
+  })
+})

package/src/api/routes/global/tests/auth.spec.ts

@@ -41,7 +41,7 @@ describe("/api/global/auth", () => {
 
   async function createSSOUser() {
     return config.doInTenant(async () => {
-      return userSdk.save(structures.users.ssoUser(), {
+      return userSdk.db.save(structures.users.ssoUser(), {
         requirePassword: false,
       })
     })
@@ -206,7 +206,7 @@ describe("/api/global/auth", () => {
         const newPassword = "newpassword"
         const res = await config.api.auth.updatePassword(code!, newPassword)
 
-        user = await config.getUser(user.email)
+        user = (await config.getUser(user.email))!
         delete user.password
 
         expect(res.body).toEqual({ message: "password reset successfully." })
@@ -245,7 +245,7 @@ describe("/api/global/auth", () => {
             const ssoUser = user as SSOUser
             ssoUser.providerType = structures.sso.providerType()
             delete ssoUser.password
-            await config.doInTenant(() => userSdk.save(ssoUser))
+            await config.doInTenant(() => userSdk.db.save(ssoUser))
 
             await testSSOUser(code!)
           })

package/src/api/routes/global/tests/scim.spec.ts

@@ -314,7 +314,7 @@ describe("scim", () => {
 
           const user = await config.getUser(email)
           expect(user).toBeDefined()
-          expect(user.email).toEqual(email)
+          expect(user!.email).toEqual(email)
         })
 
         it("if multiple emails are provided, the first primary one is used as email", async () => {
@@ -345,7 +345,7 @@ describe("scim", () => {
 
           const user = await config.getUser(email)
           expect(user).toBeDefined()
-          expect(user.email).toEqual(email)
+          expect(user!.email).toEqual(email)
         })
 
         it("if no email is provided and the user name is not an email, an exception is thrown", async () => {

package/src/api/routes/global/tests/self.spec.ts

@@ -36,7 +36,7 @@ describe("/api/global/self", () => {
         })
         .expect(200)
 
-      const dbUser = await config.getUser(user.email)
+      const dbUser = (await config.getUser(user.email))!
 
       user._rev = dbUser._rev
       user.dayPassRecordedAt = mocks.date.MOCK_DATE.toISOString()
@@ -58,7 +58,7 @@ describe("/api/global/self", () => {
       })
       .expect(200)
 
-    const dbUser = await config.getUser(user.email)
+    const dbUser = (await config.getUser(user.email))!
 
     user._rev = dbUser._rev
     user.dayPassRecordedAt = mocks.date.MOCK_DATE.toISOString()

package/src/api/routes/global/tests/users.spec.ts

@@ -66,7 +66,7 @@ describe("/api/global/users", () => {
       expect(res.body._id).toBeDefined()
       const user = await config.getUser(email)
       expect(user).toBeDefined()
-      expect(user._id).toEqual(res.body._id)
+      expect(user!._id).toEqual(res.body._id)
       expect(events.user.inviteAccepted).toBeCalledTimes(1)
       expect(events.user.inviteAccepted).toBeCalledWith(user)
     })
@@ -480,7 +480,7 @@ describe("/api/global/users", () => {
       function createSSOUser() {
         return config.doInTenant(() => {
           const user = structures.users.ssoUser()
-          return userSdk.save(user, { requirePassword: false })
+          return userSdk.db.save(user, { requirePassword: false })
         })
       }
 

package/src/api/routes/index.ts

@@ -16,10 +16,14 @@ import licenseRoutes from "./global/license"
 import migrationRoutes from "./system/migrations"
 import accountRoutes from "./system/accounts"
 import restoreRoutes from "./system/restore"
+import systemLogRoutes from "./system/logs"
+
+import env from "../../environment"
 
 export const routes: Router[] = [
   configRoutes,
   userRoutes,
+  pro.users,
   workspaceRoutes,
   authRoutes,
   templateRoutes,
@@ -38,3 +42,7 @@ export const routes: Router[] = [
   eventRoutes,
   pro.scim,
 ]
+
+if (env.SELF_HOSTED) {
+  routes.push(systemLogRoutes)
+}

package/src/api/routes/system/logs.ts

@@ -0,0 +1,9 @@
+import Router from "@koa/router"
+import { middleware } from "@budibase/backend-core"
+import * as controller from "../../controllers/system/logs"
+
+const router: Router = new Router()
+
+router.get("/api/system/logs", middleware.adminOnly, controller.getLogs)
+
+export default router

package/src/api/routes/system/tests/status.spec.ts

@@ -1,3 +1,4 @@
+import { HealthStatusResponse } from "@budibase/types"
 import { TestConfiguration } from "../../../../tests"
 import { accounts as _accounts } from "@budibase/backend-core"
 const accounts = jest.mocked(_accounts)
@@ -31,13 +32,15 @@ describe("/api/system/status", () => {
     })
 
     it("returns status in cloud", async () => {
-      const value = {
-        health: {
-          passing: false,
+      const value: HealthStatusResponse = {
+        passing: false,
+        checks: {
+          login: false,
+          search: false,
         },
       }
 
-      accounts.getStatus.mockReturnValueOnce(Promise.resolve(value))
+      accounts.getStatus.mockResolvedValue(value)
 
       const res = await config.api.status.getStatus()
 

package/src/environment.ts

@@ -1,4 +1,8 @@
-const { join } = require("path")
+import { env as coreEnv } from "@budibase/backend-core"
+import { ServiceType } from "@budibase/types"
+import { join } from "path"
+
+coreEnv._set("SERVICE_TYPE", ServiceType.WORKER)
 
 function isDev() {
   return process.env.NODE_ENV !== "production"

package/src/initPro.ts

@@ -2,12 +2,5 @@ import { sdk as proSdk } from "@budibase/pro"
 import * as userSdk from "./sdk/users"
 
 export const initPro = async () => {
-  await proSdk.init({
-    scimUserServiceConfig: {
-      functions: {
-        saveUser: userSdk.save,
-        removeUser: (id: string) => userSdk.destroy(id),
-      },
-    },
-  })
+  await proSdk.init({})
 }

package/src/migrations/functions/globalInfoSyncUsers.ts

@@ -10,7 +10,7 @@ import { platform } from "@budibase/backend-core"
  * Re-sync the global-db users to the global-info db users
  */
 export const run = async (globalDb: any) => {
-  const users = (await usersSdk.allUsers()) as User[]
+  const users = (await usersSdk.db.allUsers()) as User[]
   const promises = []
   for (let user of users) {
     promises.push(