@budibase/worker 2.5.6-alpha.9 → 2.5.7

package/jest.config.ts

@@ -20,11 +20,11 @@ const config: Config.InitialOptions = {
 }
 
 // add pro sources if they exist
-if (fs.existsSync("../pro/packages")) {
-  config.moduleNameMapper!["@budibase/pro/(.*)"] =
-    "<rootDir>/../pro/packages/pro/$1"
-  config.moduleNameMapper!["@budibase/pro"] =
-    "<rootDir>/../pro/packages/pro/src"
+if (fs.existsSync("../../../budibase-pro")) {
+  config.moduleNameMapper["@budibase/pro/(.*)"] =
+    "<rootDir>/../../../budibase-pro/packages/pro/$1"
+  config.moduleNameMapper["@budibase/pro"] =
+    "<rootDir>/../../../budibase-pro/packages/pro/src"
 }
 
 export default config

package/nodemon.json

@@ -1,10 +1,6 @@
 {
-  "watch": ["src", "../backend-core", "../pro/packages/pro"],
+  "watch": ["src", "../backend-core", "../../../budibase-pro/packages/pro"],
   "ext": "js,ts,json",
-  "ignore": [
-    "src/**/*.spec.ts",
-    "src/**/*.spec.js",
-    "../backend-core/dist/**/*"
-  ],
+  "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../backend-core/dist/**/*"],
   "exec": "ts-node src/index.ts"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "2.5.6-alpha.9",
+  "version": "2.5.7",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -22,7 +22,6 @@
     "build:docker": "docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
     "dev:stack:init": "node ./scripts/dev/manage.js init",
     "dev:builder": "npm run dev:stack:init && nodemon",
-    "dev:built": "yarn run dev:stack:init && yarn run run:docker",
     "test": "bash scripts/test.sh",
     "test:watch": "jest --watch",
     "env:multi:enable": "node scripts/multiTenancy.js enable",
@@ -37,10 +36,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "2.5.6-alpha.9",
-    "@budibase/pro": "2.5.6-alpha.6",
-    "@budibase/string-templates": "2.5.6-alpha.9",
-    "@budibase/types": "2.5.6-alpha.9",
+    "@budibase/backend-core": "^2.5.7",
+    "@budibase/pro": "2.5.6",
+    "@budibase/string-templates": "^2.5.7",
+    "@budibase/types": "^2.5.7",
     "@koa/router": "8.0.8",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "0.3.2",
@@ -57,6 +56,7 @@
     "koa-body": "4.2.0",
     "koa-compress": "4.0.1",
     "koa-passport": "4.1.4",
+    "koa-pino-logger": "3.0.0",
     "koa-send": "5.0.1",
     "koa-session": "5.13.1",
     "koa-static": "5.0.0",
@@ -66,6 +66,7 @@
     "passport-google-oauth": "2.0.0",
     "passport-jwt": "4.0.0",
     "passport-local": "1.0.0",
+    "pino-pretty": "4.8.0",
     "pouchdb": "7.3.0",
     "pouchdb-all-dbs": "1.1.1",
     "server-destroy": "1.0.1"
@@ -78,7 +79,6 @@
     "@types/jsonwebtoken": "8.5.1",
     "@types/koa": "2.13.4",
     "@types/koa__router": "8.0.8",
-    "@types/lodash": "^4.14.191",
     "@types/node": "14.18.20",
     "@types/node-fetch": "2.6.1",
     "@types/pouchdb": "6.4.0",
@@ -89,7 +89,6 @@
     "copyfiles": "2.4.1",
     "eslint": "6.8.0",
     "jest": "28.1.1",
-    "lodash": "4.17.21",
     "nodemon": "2.0.15",
     "pouchdb-adapter-memory": "7.2.2",
     "prettier": "2.3.1",
@@ -102,5 +101,5 @@
     "typescript": "4.7.3",
     "update-dotenv": "1.1.1"
   },
-  "gitHead": "560204d82570bd22b987e6e35f8e659e9c047425"
+  "gitHead": "6c37b6a6c69a1da43a5c0930419e743cc18b5b42"
 }

package/scripts/dev/manage.js

@@ -30,7 +30,6 @@ async function init() {
       DEPLOYMENT_ENVIRONMENT: "development",
       TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
       ENABLE_EMAIL_TEST_MODE: 1,
-      HTTP_LOGGING: 0,
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {

package/src/api/controllers/global/users.ts

@@ -177,7 +177,7 @@ export const destroy = async (ctx: any) => {
     ctx.throw(400, "Unable to delete self.")
   }
 
-  await userSdk.destroy(id)
+  await userSdk.destroy(id, ctx.user)
 
   ctx.body = {
     message: `User ${id} deleted.`,
@@ -197,7 +197,7 @@ export const search = async (ctx: any) => {
   if (body.paginated === false) {
     await getAppUsers(ctx)
   } else {
-    const paginated = await userSdk.core.paginatedUsers(body)
+    const paginated = await userSdk.paginatedUsers(body)
     // user hashed password shouldn't ever be returned
     for (let user of paginated.data) {
       if (user) {

package/src/api/index.ts

@@ -1,6 +1,6 @@
 import Router from "@koa/router"
 const compress = require("koa-compress")
-import zlib from "zlib"
+const zlib = require("zlib")
 import { routes } from "./routes"
 import { middleware as pro } from "@budibase/pro"
 import { auth, middleware } from "@budibase/backend-core"

package/src/api/routes/global/configs.ts

@@ -58,13 +58,6 @@ function oidcValidation() {
   }).unknown(true)
 }
 
-function scimValidation() {
-  // prettier-ignore
-  return Joi.object({
-    enabled: Joi.boolean().required(),
-  }).unknown(true)
-}
-
 function buildConfigSaveValidation() {
   // prettier-ignore
   return auth.joiValidator.body(Joi.object({
@@ -81,8 +74,7 @@ function buildConfigSaveValidation() {
           { is: ConfigType.SETTINGS, then: settingValidation() },
           { is: ConfigType.ACCOUNT, then: Joi.object().unknown(true) },
           { is: ConfigType.GOOGLE, then: googleValidation() },
-          { is: ConfigType.OIDC, then: oidcValidation() },
-          { is: ConfigType.SCIM, then: scimValidation() }
+          { is: ConfigType.OIDC, then: oidcValidation() }
         ],
       }),
   }).required().unknown(true),

package/src/api/routes/global/tests/users.spec.ts

@@ -48,7 +48,7 @@ describe("/api/global/users", () => {
         400
       )
 
-      expect(res.body.message).toBe(`Unavailable`)
+      expect(res.body.message).toBe("Unavailable")
       expect(sendMailMock).toHaveBeenCalledTimes(0)
       expect(code).toBeUndefined()
       expect(events.user.invited).toBeCalledTimes(0)
@@ -225,9 +225,7 @@ describe("/api/global/users", () => {
 
       const response = await config.api.users.saveUser(user, 400)
 
-      expect(response.body.message).toBe(
-        `Email already in use: '${user.email}'`
-      )
+      expect(response.body.message).toBe(`Unavailable`)
       expect(events.user.created).toBeCalledTimes(0)
     })
 
@@ -239,9 +237,7 @@ describe("/api/global/users", () => {
         delete user._id
         const response = await config.api.users.saveUser(user, 400)
 
-        expect(response.body.message).toBe(
-          `Email already in use: '${user.email}'`
-        )
+        expect(response.body.message).toBe(`Unavailable`)
         expect(events.user.created).toBeCalledTimes(0)
       })
     })
@@ -253,9 +249,7 @@ describe("/api/global/users", () => {
 
       const response = await config.api.users.saveUser(user, 400)
 
-      expect(response.body.message).toBe(
-        `Email already in use: '${user.email}'`
-      )
+      expect(response.body.message).toBe(`Unavailable`)
       expect(events.user.created).toBeCalledTimes(0)
     })
 

package/src/api/routes/index.ts

@@ -1,5 +1,5 @@
 import Router from "@koa/router"
-import { api as pro } from "@budibase/pro"
+import { api } from "@budibase/pro"
 import userRoutes from "./global/users"
 import configRoutes from "./global/configs"
 import workspaceRoutes from "./global/workspaces"
@@ -17,6 +17,9 @@ import migrationRoutes from "./system/migrations"
 import accountRoutes from "./system/accounts"
 import restoreRoutes from "./system/restore"
 
+let userGroupRoutes = api.groups
+let auditLogRoutes = api.auditLogs
+
 export const routes: Router[] = [
   configRoutes,
   userRoutes,
@@ -30,11 +33,10 @@ export const routes: Router[] = [
   statusRoutes,
   selfRoutes,
   licenseRoutes,
-  pro.groups,
-  pro.auditLogs,
+  userGroupRoutes,
+  auditLogRoutes,
   migrationRoutes,
   accountRoutes,
   restoreRoutes,
   eventRoutes,
-  pro.scim,
 ]

package/src/elasticApm.ts

@@ -0,0 +1,10 @@
+import apm from "elastic-apm-node"
+
+// enable APM if configured
+if (process.env.ELASTIC_APM_ENABLED) {
+  console.log("Starting elastic-apm-node")
+  apm.start({
+    serviceName: process.env.SERVICE,
+    environment: process.env.BUDIBASE_ENVIRONMENT,
+  })
+}

package/src/environment.ts

@@ -47,6 +47,7 @@ const environment = {
   // flags
   NODE_ENV: process.env.NODE_ENV,
   SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED || ""),
+  LOG_LEVEL: process.env.LOG_LEVEL,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
   SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED,

package/src/index.ts

@@ -2,6 +2,10 @@ if (process.env.DD_APM_ENABLED) {
   require("./ddApm")
 }
 
+if (process.env.ELASTIC_APM_ENABLED) {
+  require("./elasticApm")
+}
+
 // need to load environment first
 import env from "./environment"
 import { Scope } from "@sentry/node"
@@ -27,11 +31,10 @@ import api from "./api"
 import * as redis from "./utilities/redis"
 const Sentry = require("@sentry/node")
 const koaSession = require("koa-session")
+const logger = require("koa-pino-logger")
 const { userAgent } = require("koa-useragent")
 
 import destroyable from "server-destroy"
-import { initPro } from "./initPro"
-import { handleScimBody } from "./middleware/handleScimBody"
 
 // configure events to use the pro audit log write
 // can't integrate directly into backend-core due to cyclic issues
@@ -51,12 +54,10 @@ const app: Application = new Koa()
 app.keys = ["secret", "key"]
 
 // set up top level koa middleware
-app.use(handleScimBody)
 app.use(koaBody({ multipart: true }))
-
 app.use(koaSession(app))
-app.use(middleware.correlation)
-app.use(middleware.pino)
+app.use(middleware.logging)
+app.use(logger(logging.pinoSettings()))
 app.use(userAgent)
 
 // authentication
@@ -107,7 +108,6 @@ const shutdown = () => {
 
 export default server.listen(parseInt(env.PORT || "4002"), async () => {
   console.log(`Worker running on ${JSON.stringify(server.address())}`)
-  await initPro()
   await redis.init()
 })
 

package/src/sdk/users/users.ts

@@ -1,4 +1,5 @@
 import env from "../../environment"
+import * as apps from "../../utilities/appService"
 import * as eventHelpers from "./events"
 import {
   accounts,
@@ -14,8 +15,6 @@ import {
   utils,
   ViewName,
   env as coreEnv,
-  context,
-  EmailUnavailableError,
 } from "@budibase/backend-core"
 import {
   AccountMetadata,
@@ -29,6 +28,7 @@ import {
   PlatformUser,
   PlatformUserByEmail,
   RowResponse,
+  SearchUsersRequest,
   User,
   SaveUserOpts,
   Account,
@@ -38,6 +38,8 @@ import { EmailTemplatePurpose } from "../../constants"
 import * as pro from "@budibase/pro"
 import * as accountSdk from "../accounts"
 
+const PAGE_LIMIT = 8
+
 export const allUsers = async () => {
   const db = tenancy.getGlobalDB()
   const response = await db.allDocs(
@@ -67,6 +69,43 @@ export const getUsersByAppAccess = async (appId?: string) => {
   return response
 }
 
+export const paginatedUsers = async ({
+  page,
+  email,
+  appId,
+}: SearchUsersRequest = {}) => {
+  const db = tenancy.getGlobalDB()
+  // get one extra document, to have the next page
+  const opts: any = {
+    include_docs: true,
+    limit: PAGE_LIMIT + 1,
+  }
+  // add a startkey if the page was specified (anchor)
+  if (page) {
+    opts.startkey = page
+  }
+  // property specifies what to use for the page/anchor
+  let userList,
+    property = "_id",
+    getKey
+  if (appId) {
+    userList = await usersCore.searchGlobalUsersByApp(appId, opts)
+    getKey = (doc: any) => usersCore.getGlobalUserByAppPage(appId, doc)
+  } else if (email) {
+    userList = await usersCore.searchGlobalUsersByEmail(email, opts)
+    property = "email"
+  } else {
+    // no search, query allDocs
+    const response = await db.allDocs(dbUtils.getGlobalUserParams(null, opts))
+    userList = response.rows.map((row: any) => row.doc)
+  }
+  return dbUtils.pagination(userList, PAGE_LIMIT, {
+    paginate: true,
+    property,
+    getKey,
+  })
+}
+
 export async function getUserByEmail(email: string) {
   return usersCore.getGlobalUserByEmail(email)
 }
@@ -159,7 +198,7 @@ const validateUniqueUser = async (email: string, tenantId: string) => {
   if (env.MULTI_TENANCY) {
     const tenantUser = await getPlatformUser(email)
     if (tenantUser != null && tenantUser.tenantId !== tenantId) {
-      throw new EmailUnavailableError(email)
+      throw `Unavailable`
     }
   }
 
@@ -167,7 +206,7 @@ const validateUniqueUser = async (email: string, tenantId: string) => {
   if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
     const account = await accounts.getAccount(email)
     if (account && account.verified && account.tenantId !== tenantId) {
-      throw new EmailUnavailableError(email)
+      throw `Unavailable`
     }
   }
 }
@@ -238,7 +277,7 @@ export const save = async (
     // no id was specified - load from email instead
     dbUser = await usersCore.getGlobalUserByEmail(email)
     if (dbUser && dbUser._id !== _id) {
-      throw new EmailUnavailableError(email)
+      throw `Unavailable`
     }
   }
 
@@ -278,6 +317,9 @@ export const save = async (
     await platform.users.addUser(tenantId, builtUser._id!, builtUser.email)
     await cache.user.invalidateUser(response.id)
 
+    // let server know to sync user
+    await apps.syncUserInApps(_id, dbUser)
+
     await Promise.all(groupPromises)
 
     // finally returned the saved user from the db
@@ -427,6 +469,7 @@ export const bulkCreate = async (
     // instead of relying on looping tenant creation
     await platform.users.addUser(tenantId, user._id, user.email)
     await eventHelpers.handleSaveEvents(user, undefined)
+    await apps.syncUserInApps(user._id)
   }
 
   const saved = usersToBulkSave.map(user => {
@@ -540,7 +583,7 @@ export const bulkDelete = async (
   return response
 }
 
-export const destroy = async (id: string) => {
+export const destroy = async (id: string, currentUser: any) => {
   const db = tenancy.getGlobalDB()
   const dbUser = (await db.get(id)) as User
   const userId = dbUser._id as string
@@ -550,7 +593,7 @@ export const destroy = async (id: string) => {
     const email = dbUser.email
     const account = await accounts.getAccount(email)
     if (account) {
-      if (dbUser.userId === context.getIdentity()!._id) {
+      if (email === currentUser.email) {
         throw new HTTPError('Please visit "Account" to delete this user', 400)
       } else {
         throw new HTTPError("Account holder cannot be deleted", 400)
@@ -565,6 +608,8 @@ export const destroy = async (id: string) => {
   await eventHelpers.handleDeleteEvents(dbUser)
   await cache.user.invalidateUser(userId)
   await sessions.invalidateSessions(userId, { reason: "deletion" })
+  // let server know to sync user
+  await apps.syncUserInApps(userId, dbUser)
 }
 
 const bulkDeleteProcessing = async (dbUser: User) => {
@@ -573,6 +618,8 @@ const bulkDeleteProcessing = async (dbUser: User) => {
   await eventHelpers.handleDeleteEvents(dbUser)
   await cache.user.invalidateUser(userId)
   await sessions.invalidateSessions(userId, { reason: "bulk-deletion" })
+  // let server know to sync user
+  await apps.syncUserInApps(userId, dbUser)
 }
 
 export const invite = async (

package/src/tests/TestConfiguration.ts

@@ -20,18 +20,9 @@ import {
   auth,
   constants,
   env as coreEnv,
-  db as dbCore,
-  encryption,
-  utils,
 } from "@budibase/backend-core"
 import structures, { CSRF_TOKEN } from "./structures"
-import {
-  SaveUserResponse,
-  User,
-  AuthToken,
-  SCIMConfig,
-  ConfigType,
-} from "@budibase/types"
+import { SaveUserResponse, User, AuthToken } from "@budibase/types"
 import API from "./api"
 
 class TestConfiguration {
@@ -40,7 +31,6 @@ class TestConfiguration {
   api: API
   tenantId: string
   user?: User
-  apiKey?: string
   userPassword = "test"
 
   constructor(opts: { openServer: boolean } = { openServer: true }) {
@@ -59,12 +49,6 @@ class TestConfiguration {
     this.api = new API(this)
   }
 
-  async useNewTenant() {
-    this.tenantId = structures.tenant.id()
-
-    await this.beforeAll()
-  }
-
   getRequest() {
     return this.request
   }
@@ -217,12 +201,6 @@ class TestConfiguration {
     return { [constants.Header.API_KEY]: coreEnv.INTERNAL_API_KEY }
   }
 
-  bearerAPIHeaders() {
-    return {
-      [constants.Header.AUTHORIZATION]: `Bearer ${this.apiKey}`,
-    }
-  }
-
   adminOnlyResponse = () => {
     return { message: "Admin user only endpoint.", status: 403 }
   }
@@ -235,20 +213,6 @@ class TestConfiguration {
     })
     await context.doInTenant(this.tenantId!, async () => {
       this.user = await this.createUser(user)
-
-      const db = context.getGlobalDB()
-
-      const id = dbCore.generateDevInfoID(this.user._id)
-      // TODO: dry
-      this.apiKey = encryption.encrypt(
-        `${this.tenantId}${dbCore.SEPARATOR}${utils.newid()}`
-      )
-      const devInfo = {
-        _id: id,
-        userId: this.user._id,
-        apiKey: this.apiKey,
-      }
-      await db.put(devInfo)
     })
   }
 
@@ -341,19 +305,6 @@ class TestConfiguration {
       controllers.config.save
     )
   }
-
-  // CONFIGS - SCIM
-
-  async setSCIMConfig(enabled: boolean) {
-    await this.deleteConfig(Config.SCIM)
-    const config: SCIMConfig = {
-      type: ConfigType.SCIM,
-      config: { enabled },
-    }
-
-    await this._req(config, null, controllers.config.save)
-    return config
-  }
 }
 
 export default TestConfiguration

package/src/tests/api/index.ts

@@ -15,9 +15,6 @@ import { RolesAPI } from "./roles"
 import { TemplatesAPI } from "./templates"
 import { LicenseAPI } from "./license"
 import { AuditLogAPI } from "./auditLogs"
-import { ScimUsersAPI } from "./scim/users"
-import { ScimGroupsAPI } from "./scim/groups"
-
 export default class API {
   accounts: AccountAPI
   auth: AuthAPI
@@ -35,8 +32,6 @@ export default class API {
   templates: TemplatesAPI
   license: LicenseAPI
   auditLogs: AuditLogAPI
-  scimUsersAPI: ScimUsersAPI
-  scimGroupsAPI: ScimGroupsAPI
 
   constructor(config: TestConfiguration) {
     this.accounts = new AccountAPI(config)
@@ -55,7 +50,5 @@ export default class API {
     this.templates = new TemplatesAPI(config)
     this.license = new LicenseAPI(config)
     this.auditLogs = new AuditLogAPI(config)
-    this.scimUsersAPI = new ScimUsersAPI(config)
-    this.scimGroupsAPI = new ScimGroupsAPI(config)
   }
 }

package/src/tests/jestEnv.ts

@@ -2,6 +2,7 @@ process.env.SELF_HOSTED = "0"
 process.env.NODE_ENV = "jest"
 process.env.JWT_SECRET = "test-jwtsecret"
 process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
+process.env.ENABLE_4XX_HTTP_LOGGING = "0"
 process.env.MULTI_TENANCY = "1"
 process.env.MINIO_URL = "http://localhost"
 process.env.MINIO_ACCESS_KEY = "test"

package/src/tests/jestSetup.ts

@@ -1,3 +1,5 @@
+import "./logging"
+
 import { mocks, testContainerUtils } from "@budibase/backend-core/tests"
 import env from "../environment"
 import { env as coreEnv, timers } from "@budibase/backend-core"

package/src/tests/logging.ts

@@ -0,0 +1,34 @@
+export enum LogLevel {
+  TRACE = "trace",
+  DEBUG = "debug",
+  INFO = "info",
+  WARN = "warn",
+  ERROR = "error",
+}
+
+const LOG_INDEX: { [key in LogLevel]: number } = {
+  [LogLevel.TRACE]: 1,
+  [LogLevel.DEBUG]: 2,
+  [LogLevel.INFO]: 3,
+  [LogLevel.WARN]: 4,
+  [LogLevel.ERROR]: 5,
+}
+
+const setIndex = LOG_INDEX[process.env.LOG_LEVEL as LogLevel]
+
+if (setIndex > LOG_INDEX.trace) {
+  global.console.trace = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.debug) {
+  global.console.debug = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.info) {
+  global.console.info = jest.fn()
+  global.console.log = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.warn) {
+  global.console.warn = jest.fn()
+}

package/src/utilities/appService.ts

@@ -0,0 +1,46 @@
+import fetch from "node-fetch"
+import {
+  constants,
+  tenancy,
+  logging,
+  env as coreEnv,
+} from "@budibase/backend-core"
+import { checkSlashesInUrl } from "../utilities"
+import env from "../environment"
+import { SyncUserRequest, User } from "@budibase/types"
+
+async function makeAppRequest(url: string, method: string, body: any) {
+  if (env.isTest()) {
+    return
+  }
+  const request: any = { headers: {} }
+  request.headers[constants.Header.API_KEY] = coreEnv.INTERNAL_API_KEY
+  if (tenancy.isTenantIdSet()) {
+    request.headers[constants.Header.TENANT_ID] = tenancy.getTenantId()
+  }
+  if (body) {
+    request.headers["Content-Type"] = "application/json"
+    request.body = JSON.stringify(body)
+  }
+  request.method = method
+
+  // add x-budibase-correlation-id header
+  logging.correlation.setHeader(request.headers)
+
+  return fetch(checkSlashesInUrl(env.APPS_URL + url), request)
+}
+
+export async function syncUserInApps(userId: string, previousUser?: User) {
+  const body: SyncUserRequest = {
+    previousUser,
+  }
+
+  const response = await makeAppRequest(
+    `/api/users/metadata/sync/${userId}`,
+    "POST",
+    body
+  )
+  if (response && response.status !== 200) {
+    throw "Unable to sync user."
+  }
+}