@budibase/worker 3.12.5 → 3.12.7

package/nodemon.json

@@ -5,7 +5,8 @@
     "../pro",
     "../types",
     "../shared-core",
-    "../string-templates"
+    "../string-templates",
+    "../../.env"
   ],
   "ext": "js,ts,json",
   "ignore": ["**/*.spec.ts", "**/*.spec.js", "../*/dist/**/*"],

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "3.12.5",
+  "version": "3.12.7",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -21,19 +21,10 @@
     "run:docker": "node dist/index.js",
     "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
-    "dev:stack:init": "node ./scripts/dev/manage.js init",
-    "dev": "npm run dev:stack:init && nodemon",
-    "dev:built": "yarn run dev:stack:init && yarn run run:docker",
+    "dev": "nodemon",
+    "dev:built": "yarn run run:docker",
     "test": "bash scripts/test.sh",
-    "test:watch": "jest --watch",
-    "env:multi:enable": "node scripts/multiTenancy.js enable",
-    "env:multi:disable": "node scripts/multiTenancy.js disable",
-    "env:selfhost:enable": "node scripts/selfhost.js enable",
-    "env:selfhost:disable": "node scripts/selfhost.js disable",
-    "env:localdomain:enable": "node scripts/localdomain.js enable",
-    "env:localdomain:disable": "node scripts/localdomain.js disable",
-    "env:account:enable": "node scripts/account.js enable",
-    "env:account:disable": "node scripts/account.js disable"
+    "test:watch": "jest --watch"
   },
   "author": "Budibase",
   "license": "GPL-3.0",
@@ -102,8 +93,7 @@
     "superagent": "^10.1.1",
     "supertest": "6.3.3",
     "timekeeper": "2.2.0",
-    "typescript": "5.7.2",
-    "update-dotenv": "1.1.1"
+    "typescript": "5.7.2"
   },
   "resolutions": {
     "@budibase/pro": "npm:@budibase/pro@latest"
@@ -123,5 +113,5 @@
       }
     }
   },
-  "gitHead": "1b7296dc708223ebfa03d91b471c1ad4a4b49758"
+  "gitHead": "87d0fd4e2bfa88a5f45312ce133670856957b97b"
 }

package/src/api/controllers/global/configs.ts

@@ -13,7 +13,6 @@ import {
 } from "@budibase/backend-core"
 import { checkAnyUserExists } from "../../../utilities/users"
 import {
-  AIConfig,
   AIInnerConfig,
   Config,
   ConfigChecklistResponse,
@@ -36,6 +35,7 @@ import {
   SaveConfigResponse,
   SettingsBrandingConfig,
   SettingsInnerConfig,
+  SMTPInnerConfig,
   SSOConfig,
   SSOConfigType,
   UploadConfigFileResponse,
@@ -158,7 +158,23 @@ async function hasActivatedConfig(ssoConfigs?: SSOConfigs) {
   return !!Object.values(ssoConfigs).find(c => c?.activated)
 }
 
-async function verifySettingsConfig(
+async function processSMTPConfig(
+  config: SMTPInnerConfig,
+  existingConfig?: SMTPInnerConfig
+) {
+  await email.verifyConfig(config)
+  if (config.auth?.pass === PASSWORD_REPLACEMENT) {
+    // if the password is being replaced, use the existing password
+    if (existingConfig && existingConfig.auth?.pass) {
+      config.auth.pass = existingConfig.auth.pass
+    } else {
+      // otherwise, throw an error
+      throw new BadRequestError("SMTP password is required")
+    }
+  }
+}
+
+async function processSettingsConfig(
   config: SettingsInnerConfig & SettingsBrandingConfig,
   existingConfig?: SettingsInnerConfig & SettingsBrandingConfig
 ) {
@@ -203,19 +219,37 @@ async function verifySSOConfig(type: SSOConfigType, config: SSOConfig) {
   }
 }
 
-async function verifyGoogleConfig(config: GoogleInnerConfig) {
+async function processGoogleConfig(
+  config: GoogleInnerConfig,
+  existing?: GoogleInnerConfig
+) {
   await verifySSOConfig(ConfigType.GOOGLE, config)
+
+  if (existing && config.clientSecret === PASSWORD_REPLACEMENT) {
+    config.clientSecret = existing.clientSecret
+  }
 }
 
-async function verifyOIDCConfig(config: OIDCConfigs) {
+async function processOIDCConfig(config: OIDCConfigs, existing?: OIDCConfigs) {
   await verifySSOConfig(ConfigType.OIDC, config.configs[0])
+
+  if (existing) {
+    for (const c of config.configs) {
+      const existingConfig = existing.configs.find(e => e.uuid === c.uuid)
+      if (!existingConfig) {
+        continue
+      }
+      if (c.clientSecret === PASSWORD_REPLACEMENT) {
+        c.clientSecret = existingConfig.clientSecret
+      }
+    }
+  }
 }
 
 export async function processAIConfig(
   newConfig: AIInnerConfig,
   existingConfig: AIInnerConfig
 ) {
-  // ensure that the redacted API keys are not overwritten in the DB
   for (const key in existingConfig) {
     if (newConfig[key]?.apiKey === PASSWORD_REPLACEMENT) {
       newConfig[key].apiKey = existingConfig[key].apiKey
@@ -256,16 +290,16 @@ export async function save(
   try {
     switch (type) {
       case ConfigType.SMTP:
-        await email.verifyConfig(config)
+        await processSMTPConfig(config, existingConfig?.config)
         break
       case ConfigType.SETTINGS:
-        await verifySettingsConfig(config, existingConfig?.config)
+        await processSettingsConfig(config, existingConfig?.config)
         break
       case ConfigType.GOOGLE:
-        await verifyGoogleConfig(config)
+        await processGoogleConfig(config, existingConfig?.config)
         break
       case ConfigType.OIDC:
-        await verifyOIDCConfig(config)
+        await processOIDCConfig(config, existingConfig?.config)
         break
       case ConfigType.AI:
         if (existingConfig) {
@@ -353,45 +387,48 @@ async function enrichOIDCLogos(oidcLogos: OIDCLogosConfig) {
 }
 
 export async function find(ctx: UserCtx<void, FindConfigResponse>) {
-  try {
-    // Find the config with the most granular scope based on context
-    const type = ctx.params.type
-    let scopedConfig = await configs.getConfig(type)
-
-    if (scopedConfig) {
-      await handleConfigType(type, scopedConfig)
-    } else if (type === ConfigType.AI) {
-      scopedConfig = { type: ConfigType.AI, config: {} }
-      await handleAIConfig(scopedConfig)
-    } else {
-      // If no config found and not AI type, just return an empty body
-      ctx.body = {}
-      return
-    }
+  // Find the config with the most granular scope based on context
+  const type = ctx.params.type
+  let config = await configs.getConfig(type)
 
-    ctx.body = scopedConfig
-  } catch (err: any) {
-    ctx.throw(err?.status || 400, err)
+  if (!config && type === ConfigType.AI) {
+    config = { type: ConfigType.AI, config: {} }
+  }
+
+  if (!config) {
+    ctx.body = {}
+    return
   }
-}
 
-async function handleConfigType(type: ConfigType, config: Config) {
-  if (type === ConfigType.OIDC_LOGOS) {
-    await enrichOIDCLogos(config)
-  } else if (type === ConfigType.AI) {
-    await handleAIConfig(config)
+  switch (type) {
+    case ConfigType.OIDC_LOGOS:
+      await enrichOIDCLogos(config)
+      break
+    case ConfigType.AI:
+      await pro.sdk.ai.enrichAIConfig(config)
+      break
   }
-}
 
-async function handleAIConfig(config: AIConfig) {
-  await pro.sdk.ai.enrichAIConfig(config)
-  stripApiKeys(config)
+  stripSecrets(config)
+  ctx.body = config
 }
 
-function stripApiKeys(config: AIConfig) {
-  for (const key in config?.config) {
-    if (config.config[key].apiKey) {
-      config.config[key].apiKey = PASSWORD_REPLACEMENT
+function stripSecrets(config: Config) {
+  if (isAIConfig(config)) {
+    for (const key in config.config) {
+      if (config.config[key].apiKey) {
+        config.config[key].apiKey = PASSWORD_REPLACEMENT
+      }
+    }
+  } else if (isSMTPConfig(config)) {
+    if (config.config.auth?.pass) {
+      config.config.auth.pass = PASSWORD_REPLACEMENT
+    }
+  } else if (isGoogleConfig(config)) {
+    config.config.clientSecret = PASSWORD_REPLACEMENT
+  } else if (isOIDCConfig(config)) {
+    for (const c of config.config.configs) {
+      c.clientSecret = PASSWORD_REPLACEMENT
     }
   }
 }

package/src/api/controllers/global/users.ts

@@ -33,6 +33,7 @@ import {
   SaveUserResponse,
   SearchUsersRequest,
   SearchUsersResponse,
+  StrippedUser,
   UnsavedUser,
   UpdateInviteRequest,
   UpdateInviteResponse,
@@ -66,6 +67,15 @@ const generatePassword = (length: number) => {
     .slice(0, length)
 }
 
+const stripUsers = (users: (User | StrippedUser)[]): StrippedUser[] => {
+  return users.map(user => ({
+    _id: user._id,
+    email: user.email,
+    tenantId: user.tenantId,
+    userId: user.userId,
+  }))
+}
+
 export const save = async (ctx: UserCtx<UnsavedUser, SaveUserResponse>) => {
   try {
     const currentUserId = ctx.user?._id
@@ -249,18 +259,8 @@ export const destroy = async (ctx: UserCtx<void, DeleteUserResponse>) => {
   }
 }
 
-export const getAppUsers = async (ctx: Ctx<SearchUsersRequest>) => {
-  const body = ctx.request.body
-  const users = await userSdk.db.getUsersByAppAccess({
-    appId: body.appId,
-    limit: body.limit,
-  })
-
-  ctx.body = { data: users }
-}
-
 export const search = async (
-  ctx: Ctx<SearchUsersRequest, SearchUsersResponse>
+  ctx: UserCtx<SearchUsersRequest, SearchUsersResponse>
 ) => {
   const body = ctx.request.body
 
@@ -287,8 +287,13 @@ export const search = async (
     }
   }
 
+  let response: SearchUsersResponse = { data: [] }
+
   if (body.paginate === false) {
-    await getAppUsers(ctx)
+    response.data = await userSdk.db.getUsersByAppAccess({
+      appId: body.appId,
+      limit: body.limit,
+    })
   } else {
     const paginated = await userSdk.core.paginatedUsers(body)
     // user hashed password shouldn't ever be returned
@@ -297,8 +302,18 @@ export const search = async (
         delete user.password
       }
     }
-    ctx.body = paginated
+    response = {
+      data: paginated.data,
+      hasNextPage: paginated.hasNextPage,
+      nextPage: paginated.nextPage,
+    }
   }
+
+  if (!users.hasBuilderPermissions(ctx.user)) {
+    response.data = stripUsers(response.data)
+  }
+
+  ctx.body = response
 }
 
 // called internally by app server user fetch

package/src/api/routes/global/tests/configs.spec.ts

@@ -1,19 +1,17 @@
-// mock the email system
 jest.mock("nodemailer")
 import { TestConfiguration, structures, mocks } from "../../../../tests"
 
 mocks.email.mock()
-import { events } from "@budibase/backend-core"
+import { configs, events } from "@budibase/backend-core"
 import { GetPublicSettingsResponse, Config, ConfigType } from "@budibase/types"
 
+const { google, smtp, settings, oidc } = structures.configs
+
 describe("configs", () => {
   const config = new TestConfiguration()
 
   beforeEach(async () => {
     await config.beforeAll()
-  })
-
-  beforeEach(() => {
     jest.clearAllMocks()
   })
 
@@ -22,38 +20,20 @@ describe("configs", () => {
   })
 
   const saveConfig = async (conf: Config, _id?: string, _rev?: string) => {
-    const data = {
-      ...conf,
-      _id,
-      _rev,
-    }
+    const data = { ...conf, _id, _rev }
     const res = await config.api.configs.saveConfig(data)
     return { ...data, ...res }
   }
 
-  const saveSettingsConfig = async (
-    conf?: any,
-    _id?: string,
-    _rev?: string
-  ) => {
-    const settingsConfig = structures.configs.settings(conf)
-    return saveConfig(settingsConfig, _id, _rev)
-  }
-
   describe("POST /api/global/configs", () => {
     describe("google", () => {
-      const saveGoogleConfig = async (
-        conf?: any,
-        _id?: string,
-        _rev?: string
-      ) => {
-        const googleConfig = structures.configs.google(conf)
-        return saveConfig(googleConfig, _id, _rev)
-      }
+      afterEach(async () => {
+        await config.deleteConfig(ConfigType.GOOGLE)
+      })
 
       describe("create", () => {
         it("should create activated google config", async () => {
-          await saveGoogleConfig()
+          await saveConfig(google())
           expect(events.auth.SSOCreated).toHaveBeenCalledTimes(1)
           expect(events.auth.SSOCreated).toHaveBeenCalledWith(ConfigType.GOOGLE)
           expect(events.auth.SSODeactivated).not.toHaveBeenCalled()
@@ -61,25 +41,23 @@ describe("configs", () => {
           expect(events.auth.SSOActivated).toHaveBeenCalledWith(
             ConfigType.GOOGLE
           )
-          await config.deleteConfig(ConfigType.GOOGLE)
         })
 
         it("should create deactivated google config", async () => {
-          await saveGoogleConfig({ activated: false })
+          await saveConfig(google({ activated: false }))
           expect(events.auth.SSOCreated).toHaveBeenCalledTimes(1)
           expect(events.auth.SSOCreated).toHaveBeenCalledWith(ConfigType.GOOGLE)
           expect(events.auth.SSOActivated).not.toHaveBeenCalled()
           expect(events.auth.SSODeactivated).not.toHaveBeenCalled()
-          await config.deleteConfig(ConfigType.GOOGLE)
         })
       })
 
       describe("update", () => {
         it("should update google config to deactivated", async () => {
-          const googleConf = await saveGoogleConfig()
+          const googleConf = await saveConfig(google())
           jest.clearAllMocks()
-          await saveGoogleConfig(
-            { ...googleConf.config, activated: false },
+          await saveConfig(
+            google({ activated: false }),
             googleConf._id,
             googleConf._rev
           )
@@ -90,14 +68,13 @@ describe("configs", () => {
           expect(events.auth.SSODeactivated).toHaveBeenCalledWith(
             ConfigType.GOOGLE
           )
-          await config.deleteConfig(ConfigType.GOOGLE)
         })
 
         it("should update google config to activated", async () => {
-          const googleConf = await saveGoogleConfig({ activated: false })
+          const googleConf = await saveConfig(google({ activated: false }))
           jest.clearAllMocks()
-          await saveGoogleConfig(
-            { ...googleConf.config, activated: true },
+          await saveConfig(
+            google({ activated: true }),
             googleConf._id,
             googleConf._rev
           )
@@ -108,48 +85,64 @@ describe("configs", () => {
           expect(events.auth.SSOActivated).toHaveBeenCalledWith(
             ConfigType.GOOGLE
           )
-          await config.deleteConfig(ConfigType.GOOGLE)
+        })
+
+        it("should not overwrite secret when updating google config", async () => {
+          await saveConfig(google({ clientSecret: "spooky" }))
+
+          const conf = await config.api.configs.getConfig(ConfigType.GOOGLE)
+          await saveConfig(conf)
+
+          await config.doInTenant(async () => {
+            const rawConf = await configs.getGoogleConfig()
+            expect(rawConf!.clientSecret).toEqual("spooky")
+          })
+        })
+      })
+
+      describe("get", () => {
+        it("should not leak credentials", async () => {
+          await saveConfig(google())
+          const conf = await config.api.configs.getConfig(ConfigType.GOOGLE)
+          expect(conf.config.clientSecret).toEqual("--secret-value--")
         })
       })
     })
 
     describe("oidc", () => {
-      const saveOIDCConfig = async (
-        conf?: any,
-        _id?: string,
-        _rev?: string
-      ) => {
-        const oidcConfig = structures.configs.oidc(conf)
-        return saveConfig(oidcConfig, _id, _rev)
-      }
+      beforeEach(async () => {
+        await config.deleteConfig(ConfigType.OIDC)
+      })
+
+      afterEach(async () => {
+        await config.deleteConfig(ConfigType.OIDC)
+      })
 
       describe("create", () => {
         it("should create activated OIDC config", async () => {
-          await saveOIDCConfig()
+          await saveConfig(oidc())
           expect(events.auth.SSOCreated).toHaveBeenCalledTimes(1)
           expect(events.auth.SSOCreated).toHaveBeenCalledWith(ConfigType.OIDC)
           expect(events.auth.SSODeactivated).not.toHaveBeenCalled()
           expect(events.auth.SSOActivated).toHaveBeenCalledTimes(1)
           expect(events.auth.SSOActivated).toHaveBeenCalledWith(ConfigType.OIDC)
-          await config.deleteConfig(ConfigType.OIDC)
         })
 
         it("should create deactivated OIDC config", async () => {
-          await saveOIDCConfig({ activated: false })
+          await saveConfig(oidc({ activated: false }))
           expect(events.auth.SSOCreated).toHaveBeenCalledTimes(1)
           expect(events.auth.SSOCreated).toHaveBeenCalledWith(ConfigType.OIDC)
           expect(events.auth.SSOActivated).not.toHaveBeenCalled()
           expect(events.auth.SSODeactivated).not.toHaveBeenCalled()
-          await config.deleteConfig(ConfigType.OIDC)
         })
       })
 
       describe("update", () => {
         it("should update OIDC config to deactivated", async () => {
-          const oidcConf = await saveOIDCConfig()
+          const oidcConf = await saveConfig(oidc())
           jest.clearAllMocks()
-          await saveOIDCConfig(
-            { ...oidcConf.config.configs[0], activated: false },
+          await saveConfig(
+            oidc({ activated: false }),
             oidcConf._id,
             oidcConf._rev
           )
@@ -160,14 +153,13 @@ describe("configs", () => {
           expect(events.auth.SSODeactivated).toHaveBeenCalledWith(
             ConfigType.OIDC
           )
-          await config.deleteConfig(ConfigType.OIDC)
         })
 
         it("should update OIDC config to activated", async () => {
-          const oidcConf = await saveOIDCConfig({ activated: false })
+          const oidcConf = await saveConfig(oidc({ activated: false }))
           jest.clearAllMocks()
-          await saveOIDCConfig(
-            { ...oidcConf.config.configs[0], activated: true },
+          await saveConfig(
+            oidc({ activated: true }),
             oidcConf._id,
             oidcConf._rev
           )
@@ -176,50 +168,88 @@ describe("configs", () => {
           expect(events.auth.SSODeactivated).not.toHaveBeenCalled()
           expect(events.auth.SSOActivated).toHaveBeenCalledTimes(1)
           expect(events.auth.SSOActivated).toHaveBeenCalledWith(ConfigType.OIDC)
-          await config.deleteConfig(ConfigType.OIDC)
+        })
+
+        it("should not overwrite secret when updating OIDC config", async () => {
+          await saveConfig(oidc({ clientSecret: "spooky" }))
+          const conf = await config.api.configs.getConfig(ConfigType.OIDC)
+          await saveConfig(conf)
+          await config.doInTenant(async () => {
+            const rawConf = await configs.getOIDCConfig()
+            expect(rawConf!.clientSecret).toEqual("spooky")
+          })
+        })
+      })
+
+      describe("get", () => {
+        it("should not leak credentials", async () => {
+          await saveConfig(oidc({ clientSecret: "spooky" }))
+          const conf = await config.api.configs.getConfig(ConfigType.OIDC)
+          expect(conf.config.configs[0].clientSecret).toEqual(
+            "--secret-value--"
+          )
         })
       })
     })
 
     describe("smtp", () => {
-      const saveSMTPConfig = async (
-        conf?: any,
-        _id?: string,
-        _rev?: string
-      ) => {
-        const smtpConfig = structures.configs.smtp(conf)
-        return saveConfig(smtpConfig, _id, _rev)
-      }
+      beforeEach(async () => {
+        await config.deleteConfig(ConfigType.SMTP)
+      })
+
+      afterEach(async () => {
+        await config.deleteConfig(ConfigType.SMTP)
+      })
 
       describe("create", () => {
         it("should create SMTP config", async () => {
-          await config.deleteConfig(ConfigType.SMTP)
-          await saveSMTPConfig()
+          await saveConfig(smtp())
           expect(events.email.SMTPUpdated).not.toHaveBeenCalled()
           expect(events.email.SMTPCreated).toHaveBeenCalledTimes(1)
-          await config.deleteConfig(ConfigType.SMTP)
         })
       })
 
       describe("update", () => {
         it("should update SMTP config", async () => {
-          const smtpConf = await saveSMTPConfig()
+          const smtpConf = await saveConfig(smtp())
           jest.clearAllMocks()
-          await saveSMTPConfig(smtpConf.config, smtpConf._id, smtpConf._rev)
+          await saveConfig(smtp({ secure: true }), smtpConf._id, smtpConf._rev)
           expect(events.email.SMTPCreated).not.toHaveBeenCalled()
           expect(events.email.SMTPUpdated).toHaveBeenCalledTimes(1)
-          await config.deleteConfig(ConfigType.SMTP)
+        })
+
+        it("should not overwrite secret when updating SMTP config", async () => {
+          await saveConfig(smtp({ auth: { user: "jeff", pass: "spooky" } }))
+          const conf = await config.api.configs.getConfig(ConfigType.SMTP)
+          await saveConfig(conf)
+          await config.doInTenant(async () => {
+            const rawConf = await configs.getSMTPConfig()
+            expect(rawConf!.auth!.pass).toEqual("spooky")
+          })
+        })
+      })
+
+      describe("get", () => {
+        it("should not leak credentials", async () => {
+          await saveConfig(smtp({ auth: { user: "jeff", pass: "spooky" } }))
+          const conf = await config.api.configs.getConfig(ConfigType.SMTP)
+          expect(conf.config.auth!.pass).toEqual("--secret-value--")
         })
       })
     })
 
     describe("settings", () => {
-      describe("create", () => {
-        it("should create settings config with default settings", async () => {
-          await config.deleteConfig(ConfigType.SETTINGS)
+      beforeEach(async () => {
+        await config.deleteConfig(ConfigType.SETTINGS)
+      })
 
-          await saveSettingsConfig()
+      afterEach(async () => {
+        await config.deleteConfig(ConfigType.SETTINGS)
+      })
 
+      describe("create", () => {
+        it("should create settings config with default settings", async () => {
+          await saveConfig(settings())
           expect(events.org.nameUpdated).not.toHaveBeenCalled()
           expect(events.org.logoUpdated).not.toHaveBeenCalled()
           expect(events.org.platformURLUpdated).not.toHaveBeenCalled()
@@ -234,7 +264,7 @@ describe("configs", () => {
             platformUrl: "http://example.com",
           }
 
-          await saveSettingsConfig(conf)
+          await saveConfig(settings(conf))
 
           expect(events.org.nameUpdated).toHaveBeenCalledTimes(1)
           expect(events.org.logoUpdated).toHaveBeenCalledTimes(1)
@@ -247,13 +277,13 @@ describe("configs", () => {
         it("should update settings config", async () => {
           config.selfHosted()
           await config.deleteConfig(ConfigType.SETTINGS)
-          const settingsConfig = await saveSettingsConfig()
+          const settingsConfig = await saveConfig(settings())
           settingsConfig.config.company = "acme"
           settingsConfig.config.logoUrl = "http://example.com"
           settingsConfig.config.platformUrl = "http://example.com"
 
-          await saveSettingsConfig(
-            settingsConfig.config,
+          await saveConfig(
+            settingsConfig,
             settingsConfig._id,
             settingsConfig._rev
           )
@@ -282,16 +312,24 @@ describe("configs", () => {
   })
 
   describe("GET /api/global/configs/public", () => {
+    beforeEach(async () => {
+      await config.deleteConfig(ConfigType.SETTINGS)
+    })
+
+    afterEach(async () => {
+      await config.deleteConfig(ConfigType.SETTINGS)
+    })
+
     it("should return the expected public settings", async () => {
-      await saveSettingsConfig()
+      await saveConfig(settings())
       mocks.pro.features.isSSOEnforced.mockResolvedValue(false)
 
       const res = await config.api.configs.getPublicSettings()
       const body = res.body as GetPublicSettingsResponse
 
       const expected = {
-        _id: "config_settings",
-        type: "settings",
+        _id: `config_${ConfigType.SETTINGS}`,
+        type: ConfigType.SETTINGS,
         config: {
           company: "Budibase",
           emailBrandingEnabled: true,

package/src/api/routes/global/tests/users.spec.ts

@@ -704,6 +704,24 @@ describe("/api/global/users", () => {
       expect(response.body.data.length).toBe(1)
       expect(response.body.data[0].email).toBe(user.email)
     })
+
+    it("should strip users if accessing as an end user", async () => {
+      const user = await config.createUser({
+        admin: { global: false },
+        builder: { global: false },
+      })
+      const response = await config.api.users.searchUsers(
+        {
+          query: {},
+        },
+        { useHeaders: await config.login(user) }
+      )
+      for (let user of response.body.data) {
+        expect(user.roles).toBeUndefined()
+        expect(user.builder).toBeUndefined()
+        expect(user.admin).toBeUndefined()
+      }
+    })
   })
 
   describe("DELETE /api/global/users/:userId", () => {

package/src/environment.ts

@@ -1,14 +1,18 @@
 import { env as coreEnv } from "@budibase/backend-core"
 import { ServiceType } from "@budibase/types"
-import { join } from "path"
+import { join, resolve } from "path"
 import cloneDeep from "lodash/cloneDeep"
 
 coreEnv._set("SERVICE_TYPE", ServiceType.WORKER)
 
+const TOP_LEVEL_PATH =
+  process.env.TOP_LEVEL_PATH ||
+  process.env.WORKER_TOP_LEVEL_PATH ||
+  resolve(join(__dirname, "..", "..", ".."))
 let LOADED = false
 if (!LOADED && coreEnv.isDev() && !coreEnv.isTest()) {
   require("dotenv").config({
-    path: join(__dirname, "..", ".env"),
+    path: join(TOP_LEVEL_PATH, ".env"),
   })
   LOADED = true
 }
@@ -40,6 +44,7 @@ const environment = {
   // prefer worker port to generic port
   PORT: process.env.WORKER_PORT || process.env.PORT,
   CLUSTER_PORT: process.env.CLUSTER_PORT,
+  WORKER_SERVICE: process.env.WORKER_SERVICE,
   // flags
   NODE_ENV: process.env.NODE_ENV,
   SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED || ""),
@@ -59,6 +64,7 @@ const environment = {
   SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD,
   ENCRYPTED_TEST_PUBLIC_API_KEY: process.env.ENCRYPTED_TEST_PUBLIC_API_KEY,
   SESSION_EXPIRY_SECONDS: process.env.SESSION_EXPIRY_SECONDS,
+  TOP_LEVEL_PATH: TOP_LEVEL_PATH,
   /**
    * Mock the email service in use - links to ethereal hosted emails are logged instead.
    */

package/src/index.ts

@@ -85,10 +85,12 @@ app.use(api.routes())
 
 const server = http.createServer(app.callback())
 
-const shutdown = async () => {
-  console.log("Worker service shutting down gracefully...")
+const shutdown = async (signal?: string) => {
+  console.log(
+    `Worker service shutting down gracefully... ${signal ? `Signal: ${signal}` : ""}`
+  )
   timers.cleanup()
-  events.shutdown()
+  await events.shutdown()
   await redis.clients.shutdown()
   await queue.shutdown()
 }
@@ -97,7 +99,7 @@ gracefulShutdown(server, {
   signals: "SIGINT SIGTERM",
   timeout: 30000,
   onShutdown: shutdown,
-  forceExit: !env.isTest,
+  forceExit: !env.isTest(),
   finally: () => {
     console.log("Worker service shutdown complete")
   },
@@ -106,7 +108,7 @@ gracefulShutdown(server, {
 process.on("uncaughtException", async err => {
   logging.logAlert("Uncaught exception.", err)
   await shutdown()
-  if (!env.isTest) {
+  if (!env.isTest()) {
     process.exit(1)
   }
 })
@@ -114,7 +116,7 @@ process.on("uncaughtException", async err => {
 process.on("unhandledRejection", async reason => {
   logging.logAlert("Unhandled Promise Rejection", reason as Error)
   await shutdown()
-  if (!env.isTest) {
+  if (!env.isTest()) {
     process.exit(1)
   }
 })

package/src/tests/TestConfiguration.ts

@@ -37,15 +37,20 @@ import {
 } from "@budibase/types"
 import API from "./api"
 import jwt, { Secret } from "jsonwebtoken"
+import http from "http"
 
 class TestConfiguration {
-  server: any
-  request: any
+  server: http.Server<typeof http.IncomingMessage, typeof http.ServerResponse> =
+    undefined!
+
+  request: supertest.SuperTest<supertest.Test> = undefined!
+
   api: API
   tenantId: string
   user?: User
   apiKey?: string
   userPassword = "password123!"
+  sessions: string[] = []
 
   constructor(opts: { openServer: boolean } = { openServer: true }) {
     // default to cloud hosting
@@ -185,12 +190,19 @@ class TestConfiguration {
     })
   }
 
+  hasSession(user: User) {
+    return this.sessions.includes(user._id!)
+  }
+
   async createSession(user: User) {
-    return this._createSession({
-      userId: user._id!,
-      tenantId: user.tenantId,
-      email: user.email,
-    })
+    if (!this.hasSession(user)) {
+      this.sessions.push(user._id!)
+      return this._createSession({
+        userId: user._id!,
+        tenantId: user.tenantId,
+        email: user.email,
+      })
+    }
   }
 
   cookieHeader(cookies: any) {
@@ -212,6 +224,11 @@ class TestConfiguration {
     }
   }
 
+  async login(user: User) {
+    await this.createSession(user)
+    return this.authHeaders(user)
+  }
+
   authHeaders(user: User) {
     const authToken: AuthToken = {
       userId: user._id!,
@@ -279,10 +296,10 @@ class TestConfiguration {
     })
   }
 
-  async createUser(opts?: Partial<User>) {
+  async createUser(userCfg?: Partial<User>) {
     let user = structures.users.user()
     if (user) {
-      user = { ...user, ...opts }
+      user = { ...user, ...userCfg }
     }
     const response = await this._req(user, null, controllers.users.save)
     const body = response as SaveUserResponse

package/src/tests/api/configs.ts

@@ -1,5 +1,6 @@
 import {
-  AIConfig,
+  ConfigType,
+  ConfigTypeToConfig,
   SaveConfigRequest,
   SaveConfigResponse,
 } from "@budibase/types"
@@ -23,12 +24,16 @@ export class ConfigAPI extends TestAPI {
   }
 
   getAIConfig = async () => {
+    return await this.getConfig(ConfigType.AI)
+  }
+
+  getConfig = async <T extends ConfigType>(type: T) => {
     const resp = await this.request
-      .get(`/api/global/configs/ai`)
+      .get(`/api/global/configs/${type}`)
       .set(this.config.defaultHeaders())
       .expect(200)
       .expect("Content-Type", /json/)
-    return resp.body as AIConfig
+    return resp.body as ConfigTypeToConfig<T>
   }
 
   saveConfig = async (

package/src/tests/api/users.ts

@@ -152,14 +152,20 @@ export class UserAPI extends TestAPI {
 
   searchUsers = (
     { query }: { query?: SearchFilters },
-    opts?: { status?: number; noHeaders?: boolean }
+    opts?: {
+      status?: number
+      noHeaders?: boolean
+      useHeaders?: Record<string, string>
+    }
   ) => {
     const req = this.request
       .post("/api/global/users/search")
       .send({ query })
       .expect("Content-Type", /json/)
       .expect(opts?.status ? opts.status : 200)
-    if (!opts?.noHeaders) {
+    if (opts?.useHeaders) {
+      req.set(opts.useHeaders)
+    } else if (!opts?.noHeaders) {
       req.set(this.config.defaultHeaders())
     }
     return req

package/src/tests/structures/configs.ts

@@ -5,9 +5,13 @@ import {
   SMTPConfig,
   GoogleConfig,
   OIDCConfig,
+  GoogleInnerConfig,
+  OIDCInnerConfig,
+  SMTPInnerConfig,
+  SettingsInnerConfig,
 } from "@budibase/types"
 
-export function oidc(conf?: any): OIDCConfig {
+export function oidc(conf?: Partial<OIDCInnerConfig>): OIDCConfig {
   return {
     type: ConfigType.OIDC,
     config: {
@@ -20,6 +24,7 @@ export function oidc(conf?: any): OIDCConfig {
           name: "Active Directory",
           uuid: utils.newid(),
           activated: true,
+          scopes: [],
           ...conf,
         },
       ],
@@ -27,7 +32,7 @@ export function oidc(conf?: any): OIDCConfig {
   }
 }
 
-export function google(conf?: any): GoogleConfig {
+export function google(conf?: Partial<GoogleInnerConfig>): GoogleConfig {
   return {
     type: ConfigType.GOOGLE,
     config: {
@@ -39,7 +44,7 @@ export function google(conf?: any): GoogleConfig {
   }
 }
 
-export function smtp(conf?: any): SMTPConfig {
+export function smtp(conf?: Partial<SMTPInnerConfig>): SMTPConfig {
   return {
     type: ConfigType.SMTP,
     config: {
@@ -70,7 +75,7 @@ export function smtpEthereal(): SMTPConfig {
   }
 }
 
-export function settings(conf?: any): SettingsConfig {
+export function settings(conf?: Partial<SettingsInnerConfig>): SettingsConfig {
   return {
     type: ConfigType.SETTINGS,
     config: {

package/scripts/account.js

@@ -1,8 +0,0 @@
-#!/usr/bin/env node
-const updateDotEnv = require("update-dotenv")
-
-const arg = process.argv.slice(2)[0]
-
-updateDotEnv({
-  DISABLE_ACCOUNT_PORTAL: arg === "enable" ? "" : "1",
-}).then(() => console.log("Updated worker!"))

package/scripts/dev/manage.js

@@ -1,54 +0,0 @@
-#!/usr/bin/env node
-const { parsed: existingConfig } = require("dotenv").config()
-const updateDotEnv = require("update-dotenv")
-
-async function init() {
-  let config = {
-    SELF_HOSTED: "1",
-    PORT: "4002",
-    CLUSTER_PORT: "10000",
-    JWT_SECRET: "testsecret",
-    INTERNAL_API_KEY: "budibase",
-    MINIO_ACCESS_KEY: "budibase",
-    MINIO_SECRET_KEY: "budibase",
-    REDIS_URL: "localhost:6379",
-    REDIS_PASSWORD: "budibase",
-    MINIO_URL: "http://localhost:4004",
-    COUCH_DB_URL: "http://budibase:budibase@localhost:4005",
-    COUCH_DB_USERNAME: "budibase",
-    COUCH_DB_PASSWORD: "budibase",
-    // empty string is false
-    MULTI_TENANCY: "",
-    DISABLE_ACCOUNT_PORTAL: "1",
-    ACCOUNT_PORTAL_URL: "http://localhost:10001",
-    PLATFORM_URL: "http://localhost:10000",
-    APPS_URL: "http://localhost:4001",
-    SERVICE: "worker-service",
-    DEPLOYMENT_ENVIRONMENT: "development",
-    ENABLE_EMAIL_TEST_MODE: "1",
-    HTTP_LOGGING: "0",
-    VERSION: "0.0.0+local",
-    PASSWORD_MIN_LENGTH: "1",
-  }
-
-  config = { ...config, ...existingConfig }
-
-  await updateDotEnv(config)
-}
-
-// if more than init required use this to determine the command type
-//const managementCommand = process.argv.slice(2)[0]
-
-// for now only one command
-let command = init
-
-command()
-  .then(() => {
-    console.log("Done! 🎉")
-  })
-  .catch(err => {
-    console.error(
-      "Something went wrong while managing budibase dev worker:",
-      err.message
-    )
-  })

package/scripts/localdomain.js

@@ -1,52 +0,0 @@
-#!/usr/bin/env node
-const updateDotEnv = require("update-dotenv")
-
-const arg = process.argv.slice(2)[0]
-const isEnable = arg === "enable"
-
-let domain = process.argv.slice(2)[1]
-if (!domain) {
-  domain = "local.com"
-}
-
-const getAccountPortalUrl = () => {
-  if (isEnable) {
-    return `http://account.${domain}:10001`
-  } else {
-    return `http://localhost:10001`
-  }
-}
-
-const getBudibaseUrl = () => {
-  if (isEnable) {
-    return `http://${domain}:10000`
-  } else {
-    return `http://localhost:10000`
-  }
-}
-
-const getCookieDomain = () => {
-  if (isEnable) {
-    return `.${domain}`
-  } else {
-    return ""
-  }
-}
-
-/**
- * For testing multi tenancy sub domains locally.
- *
- * Relies on an entry in /etc/hosts e.g:
- *
- * 127.0.0.1   local.com
- *
- * and an entry for each tenant you wish to test locally e.g:
- *
- * 127.0.0.1   t1.local.com
- * 127.0.0.1   t2.local.com
- */
-updateDotEnv({
-  ACCOUNT_PORTAL_URL: getAccountPortalUrl(),
-  COOKIE_DOMAIN: getCookieDomain(),
-  PLATFORM_URL: getBudibaseUrl(),
-}).then(() => console.log("Updated worker!"))

package/scripts/multiTenancy.js

@@ -1,8 +0,0 @@
-#!/usr/bin/env node
-const updateDotEnv = require("update-dotenv")
-
-const arg = process.argv.slice(2)[0]
-
-updateDotEnv({
-  MULTI_TENANCY: arg === "enable" ? "1" : "",
-}).then(() => console.log("Updated worker!"))

package/scripts/selfhost.js

@@ -1,8 +0,0 @@
-#!/usr/bin/env node
-const updateDotEnv = require("update-dotenv")
-
-const arg = process.argv.slice(2)[0]
-
-updateDotEnv({
-  SELF_HOSTED: arg === "enable" ? "1" : "0",
-}).then(() => console.log("Updated worker!"))