@budibase/worker 2.3.18-alpha.14 → 2.3.18-alpha.16

package/jest.config.ts

@@ -7,7 +7,7 @@ const config: Config.InitialOptions = {
   preset: "@trendyol/jest-testcontainers",
   setupFiles: ["./src/tests/jestEnv.ts"],
   setupFilesAfterEnv: ["./src/tests/jestSetup.ts"],
-  collectCoverageFrom: ["src/**/*.{js,ts}"],
+  collectCoverageFrom: ["src/**/*.{js,ts}", "../backend-core/src/**/*.{js,ts}"],
   coverageReporters: ["lcov", "json", "clover"],
   transform: {
     "^.+\\.ts?$": "@swc/jest",

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "2.3.18-alpha.14",
+  "version": "2.3.18-alpha.16",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -36,10 +36,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "2.3.18-alpha.14",
-    "@budibase/pro": "2.3.18-alpha.13",
-    "@budibase/string-templates": "2.3.18-alpha.14",
-    "@budibase/types": "2.3.18-alpha.14",
+    "@budibase/backend-core": "2.3.18-alpha.16",
+    "@budibase/pro": "2.3.18-alpha.15",
+    "@budibase/string-templates": "2.3.18-alpha.16",
+    "@budibase/types": "2.3.18-alpha.16",
     "@koa/router": "8.0.8",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "0.3.2",
@@ -60,6 +60,7 @@
     "koa-send": "5.0.1",
     "koa-session": "5.13.1",
     "koa-static": "5.0.0",
+    "koa-useragent": "^4.1.0",
     "node-fetch": "2.6.7",
     "nodemailer": "6.7.2",
     "passport-google-oauth": "2.0.0",
@@ -100,5 +101,5 @@
     "typescript": "4.7.3",
     "update-dotenv": "1.1.1"
   },
-  "gitHead": "6303105b71c3ef85e5bc8a89a7fd763cf55179e4"
+  "gitHead": "c350c15ae0869875936c804b89fcd46e4da4d8bb"
 }

package/src/api/controllers/global/auth.ts

@@ -69,8 +69,8 @@ export const login = async (ctx: Ctx<LoginRequest>, next: any) => {
     "local",
     async (err: any, user: User, info: any) => {
       await passportCallback(ctx, user, err, info)
-      await context.identity.doInUserContext(user, async () => {
-        await events.auth.login("local")
+      await context.identity.doInUserContext(user, ctx, async () => {
+        await events.auth.login("local", user.email)
       })
       ctx.status = 200
     }
@@ -207,8 +207,8 @@ export const googleCallback = async (ctx: any, next: any) => {
     { successRedirect: "/", failureRedirect: "/error" },
     async (err: any, user: SSOUser, info: any) => {
       await passportCallback(ctx, user, err, info)
-      await context.identity.doInUserContext(user, async () => {
-        await events.auth.login("google-internal")
+      await context.identity.doInUserContext(user, ctx, async () => {
+        await events.auth.login("google-internal", user.email)
       })
       ctx.redirect("/")
     }
@@ -272,8 +272,8 @@ export const oidcCallback = async (ctx: any, next: any) => {
     { successRedirect: "/", failureRedirect: "/error" },
     async (err: any, user: SSOUser, info: any) => {
       await passportCallback(ctx, user, err, info)
-      await context.identity.doInUserContext(user, async () => {
-        await events.auth.login("oidc")
+      await context.identity.doInUserContext(user, ctx, async () => {
+        await events.auth.login("oidc", user.email)
       })
       ctx.redirect("/")
     }

package/src/api/controllers/global/configs.ts

@@ -17,10 +17,15 @@ import {
   Ctx,
   GetPublicOIDCConfigResponse,
   GetPublicSettingsResponse,
+  GoogleInnerConfig,
   isGoogleConfig,
   isOIDCConfig,
   isSettingsConfig,
   isSMTPConfig,
+  OIDCConfigs,
+  SettingsInnerConfig,
+  SSOConfig,
+  SSOConfigType,
   UserCtx,
 } from "@budibase/types"
 import * as pro from "@budibase/pro"
@@ -119,6 +124,61 @@ const getEventFns = async (config: Config, existing?: Config) => {
   return fns
 }
 
+type SSOConfigs = { [key in SSOConfigType]: SSOConfig | undefined }
+
+async function getSSOConfigs(): Promise<SSOConfigs> {
+  const google = await configs.getGoogleConfig()
+  const oidc = await configs.getOIDCConfig()
+  return {
+    [ConfigType.GOOGLE]: google,
+    [ConfigType.OIDC]: oidc,
+  }
+}
+
+async function hasActivatedConfig(ssoConfigs?: SSOConfigs) {
+  if (!ssoConfigs) {
+    ssoConfigs = await getSSOConfigs()
+  }
+  return !!Object.values(ssoConfigs).find(c => c?.activated)
+}
+
+async function verifySettingsConfig(config: SettingsInnerConfig) {
+  if (config.isSSOEnforced) {
+    const valid = await hasActivatedConfig()
+    if (!valid) {
+      throw new Error("Cannot enforce SSO without an activated configuration")
+    }
+  }
+}
+
+async function verifySSOConfig(type: SSOConfigType, config: SSOConfig) {
+  const settings = await configs.getSettingsConfig()
+  if (settings.isSSOEnforced && !config.activated) {
+    // config is being saved as deactivated
+    // ensure there is at least one other activated sso config
+    const ssoConfigs = await getSSOConfigs()
+
+    // overwrite the config being updated
+    // to reflect the desired state
+    ssoConfigs[type] = config
+
+    const activated = await hasActivatedConfig(ssoConfigs)
+    if (!activated) {
+      throw new Error(
+        "Configuration cannot be deactivated while SSO is enforced"
+      )
+    }
+  }
+}
+
+async function verifyGoogleConfig(config: GoogleInnerConfig) {
+  await verifySSOConfig(ConfigType.GOOGLE, config)
+}
+
+async function verifyOIDCConfig(config: OIDCConfigs) {
+  await verifySSOConfig(ConfigType.OIDC, config.configs[0])
+}
+
 export async function save(ctx: UserCtx<Config>) {
   const body = ctx.request.body
   const type = body.type
@@ -133,10 +193,19 @@ export async function save(ctx: UserCtx<Config>) {
 
   try {
     // verify the configuration
-    switch (config.type) {
+    switch (type) {
       case ConfigType.SMTP:
         await email.verifyConfig(config)
         break
+      case ConfigType.SETTINGS:
+        await verifySettingsConfig(config)
+        break
+      case ConfigType.GOOGLE:
+        await verifyGoogleConfig(config)
+        break
+      case ConfigType.OIDC:
+        await verifyOIDCConfig(config)
+        break
     }
   } catch (err: any) {
     ctx.throw(400, err)

package/src/api/routes/global/configs.ts

@@ -34,8 +34,8 @@ function settingValidation() {
 function googleValidation() {
   // prettier-ignore
   return Joi.object({
-    clientID: Joi.when('activated', { is: true, then: Joi.string().required() }),
-    clientSecret: Joi.when('activated', { is: true, then: Joi.string().required() }),
+    clientID: Joi.string().required(),
+    clientSecret: Joi.string().required(),
     activated: Joi.boolean().required(),
   }).unknown(true)
 }
@@ -45,12 +45,12 @@ function oidcValidation() {
   return Joi.object({
     configs: Joi.array().items(
       Joi.object({
-        clientID: Joi.when('activated', { is: true, then: Joi.string().required() }),
-        clientSecret: Joi.when('activated', { is: true, then: Joi.string().required() }),
-        configUrl: Joi.when('activated', { is: true, then: Joi.string().required() }),
+        clientID: Joi.string().required(),
+        clientSecret: Joi.string().required(),
+        configUrl: Joi.string().required(),
         logo: Joi.string().allow("", null),
         name: Joi.string().allow("", null),
-        uuid: Joi.when('activated', { is: true, then: Joi.string().required() }),
+        uuid: Joi.string().required(),
         activated: Joi.boolean().required(),
         scopes: Joi.array().optional()
       })

package/src/api/routes/global/tests/auditLogs.spec.ts

@@ -0,0 +1,111 @@
+import { mocks, structures } from "@budibase/backend-core/tests"
+import { context, events } from "@budibase/backend-core"
+import { Event, IdentityType } from "@budibase/types"
+import { TestConfiguration } from "../../../../tests"
+
+mocks.licenses.useAuditLogs()
+
+const BASE_IDENTITY = {
+  account: undefined,
+  type: IdentityType.USER,
+}
+const USER_AUDIT_LOG_COUNT = 3
+const APP_ID = "app_1"
+
+describe("/api/global/auditlogs", () => {
+  const config = new TestConfiguration()
+
+  beforeAll(async () => {
+    await config.beforeAll()
+  })
+
+  afterAll(async () => {
+    await config.afterAll()
+  })
+
+  describe("POST /api/global/auditlogs/search", () => {
+    it("should be able to fire some events (create audit logs)", async () => {
+      await context.doInTenant(config.tenantId, async () => {
+        const userId = config.user!._id!
+        const identity = {
+          ...BASE_IDENTITY,
+          _id: userId,
+          tenantId: config.tenantId,
+        }
+        await context.doInIdentityContext(identity, async () => {
+          for (let i = 0; i < USER_AUDIT_LOG_COUNT; i++) {
+            await events.user.created(structures.users.user())
+          }
+          await context.doInAppContext(APP_ID, async () => {
+            await events.app.created(structures.apps.app(APP_ID))
+          })
+          // fetch the user created events
+          const response = await config.api.auditLogs.search({
+            events: [Event.USER_CREATED],
+          })
+          expect(response.data).toBeDefined()
+          // there will be an initial event which comes from the default user creation
+          expect(response.data.length).toBe(USER_AUDIT_LOG_COUNT + 1)
+        })
+      })
+    })
+
+    it("should be able to search by event", async () => {
+      const response = await config.api.auditLogs.search({
+        events: [Event.USER_CREATED],
+      })
+      expect(response.data.length).toBeGreaterThan(0)
+      for (let log of response.data) {
+        expect(log.event).toBe(Event.USER_CREATED)
+      }
+    })
+
+    it("should be able to search by time range (frozen)", async () => {
+      // this is frozen, only need to add 1 and minus 1
+      const now = new Date()
+      const start = new Date()
+      start.setSeconds(now.getSeconds() - 1)
+      const end = new Date()
+      end.setSeconds(now.getSeconds() + 1)
+      const response = await config.api.auditLogs.search({
+        startDate: start.toISOString(),
+        endDate: end.toISOString(),
+      })
+      expect(response.data.length).toBeGreaterThan(0)
+      for (let log of response.data) {
+        expect(log.timestamp).toBe(now.toISOString())
+      }
+    })
+
+    it("should be able to search by user ID", async () => {
+      const userId = config.user!._id!
+      const response = await config.api.auditLogs.search({
+        userIds: [userId],
+      })
+      expect(response.data.length).toBeGreaterThan(0)
+      for (let log of response.data) {
+        expect(log.user._id).toBe(userId)
+      }
+    })
+
+    it("should be able to search by app ID", async () => {
+      const response = await config.api.auditLogs.search({
+        appIds: [APP_ID],
+      })
+      expect(response.data.length).toBeGreaterThan(0)
+      for (let log of response.data) {
+        expect(log.app?._id).toBe(APP_ID)
+      }
+    })
+
+    it("should be able to search by full string", async () => {
+      const response = await config.api.auditLogs.search({
+        fullSearch: "User",
+      })
+      expect(response.data.length).toBeGreaterThan(0)
+      for (let log of response.data) {
+        expect(log.name.includes("User")).toBe(true)
+      }
+    })
+  })
+})

package/src/api/routes/global/tests/auth.spec.ts

@@ -367,7 +367,7 @@ describe("/api/global/auth", () => {
 
         const res = await config.api.configs.OIDCCallback(configId, preAuthRes)
 
-        expect(events.auth.login).toBeCalledWith("oidc")
+        expect(events.auth.login).toBeCalledWith("oidc", "oauth@example.com")
         expect(events.auth.login).toBeCalledTimes(1)
         expect(res.status).toBe(302)
         const location: string = res.get("location")

package/src/api/routes/index.ts

@@ -18,6 +18,8 @@ import accountRoutes from "./system/accounts"
 import restoreRoutes from "./system/restore"
 
 let userGroupRoutes = api.groups
+let auditLogRoutes = api.auditLogs
+
 export const routes: Router[] = [
   configRoutes,
   userRoutes,
@@ -32,6 +34,7 @@ export const routes: Router[] = [
   selfRoutes,
   licenseRoutes,
   userGroupRoutes,
+  auditLogRoutes,
   migrationRoutes,
   accountRoutes,
   restoreRoutes,

package/src/db/index.ts

@@ -1,10 +1,16 @@
 import * as core from "@budibase/backend-core"
 import env from "../environment"
 
-export const init = () => {
-  const dbConfig: any = {}
+export function init() {
+  const dbConfig: any = {
+    replication: true,
+    find: true,
+  }
+
   if (env.isTest() && !env.COUCH_DB_URL) {
     dbConfig.inMemory = true
+    dbConfig.allDbs = true
   }
+
   core.init({ db: dbConfig })
 }

package/src/index.ts

@@ -13,11 +13,13 @@ import { Event } from "@sentry/types/dist/event"
 import Application from "koa"
 import { bootstrap } from "global-agent"
 import * as db from "./db"
+import { sdk as proSdk } from "@budibase/pro"
 import {
   auth,
   logging,
   events,
   middleware,
+  queue,
   env as coreEnv,
 } from "@budibase/backend-core"
 db.init()
@@ -29,8 +31,14 @@ import * as redis from "./utilities/redis"
 const Sentry = require("@sentry/node")
 const koaSession = require("koa-session")
 const logger = require("koa-pino-logger")
+const { userAgent } = require("koa-useragent")
+
 import destroyable from "server-destroy"
 
+// configure events to use the pro audit log write
+// can't integrate directly into backend-core due to cyclic issues
+events.processors.init(proSdk.auditLogs.write)
+
 if (coreEnv.ENABLE_SSO_MAINTENANCE_MODE) {
   console.warn(
     "Warning: ENABLE_SSO_MAINTENANCE_MODE is set. It is recommended this flag is disabled if maintenance is not in progress"
@@ -49,6 +57,7 @@ app.use(koaBody({ multipart: true }))
 app.use(koaSession(app))
 app.use(middleware.logging)
 app.use(logger(logging.pinoSettings()))
+app.use(userAgent)
 
 // authentication
 app.use(auth.passport.initialize())
@@ -84,6 +93,7 @@ server.on("close", async () => {
   console.log("Server Closed")
   await redis.shutdown()
   await events.shutdown()
+  await queue.shutdown()
   if (!env.isTest()) {
     process.exit(errCode)
   }

package/src/sdk/users/users.ts

@@ -637,7 +637,7 @@ export const invite = async (
       }
       await sendEmail(user.email, EmailTemplatePurpose.INVITATION, opts)
       response.successful.push({ email: user.email })
-      await events.user.invited()
+      await events.user.invited(user.email)
     } catch (e) {
       console.error(`Failed to send email invitation email=${user.email}`, e)
       response.unsuccessful.push({

package/src/tests/api/auditLogs.ts

@@ -0,0 +1,26 @@
+import { AuditLogSearchParams, SearchAuditLogsResponse } from "@budibase/types"
+import TestConfiguration from "../TestConfiguration"
+import { TestAPI } from "./base"
+
+export class AuditLogAPI extends TestAPI {
+  constructor(config: TestConfiguration) {
+    super(config)
+  }
+
+  search = async (search: AuditLogSearchParams) => {
+    const res = await this.request
+      .post("/api/global/auditlogs/search")
+      .send(search)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+    return res.body as SearchAuditLogsResponse
+  }
+
+  download = (search: AuditLogSearchParams) => {
+    const query = encodeURIComponent(JSON.stringify(search))
+    return this.request
+      .get(`/api/global/auditlogs/download?query=${query}`)
+      .set(this.config.defaultHeaders())
+  }
+}

package/src/tests/api/index.ts

@@ -14,6 +14,7 @@ import { GroupsAPI } from "./groups"
 import { RolesAPI } from "./roles"
 import { TemplatesAPI } from "./templates"
 import { LicenseAPI } from "./license"
+import { AuditLogAPI } from "./auditLogs"
 export default class API {
   accounts: AccountAPI
   auth: AuthAPI
@@ -30,6 +31,7 @@ export default class API {
   roles: RolesAPI
   templates: TemplatesAPI
   license: LicenseAPI
+  auditLogs: AuditLogAPI
 
   constructor(config: TestConfiguration) {
     this.accounts = new AccountAPI(config)
@@ -47,5 +49,6 @@ export default class API {
     this.roles = new RolesAPI(config)
     this.templates = new TemplatesAPI(config)
     this.license = new LicenseAPI(config)
+    this.auditLogs = new AuditLogAPI(config)
   }
 }

package/src/utilities/email.ts

@@ -203,7 +203,7 @@ export async function sendEmail(
  * @param {object} config an SMTP configuration - this is based on the nodemailer API.
  * @return {Promise<boolean>} returns true if the configuration is valid.
  */
-export async function verifyConfig(config: any) {
+export async function verifyConfig(config: SMTPInnerConfig) {
   const transport = createSMTPTransport(config)
   await transport.verify()
 }