@budibase/worker 1.0.198 → 1.0.199-alpha.0

package/Dockerfile

@@ -21,4 +21,7 @@ EXPOSE 4001
 # which are actually needed to get this environment up and running
 ENV NODE_ENV=production
 ENV CLUSTER_MODE=${CLUSTER_MODE}
+ENV SERVICE=worker-service
+ENV POSTHOG_TOKEN=phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS
+
 CMD ["./docker_run.sh"]

package/nodemon.json

@@ -1,6 +1,6 @@
 {
   "watch": ["src", "../backend-core", "../../../budibase-pro/packages/pro"],
   "ext": "js,ts,json",
-  "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js"],
+  "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../backend-core/dist/**/*"],
   "exec": "ts-node src/index.ts"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "1.0.198",
+  "version": "1.0.199-alpha.0",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -12,8 +12,10 @@
     "budibase"
   ],
   "scripts": {
-    "build": "rimraf dist/ && tsc",
+    "prebuild": "rimraf dist/",
+    "build": "tsc -p tsconfig.build.json",
     "postbuild": "copyfiles -u 1 src/**/*.hbs dist/",
+    "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "run:docker": "node dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
     "build:docker": "docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
@@ -32,63 +34,71 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "^1.0.198",
-    "@budibase/pro": "1.0.197",
-    "@budibase/string-templates": "^1.0.198",
-    "@koa/router": "^8.0.0",
+    "@budibase/backend-core": "^1.0.199-alpha.0",
+    "@budibase/pro": "1.0.198",
+    "@budibase/string-templates": "^1.0.199-alpha.0",
+    "@koa/router": "8.0.8",
     "@sentry/node": "6.17.7",
-    "@techpass/passport-openidconnect": "^0.3.0",
-    "@types/global-agent": "^2.1.1",
-    "aws-sdk": "^2.811.0",
-    "bcryptjs": "^2.4.3",
-    "dotenv": "^8.2.0",
-    "global-agent": "^3.0.0",
-    "got": "^11.8.1",
-    "joi": "^17.4.0",
-    "koa": "^2.7.0",
-    "koa-body": "^4.2.0",
-    "koa-compress": "^4.0.1",
-    "koa-passport": "^4.1.4",
-    "koa-pino-logger": "^3.0.0",
-    "koa-send": "^5.0.0",
-    "koa-session": "^5.12.0",
-    "koa-static": "^5.0.0",
-    "node-fetch": "^2.6.1",
-    "nodemailer": "^6.5.0",
-    "passport-google-oauth": "^2.0.0",
-    "passport-jwt": "^4.0.0",
-    "passport-local": "^1.0.0",
-    "pino-pretty": "^4.0.0",
+    "@techpass/passport-openidconnect": "0.3.2",
+    "@types/global-agent": "2.1.1",
+    "aws-sdk": "2.1030.0",
+    "bcryptjs": "2.4.3",
+    "dotenv": "8.6.0",
+    "global-agent": "3.0.0",
+    "got": "11.8.3",
+    "joi": "17.6.0",
+    "koa": "2.13.4",
+    "koa-body": "4.2.0",
+    "koa-compress": "4.0.1",
+    "koa-passport": "4.1.4",
+    "koa-pino-logger": "3.0.0",
+    "koa-send": "5.0.1",
+    "koa-session": "5.13.1",
+    "koa-static": "5.0.0",
+    "node-fetch": "2.6.7",
+    "nodemailer": "6.7.2",
+    "passport-google-oauth": "2.0.0",
+    "passport-jwt": "4.0.0",
+    "passport-local": "1.0.0",
+    "pino-pretty": "4.8.0",
     "pouchdb": "7.3.0",
-    "pouchdb-all-dbs": "^1.0.2",
-    "server-destroy": "^1.0.1"
+    "pouchdb-all-dbs": "1.1.1",
+    "server-destroy": "1.0.1"
   },
   "devDependencies": {
-    "@types/jest": "^26.0.23",
-    "@types/koa": "^2.13.3",
-    "@types/koa-router": "^7.4.2",
-    "@types/koa__router": "^8.0.11",
-    "@types/node": "^15.12.4",
+    "@budibase/types": "^1.0.199-alpha.0",
+    "@types/jest": "26.0.23",
+    "@types/koa": "2.13.4",
+    "@types/koa-router": "7.4.4",
+    "@types/koa__router": "8.0.11",
+    "@types/node": "14.18.20",
     "@typescript-eslint/parser": "5.12.0",
-    "copyfiles": "^2.4.1",
-    "eslint": "^6.8.0",
-    "jest": "^27.0.5",
-    "nodemon": "^2.0.7",
-    "pouchdb-adapter-memory": "^7.2.2",
+    "copyfiles": "2.4.1",
+    "eslint": "6.8.0",
+    "jest": "27.4.7",
+    "nodemon": "2.0.15",
+    "pouchdb-adapter-memory": "7.2.2",
     "prettier": "2.3.1",
-    "rimraf": "^3.0.2",
-    "supertest": "^6.1.3",
-    "ts-jest": "^27.0.3",
-    "ts-node": "^10.0.0",
+    "rimraf": "3.0.2",
+    "supertest": "6.2.2",
+    "timekeeper": "2.2.0",
+    "ts-jest": "27.1.3",
+    "ts-node": "10.4.0",
+    "tsconfig-paths": "4.0.0",
     "typescript": "4.5.5",
-    "update-dotenv": "^1.1.1"
+    "update-dotenv": "1.1.1"
   },
   "jest": {
     "preset": "ts-jest",
     "testEnvironment": "node",
+    "moduleNameMapper": {
+      "@budibase/backend-core/(.*)": "<rootDir>/../backend-core/$1",
+      "@budibase/backend-core": "<rootDir>/../backend-core/src",
+      "@budibase/types": "<rootDir>/../types/src"
+    },
     "setupFiles": [
       "./scripts/jestSetup.js"
     ]
   },
-  "gitHead": "f16afcf7809998f9fca109e2199778af79c61e30"
+  "gitHead": "669d080a99dbec21584af40ad3b8fbc697051586"
 }

package/scripts/dev/manage.js

@@ -26,6 +26,8 @@ async function init() {
       ACCOUNT_PORTAL_API_KEY: "budibase",
       PLATFORM_URL: "http://localhost:10000",
       APPS_URL: "http://localhost:4001",
+      SERVICE: "worker-service",
+      DEPLOYMENT_ENVIRONMENT: "development",
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {

package/scripts/jestSetup.js

@@ -5,3 +5,12 @@ env._set("NODE_ENV", "jest")
 env._set("JWT_SECRET", "test-jwtsecret")
 env._set("LOG_LEVEL", "silent")
 env._set("MULTI_TENANCY", true)
+
+const { mocks } = require("@budibase/backend-core/tests")
+
+// mock all dates to 2020-01-01T00:00:00.000Z
+// use tk.reset() to use real dates in individual tests
+const tk = require("timekeeper")
+tk.freeze(mocks.date.MOCK_DATE)
+
+global.console.log = jest.fn() // console.log are ignored in tests

package/scripts/load/users.js

@@ -6,7 +6,7 @@ const {
   generateGlobalUserID,
 } = require("@budibase/backend-core/db")
 const { doInTenant, getGlobalDB } = require("@budibase/backend-core/tenancy")
-const { internalSaveUser } = require("@budibase/backend-core/utils")
+const users = require("../../src/sdk/users")
 const { publicApiUserFix } = require("../../src/utilities/users")
 const { hash } = require("@budibase/backend-core/utils")
 
@@ -73,15 +73,15 @@ async function run() {
           },
         })
         userSavePromises.push(
-          internalSaveUser(ctx.request.body, TENANT_ID, {
+          users.save(ctx.request.body, {
             hashPassword: false,
             requirePassword: true,
             bulkCreate: true,
           })
         )
       }
-      const users = await Promise.all(userSavePromises)
-      await db.bulkDocs(users)
+      const allUsers = await Promise.all(userSavePromises)
+      await db.bulkDocs(allUsers)
       console.log(`${i + BATCH_SIZE} users have been created.`)
     }
   })

package/src/api/controllers/global/auth.ts

@@ -4,14 +4,7 @@ const { google } = require("@budibase/backend-core/middleware")
 const { oidc } = require("@budibase/backend-core/middleware")
 const { Configs, EmailTemplatePurpose } = require("../../../constants")
 const { sendEmail, isEmailConfigured } = require("../../../utilities/email")
-const {
-  setCookie,
-  getCookie,
-  clearCookie,
-  getGlobalUserByEmail,
-  hash,
-  platformLogout,
-} = core.utils
+const { setCookie, getCookie, clearCookie, hash, platformLogout } = core.utils
 const { Cookies, Headers } = core.constants
 const { passport } = core.auth
 const { checkResetPasswordCode } = require("../../../utilities/redis")
@@ -21,7 +14,9 @@ const {
   isMultiTenant,
 } = require("@budibase/backend-core/tenancy")
 const env = require("../../../environment")
-import { users } from "@budibase/pro"
+import { events, users as usersCore, context } from "@budibase/backend-core"
+import { users } from "../../../sdk"
+import { User } from "@budibase/types"
 
 const ssoCallbackUrl = async (config: any, type: any) => {
   // incase there is a callback URL from before
@@ -75,8 +70,11 @@ async function authInternal(ctx: any, user: any, err = null, info = null) {
 export const authenticate = async (ctx: any, next: any) => {
   return passport.authenticate(
     "local",
-    async (err: any, user: any, info: any) => {
+    async (err: any, user: User, info: any) => {
       await authInternal(ctx, user, err, info)
+      await context.identity.doInUserContext(user, async () => {
+        await events.auth.login("local")
+      })
       ctx.status = 200
     }
   )(ctx, next)
@@ -110,13 +108,14 @@ export const reset = async (ctx: any) => {
     )
   }
   try {
-    const user = await getGlobalUserByEmail(email)
+    const user = (await usersCore.getGlobalUserByEmail(email)) as User
     // only if user exists, don't error though if they don't
     if (user) {
       await sendEmail(email, EmailTemplatePurpose.PASSWORD_RECOVERY, {
         user,
         subject: "{{ company }} platform password reset",
       })
+      await events.user.passwordResetRequested(user)
     }
   } catch (err) {
     console.log(err)
@@ -141,7 +140,11 @@ export const resetUpdate = async (ctx: any) => {
     ctx.body = {
       message: "password reset successfully.",
     }
+    // remove password from the user before sending events
+    delete user.password
+    await events.user.passwordReset(user)
   } catch (err) {
+    console.error(err)
     ctx.throw(400, "Cannot reset password.")
   }
 }
@@ -211,9 +214,11 @@ export const googleAuth = async (ctx: any, next: any) => {
   return passport.authenticate(
     strategy,
     { successRedirect: "/", failureRedirect: "/error" },
-    async (err: any, user: any, info: any) => {
+    async (err: any, user: User, info: any) => {
       await authInternal(ctx, user, err, info)
-
+      await context.identity.doInUserContext(user, async () => {
+        await events.auth.login("google-internal")
+      })
       ctx.redirect("/")
     }
   )(ctx, next)
@@ -257,7 +262,9 @@ export const oidcAuth = async (ctx: any, next: any) => {
     { successRedirect: "/", failureRedirect: "/error" },
     async (err: any, user: any, info: any) => {
       await authInternal(ctx, user, err, info)
-
+      await context.identity.doInUserContext(user, async () => {
+        await events.auth.login("oidc")
+      })
       ctx.redirect("/")
     }
   )(ctx, next)

package/src/api/controllers/global/configs.js

@@ -19,13 +19,132 @@ const {
   CacheKeys,
   bustCache,
 } = require("@budibase/backend-core/cache")
+const { events } = require("@budibase/backend-core")
 
 const BB_TENANT_CDN = "https://tenants.cdn.budi.live"
 
+const getEventFns = async (db, config) => {
+  const fns = []
+  const type = config.type
+
+  let existing
+  if (config._id) {
+    existing = await db.get(config._id)
+  }
+
+  if (!existing) {
+    switch (config.type) {
+      case Configs.SMTP: {
+        fns.push(events.email.SMTPCreated)
+        break
+      }
+      case Configs.GOOGLE: {
+        fns.push(() => events.auth.SSOCreated(type))
+        if (config.config.activated) {
+          fns.push(() => events.auth.SSOActivated(type))
+        }
+        break
+      }
+      case Configs.OIDC: {
+        fns.push(() => events.auth.SSOCreated(type))
+        if (config.config.configs[0].activated) {
+          fns.push(() => events.auth.SSOActivated(type))
+        }
+        break
+      }
+      case Configs.SETTINGS: {
+        // company
+        const company = config.config.company
+        if (company && company !== "Budibase") {
+          fns.push(events.org.nameUpdated)
+        }
+
+        // logo
+        const logoUrl = config.config.logoUrl
+        if (logoUrl) {
+          fns.push(events.org.logoUpdated)
+        }
+
+        // platform url
+        const platformUrl = config.config.platformUrl
+        if (
+          platformUrl &&
+          platformUrl !== "http://localhost:10000" &&
+          env.SELF_HOSTED
+        ) {
+          fns.push(events.org.platformURLUpdated)
+        }
+        break
+      }
+    }
+  } else {
+    switch (config.type) {
+      case Configs.SMTP: {
+        fns.push(events.email.SMTPUpdated)
+        break
+      }
+      case Configs.GOOGLE: {
+        fns.push(() => events.auth.SSOUpdated(type))
+        if (!existing.config.activated && config.config.activated) {
+          fns.push(() => events.auth.SSOActivated(type))
+        } else if (existing.config.activated && !config.config.activated) {
+          fns.push(() => events.auth.SSODeactivated(type))
+        }
+        break
+      }
+      case Configs.OIDC: {
+        fns.push(() => events.auth.SSOUpdated(type))
+        if (
+          !existing.config.configs[0].activated &&
+          config.config.configs[0].activated
+        ) {
+          fns.push(() => events.auth.SSOActivated(type))
+        } else if (
+          existing.config.configs[0].activated &&
+          !config.config.configs[0].activated
+        ) {
+          fns.push(() => events.auth.SSODeactivated(type))
+        }
+        break
+      }
+      case Configs.SETTINGS: {
+        // company
+        const existingCompany = existing.config.company
+        const company = config.config.company
+        if (company && company !== "Budibase" && existingCompany !== company) {
+          fns.push(events.org.nameUpdated)
+        }
+
+        // logo
+        const existingLogoUrl = existing.config.logoUrl
+        const logoUrl = config.config.logoUrl
+        if (logoUrl && existingLogoUrl !== logoUrl) {
+          fns.push(events.org.logoUpdated)
+        }
+
+        // platform url
+        const existingPlatformUrl = existing.config.platformUrl
+        const platformUrl = config.config.platformUrl
+        if (
+          platformUrl &&
+          platformUrl !== "http://localhost:10000" &&
+          existingPlatformUrl !== platformUrl &&
+          env.SELF_HOSTED
+        ) {
+          fns.push(events.org.platformURLUpdated)
+        }
+        break
+      }
+    }
+  }
+
+  return fns
+}
+
 exports.save = async function (ctx) {
   const db = getGlobalDB()
   const { type, workspace, user, config } = ctx.request.body
-
+  let eventFns = await getEventFns(db, ctx.request.body)
   // Config does not exist yet
   if (!ctx.request.body._id) {
     ctx.request.body._id = generateConfigID({
@@ -34,7 +153,6 @@ exports.save = async function (ctx) {
       user,
     })
   }
-
   try {
     // verify the configuration
     switch (type) {
@@ -49,6 +167,12 @@ exports.save = async function (ctx) {
   try {
     const response = await db.put(ctx.request.body)
     await bustCache(CacheKeys.CHECKLIST)
+    await bustCache(CacheKeys.ANALYTICS_ENABLED)
+
+    for (const fn of eventFns) {
+      await fn()
+    }
+
     ctx.body = {
       type,
       _id: response.id,

package/src/api/controllers/global/roles.js

@@ -7,7 +7,7 @@ const {
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
 const { user: userCache } = require("@budibase/backend-core/cache")
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const { allUsers } = require("../../utilities")
+const { users } = require("../../../sdk")
 
 exports.fetch = async ctx => {
   const tenantId = ctx.user.tenantId
@@ -49,10 +49,10 @@ exports.find = async ctx => {
 exports.removeAppRole = async ctx => {
   const { appId } = ctx.params
   const db = getGlobalDB()
-  const users = await allUsers(ctx)
+  const allUsers = await users.allUsers(ctx)
   const bulk = []
   const cacheInvalidations = []
-  for (let user of users) {
+  for (let user of allUsers) {
     if (user.roles[appId]) {
       cacheInvalidations.push(userCache.invalidateUser(user._id))
       delete user.roles[appId]

package/src/api/controllers/global/self.js

@@ -13,9 +13,9 @@ const {
 } = require("@budibase/backend-core/utils")
 const { encrypt } = require("@budibase/backend-core/encryption")
 const { newid } = require("@budibase/backend-core/utils")
-const { getUser } = require("../../utilities")
+const { users } = require("../../../sdk")
 const { Cookies } = require("@budibase/backend-core/constants")
-const { featureFlags } = require("@budibase/backend-core")
+const { events, featureFlags } = require("@budibase/backend-core")
 
 function newApiKey() {
   return encrypt(`${getTenantId()}${SEPARATOR}${newid()}`)
@@ -104,7 +104,7 @@ exports.getSelf = async ctx => {
   checkCurrentApp(ctx)
 
   // get the main body of the user
-  ctx.body = await getUser(userId)
+  ctx.body = await users.getUser(userId)
 
   // add the feature flags for this tenant
   const tenantId = getTenantId()
@@ -116,8 +116,10 @@ exports.getSelf = async ctx => {
 exports.updateSelf = async ctx => {
   const db = getGlobalDB()
   const user = await db.get(ctx.user._id)
+  let passwordChange = false
   if (ctx.request.body.password) {
     // changing password
+    passwordChange = true
     ctx.request.body.password = await hash(ctx.request.body.password)
     // Log all other sessions out apart from the current one
     await platformLogout({
@@ -140,4 +142,11 @@ exports.updateSelf = async ctx => {
     _id: response.id,
     _rev: response.rev,
   }
+
+  // remove the old password from the user before sending events
+  delete user.password
+  await events.user.updated(user)
+  if (passwordChange) {
+    await events.user.passwordUpdated(user)
+  }
 }