@budibase/worker 1.0.123 → 1.0.125

package/nodemon.json

@@ -1,5 +1,5 @@
 {
-  "watch": ["src", "../backend-core"],
+  "watch": ["src", "../backend-core", "../../../budibase-pro/packages/pro"],
   "ext": "js,ts,json",
   "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js"],
   "exec": "ts-node src/index.ts"

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "1.0.123",
+  "version": "1.0.125",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -31,14 +31,17 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "^1.0.123",
-    "@budibase/string-templates": "^1.0.123",
+    "@budibase/backend-core": "^1.0.125",
+    "@budibase/pro": "1.0.124-alpha.0",
+    "@budibase/string-templates": "^1.0.125",
     "@koa/router": "^8.0.0",
-    "@sentry/node": "^6.0.0",
+    "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "^0.3.0",
+    "@types/global-agent": "^2.1.1",
     "aws-sdk": "^2.811.0",
     "bcryptjs": "^2.4.3",
     "dotenv": "^8.2.0",
+    "global-agent": "^3.0.0",
     "got": "^11.8.1",
     "joi": "^17.4.0",
     "koa": "^2.7.0",
@@ -63,7 +66,9 @@
     "@types/jest": "^26.0.23",
     "@types/koa": "^2.13.3",
     "@types/koa-router": "^7.4.2",
+    "@types/koa__router": "^8.0.11",
     "@types/node": "^15.12.4",
+    "@typescript-eslint/parser": "5.12.0",
     "copyfiles": "^2.4.1",
     "eslint": "^6.8.0",
     "jest": "^27.0.5",
@@ -74,7 +79,7 @@
     "supertest": "^6.1.3",
     "ts-jest": "^27.0.3",
     "ts-node": "^10.0.0",
-    "typescript": "4.3.5",
+    "typescript": "4.5.5",
     "update-dotenv": "^1.1.1"
   },
   "jest": {
@@ -84,5 +89,5 @@
       "./scripts/jestSetup.js"
     ]
   },
-  "gitHead": "db99c31b53d0dbbb788335126e45d1cc2f39fbf0"
+  "gitHead": "e96d44823de430b3f487705d18d34954f8fb9bf7"
 }

package/src/api/controllers/global/{auth.js → auth.ts}

@@ -21,8 +21,9 @@ const {
   isMultiTenant,
 } = require("@budibase/backend-core/tenancy")
 const env = require("../../../environment")
+import { users } from "@budibase/pro"
 
-const ssoCallbackUrl = async (config, type) => {
+const ssoCallbackUrl = async (config: any, type: any) => {
   // incase there is a callback URL from before
   if (config && config.callbackURL) {
     return config.callbackURL
@@ -42,15 +43,15 @@ const ssoCallbackUrl = async (config, type) => {
   return `${publicConfig.platformUrl}${callbackUrl}`
 }
 
-exports.googleCallbackUrl = async config => {
+export const googleCallbackUrl = async (config: any) => {
   return ssoCallbackUrl(config, "google")
 }
 
-exports.oidcCallbackUrl = async config => {
+export const oidcCallbackUrl = async (config: any) => {
   return ssoCallbackUrl(config, "oidc")
 }
 
-async function authInternal(ctx, user, err = null, info = null) {
+async function authInternal(ctx: any, user: any, err = null, info = null) {
   if (err) {
     console.error("Authentication error", err)
     return ctx.throw(403, info ? info : "Unauthorized")
@@ -71,20 +72,23 @@ async function authInternal(ctx, user, err = null, info = null) {
   }
 }
 
-exports.authenticate = async (ctx, next) => {
-  return passport.authenticate("local", async (err, user, info) => {
-    await authInternal(ctx, user, err, info)
-    ctx.status = 200
-  })(ctx, next)
+export const authenticate = async (ctx: any, next: any) => {
+  return passport.authenticate(
+    "local",
+    async (err: any, user: any, info: any) => {
+      await authInternal(ctx, user, err, info)
+      ctx.status = 200
+    }
+  )(ctx, next)
 }
 
-exports.setInitInfo = ctx => {
+export const setInitInfo = (ctx: any) => {
   const initInfo = ctx.request.body
   setCookie(ctx, initInfo, Cookies.Init)
   ctx.status = 200
 }
 
-exports.getInitInfo = ctx => {
+export const getInitInfo = (ctx: any) => {
   try {
     ctx.body = getCookie(ctx, Cookies.Init) || {}
   } catch (err) {
@@ -96,7 +100,7 @@ exports.getInitInfo = ctx => {
 /**
  * Reset the user password, used as part of a forgotten password flow.
  */
-exports.reset = async ctx => {
+export const reset = async (ctx: any) => {
   const { email } = ctx.request.body
   const configured = await isEmailConfigured()
   if (!configured) {
@@ -126,7 +130,7 @@ exports.reset = async ctx => {
 /**
  * Perform the user password update if the provided reset code is valid.
  */
-exports.resetUpdate = async ctx => {
+export const resetUpdate = async (ctx: any) => {
   const { resetCode, password } = ctx.request.body
   try {
     const { userId } = await checkResetPasswordCode(resetCode)
@@ -142,14 +146,14 @@ exports.resetUpdate = async ctx => {
   }
 }
 
-exports.logout = async ctx => {
+export const logout = async (ctx: any) => {
   if (ctx.user && ctx.user._id) {
     await platformLogout({ ctx, userId: ctx.user._id })
   }
   ctx.body = { message: "User logged out." }
 }
 
-exports.datasourcePreAuth = async (ctx, next) => {
+export const datasourcePreAuth = async (ctx: any, next: any) => {
   const provider = ctx.params.provider
   const middleware = require(`@budibase/backend-core/middleware`)
   const handler = middleware.datasource[provider]
@@ -167,7 +171,7 @@ exports.datasourcePreAuth = async (ctx, next) => {
   return handler.preAuth(passport, ctx, next)
 }
 
-exports.datasourceAuth = async (ctx, next) => {
+export const datasourceAuth = async (ctx: any, next: any) => {
   const authStateCookie = getCookie(ctx, Cookies.DatasourceAuth)
   const provider = authStateCookie.provider
   const middleware = require(`@budibase/backend-core/middleware`)
@@ -179,7 +183,7 @@ exports.datasourceAuth = async (ctx, next) => {
  * The initial call that google authentication makes to take you to the google login screen.
  * On a successful login, you will be redirected to the googleAuth callback route.
  */
-exports.googlePreAuth = async (ctx, next) => {
+export const googlePreAuth = async (ctx: any, next: any) => {
   const db = getGlobalDB()
 
   const config = await core.db.getScopedConfig(db, {
@@ -187,14 +191,14 @@ exports.googlePreAuth = async (ctx, next) => {
     workspace: ctx.query.workspace,
   })
   let callbackUrl = await exports.googleCallbackUrl(config)
-  const strategy = await google.strategyFactory(config, callbackUrl)
+  const strategy = await google.strategyFactory(config, callbackUrl, users.save)
 
   return passport.authenticate(strategy, {
     scope: ["profile", "email"],
   })(ctx, next)
 }
 
-exports.googleAuth = async (ctx, next) => {
+export const googleAuth = async (ctx: any, next: any) => {
   const db = getGlobalDB()
 
   const config = await core.db.getScopedConfig(db, {
@@ -202,12 +206,12 @@ exports.googleAuth = async (ctx, next) => {
     workspace: ctx.query.workspace,
   })
   const callbackUrl = await exports.googleCallbackUrl(config)
-  const strategy = await google.strategyFactory(config, callbackUrl)
+  const strategy = await google.strategyFactory(config, callbackUrl, users.save)
 
   return passport.authenticate(
     strategy,
     { successRedirect: "/", failureRedirect: "/error" },
-    async (err, user, info) => {
+    async (err: any, user: any, info: any) => {
       await authInternal(ctx, user, err, info)
 
       ctx.redirect("/")
@@ -215,24 +219,24 @@ exports.googleAuth = async (ctx, next) => {
   )(ctx, next)
 }
 
-async function oidcStrategyFactory(ctx, configId) {
+async function oidcStrategyFactory(ctx: any, configId: any) {
   const db = getGlobalDB()
   const config = await core.db.getScopedConfig(db, {
     type: Configs.OIDC,
     group: ctx.query.group,
   })
 
-  const chosenConfig = config.configs.filter(c => c.uuid === configId)[0]
+  const chosenConfig = config.configs.filter((c: any) => c.uuid === configId)[0]
   let callbackUrl = await exports.oidcCallbackUrl(chosenConfig)
 
-  return oidc.strategyFactory(chosenConfig, callbackUrl)
+  return oidc.strategyFactory(chosenConfig, callbackUrl, users.save)
 }
 
 /**
  * The initial call that OIDC authentication makes to take you to the configured OIDC login screen.
  * On a successful login, you will be redirected to the oidcAuth callback route.
  */
-exports.oidcPreAuth = async (ctx, next) => {
+export const oidcPreAuth = async (ctx: any, next: any) => {
   const { configId } = ctx.params
   const strategy = await oidcStrategyFactory(ctx, configId)
 
@@ -244,14 +248,14 @@ exports.oidcPreAuth = async (ctx, next) => {
   })(ctx, next)
 }
 
-exports.oidcAuth = async (ctx, next) => {
+export const oidcAuth = async (ctx: any, next: any) => {
   const configId = getCookie(ctx, Cookies.OIDC_CONFIG)
   const strategy = await oidcStrategyFactory(ctx, configId)
 
   return passport.authenticate(
     strategy,
     { successRedirect: "/", failureRedirect: "/error" },
-    async (err, user, info) => {
+    async (err: any, user: any, info: any) => {
       await authInternal(ctx, user, err, info)
 
       ctx.redirect("/")

package/src/api/controllers/global/license.ts

@@ -0,0 +1,30 @@
+import { licensing, quotas } from "@budibase/pro"
+
+export const activate = async (ctx: any) => {
+  const { licenseKey } = ctx.request.body
+  if (!licenseKey) {
+    ctx.throw(400, "licenseKey is required")
+  }
+
+  await licensing.activateLicenseKey(licenseKey)
+  ctx.status = 200
+}
+
+export const refresh = async (ctx: any) => {
+  await licensing.cache.refresh()
+  ctx.status = 200
+}
+
+export const getInfo = async (ctx: any) => {
+  const licenseInfo = await licensing.getLicenseInfo()
+  if (licenseInfo) {
+    licenseInfo.licenseKey = "*"
+    ctx.body = licenseInfo
+  }
+  ctx.status = 200
+}
+
+export const getQuotaUsage = async (ctx: any) => {
+  const usage = await quotas.getQuotaUsage()
+  ctx.body = usage
+}

package/src/api/controllers/global/self.js

@@ -15,6 +15,7 @@ const { encrypt } = require("@budibase/backend-core/encryption")
 const { newid } = require("@budibase/backend-core/utils")
 const { getUser } = require("../../utilities")
 const { Cookies } = require("@budibase/backend-core/constants")
+const { featureFlags } = require("@budibase/backend-core")
 
 function newApiKey() {
   return encrypt(`${getTenantId()}${SEPARATOR}${newid()}`)
@@ -68,6 +69,29 @@ const checkCurrentApp = ctx => {
   }
 }
 
+/**
+ * Add the attributes that are session based to the current user.
+ */
+const addSessionAttributesToUser = ctx => {
+  ctx.body.account = ctx.user.account
+  ctx.body.license = ctx.user.license
+  ctx.body.budibaseAccess = ctx.user.budibaseAccess
+  ctx.body.accountPortalAccess = ctx.user.accountPortalAccess
+  ctx.body.csrfToken = ctx.user.csrfToken
+}
+
+/**
+ * Remove the attributes that are session based from the current user,
+ * so that stale values are not written to the db
+ */
+const removeSessionAttributesFromUser = ctx => {
+  delete ctx.request.body.csrfToken
+  delete ctx.request.body.account
+  delete ctx.request.body.accountPortalAccess
+  delete ctx.request.body.budibaseAccess
+  delete ctx.request.body.license
+}
+
 exports.getSelf = async ctx => {
   if (!ctx.user) {
     ctx.throw(403, "User not logged in")
@@ -81,11 +105,12 @@ exports.getSelf = async ctx => {
 
   // get the main body of the user
   ctx.body = await getUser(userId)
-  // forward session information not found in db
-  ctx.body.account = ctx.user.account
-  ctx.body.budibaseAccess = ctx.user.budibaseAccess
-  ctx.body.accountPortalAccess = ctx.user.accountPortalAccess
-  ctx.body.csrfToken = ctx.user.csrfToken
+
+  // add the feature flags for this tenant
+  const tenantId = getTenantId()
+  ctx.body.featureFlags = featureFlags.getTenantFeatureFlags(tenantId)
+
+  addSessionAttributesToUser(ctx)
 }
 
 exports.updateSelf = async ctx => {
@@ -104,8 +129,8 @@ exports.updateSelf = async ctx => {
   // don't allow sending up an ID/Rev, always use the existing one
   delete ctx.request.body._id
   delete ctx.request.body._rev
-  // don't allow setting the csrf token
-  delete ctx.request.body.csrfToken
+  removeSessionAttributesFromUser(ctx)
+
   const response = await db.put({
     ...user,
     ...ctx.request.body,

package/src/api/controllers/global/{users.js → users.ts}

@@ -1,15 +1,11 @@
 const {
   getGlobalUserParams,
   StaticDatabases,
-  generateNewUsageQuotaDoc,
 } = require("@budibase/backend-core/db")
-const {
-  getGlobalUserByEmail,
-  saveUser,
-} = require("@budibase/backend-core/utils")
-const { EmailTemplatePurpose } = require("../../../constants")
-const { checkInviteCode } = require("../../../utilities/redis")
-const { sendEmail } = require("../../../utilities/email")
+const { getGlobalUserByEmail } = require("@budibase/backend-core/utils")
+import { EmailTemplatePurpose } from "../../../constants"
+import { checkInviteCode } from "../../../utilities/redis"
+import { sendEmail } from "../../../utilities/email"
 const { user: userCache } = require("@budibase/backend-core/cache")
 const { invalidateSessions } = require("@budibase/backend-core/sessions")
 const accounts = require("@budibase/backend-core/accounts")
@@ -21,26 +17,28 @@ const {
   doesTenantExist,
 } = require("@budibase/backend-core/tenancy")
 const { removeUserFromInfoDB } = require("@budibase/backend-core/deprovision")
-const env = require("../../../environment")
-const { syncUserInApps } = require("../../../utilities/appService")
-const { allUsers, getUser } = require("../../utilities")
+import env from "../../../environment"
+import { syncUserInApps } from "../../../utilities/appService"
+import { quotas, users } from "@budibase/pro"
+const { errors } = require("@budibase/backend-core")
+import { allUsers, getUser } from "../../utilities"
 
-exports.save = async ctx => {
+export const save = async (ctx: any) => {
   try {
-    const user = await saveUser(ctx.request.body, getTenantId())
+    const user: any = await users.save(ctx.request.body, getTenantId())
     // let server know to sync user
     await syncUserInApps(user._id)
     ctx.body = user
-  } catch (err) {
+  } catch (err: any) {
     ctx.throw(err.status || 400, err)
   }
 }
 
-const parseBooleanParam = param => {
+const parseBooleanParam = (param: any) => {
   return !(param && param === "false")
 }
 
-exports.adminUser = async ctx => {
+export const adminUser = async (ctx: any) => {
   const { email, password, tenantId } = ctx.request.body
 
   // account portal sends a pre-hashed password - honour param to prevent double hashing
@@ -52,7 +50,7 @@ exports.adminUser = async ctx => {
     ctx.throw(403, "Organisation already exists.")
   }
 
-  const response = await doWithGlobalDB(tenantId, async db => {
+  const response = await doWithGlobalDB(tenantId, async (db: any) => {
     const response = await db.allDocs(
       getGlobalUserParams(null, {
         include_docs: true,
@@ -69,12 +67,12 @@ exports.adminUser = async ctx => {
       } catch (err) {
         // don't worry about errors
       }
-      await db.put(generateNewUsageQuotaDoc())
+      await db.put(quotas.generateNewQuotaUsage())
     }
     return response
   })
 
-  if (response.rows.some(row => row.doc.admin)) {
+  if (response.rows.some((row: any) => row.doc.admin)) {
     ctx.throw(
       403,
       "You cannot initialise once an global user has been created."
@@ -95,13 +93,13 @@ exports.adminUser = async ctx => {
     tenantId,
   }
   try {
-    ctx.body = await saveUser(user, tenantId, hashPassword, requirePassword)
-  } catch (err) {
+    ctx.body = await users.save(user, tenantId, hashPassword, requirePassword)
+  } catch (err: any) {
     ctx.throw(err.status || 400, err)
   }
 }
 
-exports.destroy = async ctx => {
+export const destroy = async (ctx: any) => {
   const db = getGlobalDB()
   const dbUser = await db.get(ctx.params.id)
 
@@ -120,6 +118,7 @@ exports.destroy = async ctx => {
 
   await removeUserFromInfoDB(dbUser)
   await db.remove(dbUser._id, dbUser._rev)
+  await quotas.removeUser(dbUser)
   await userCache.invalidateUser(dbUser._id)
   await invalidateSessions(dbUser._id)
   // let server know to sync user
@@ -130,23 +129,23 @@ exports.destroy = async ctx => {
 }
 
 // called internally by app server user fetch
-exports.fetch = async ctx => {
-  const users = await allUsers(ctx)
+export const fetch = async (ctx: any) => {
+  const all = await allUsers()
   // user hashed password shouldn't ever be returned
-  for (let user of users) {
+  for (let user of all) {
     if (user) {
       delete user.password
     }
   }
-  ctx.body = users
+  ctx.body = all
 }
 
 // called internally by app server user find
-exports.find = async ctx => {
+export const find = async (ctx: any) => {
   ctx.body = await getUser(ctx.params.id)
 }
 
-exports.tenantUserLookup = async ctx => {
+export const tenantUserLookup = async (ctx: any) => {
   const id = ctx.params.id
   const user = await getTenantUser(id)
   if (user) {
@@ -156,7 +155,7 @@ exports.tenantUserLookup = async ctx => {
   }
 }
 
-exports.invite = async ctx => {
+export const invite = async (ctx: any) => {
   let { email, userInfo } = ctx.request.body
   const existing = await getGlobalUserByEmail(email)
   if (existing) {
@@ -166,21 +165,22 @@ exports.invite = async ctx => {
     userInfo = {}
   }
   userInfo.tenantId = getTenantId()
-  await sendEmail(email, EmailTemplatePurpose.INVITATION, {
+  const opts: any = {
     subject: "{{ company }} platform invitation",
     info: userInfo,
-  })
+  }
+  await sendEmail(email, EmailTemplatePurpose.INVITATION, opts)
   ctx.body = {
     message: "Invitation has been sent.",
   }
 }
 
-exports.inviteAccept = async ctx => {
+export const inviteAccept = async (ctx: any) => {
   const { inviteCode, password, firstName, lastName } = ctx.request.body
   try {
     // info is an extension of the user object that was stored by global
-    const { email, info } = await checkInviteCode(inviteCode)
-    ctx.body = await saveUser(
+    const { email, info }: any = await checkInviteCode(inviteCode)
+    ctx.body = await users.save(
       {
         firstName,
         lastName,
@@ -190,7 +190,11 @@ exports.inviteAccept = async ctx => {
       },
       info.tenantId
     )
-  } catch (err) {
+  } catch (err: any) {
+    if (err.code === errors.codes.USAGE_LIMIT_EXCEEDED) {
+      // explicitly re-throw limit exceeded errors
+      ctx.throw(400, err)
+    }
     ctx.throw(400, "Unable to create new user, invitation invalid.")
   }
 }

package/src/api/index.js

@@ -8,6 +8,8 @@ const {
   buildTenancyMiddleware,
   buildCsrfMiddleware,
 } = require("@budibase/backend-core/auth")
+const { middleware: pro } = require("@budibase/pro")
+const { errors } = require("@budibase/backend-core")
 
 const PUBLIC_ENDPOINTS = [
   // old deprecated endpoints kept for backwards compat
@@ -98,6 +100,7 @@ router
   .use(buildAuthMiddleware(PUBLIC_ENDPOINTS))
   .use(buildTenancyMiddleware(PUBLIC_ENDPOINTS, NO_TENANCY_ENDPOINTS))
   .use(buildCsrfMiddleware({ noCsrfPatterns: NO_CSRF_ENDPOINTS }))
+  .use(pro.licensing())
   // for now no public access is allowed to worker (bar health check)
   .use((ctx, next) => {
     if (ctx.publicEndpoint) {
@@ -110,16 +113,18 @@ router
   })
   .use(auditLog)
 
-// error handling middleware
+// error handling middleware - TODO: This could be moved to backend-core
 router.use(async (ctx, next) => {
   try {
     await next()
   } catch (err) {
     ctx.log.error(err)
     ctx.status = err.status || err.statusCode || 500
+    const error = errors.getPublicError(err)
     ctx.body = {
       message: err.message,
       status: ctx.status,
+      error,
     }
   }
 })

package/src/api/routes/global/license.ts

@@ -0,0 +1,12 @@
+import Router from "@koa/router"
+import * as controller from "../../controllers/global/license"
+
+const router = new Router()
+
+router
+  .post("/api/global/license/activate", controller.activate)
+  .post("/api/global/license/refresh", controller.refresh)
+  .get("/api/global/license/info", controller.getInfo)
+  .get("/api/global/license/usage", controller.getQuotaUsage)
+
+export = router

package/src/api/routes/index.js

@@ -10,6 +10,7 @@ const environmentRoutes = require("./system/environment")
 const tenantsRoutes = require("./system/tenants")
 const statusRoutes = require("./system/status")
 const selfRoutes = require("./global/self")
+const licenseRoutes = require("./global/license")
 
 exports.routes = [
   configRoutes,
@@ -24,4 +25,5 @@ exports.routes = [
   environmentRoutes,
   statusRoutes,
   selfRoutes,
+  licenseRoutes,
 ]

package/src/api/routes/tests/auth.spec.js

@@ -75,7 +75,8 @@ describe("/api/global/auth", () => {
     afterEach(() => {
       expect(strategyFactory).toBeCalledWith(
         chosenConfig, 
-        `http://localhost:10000/api/global/auth/${TENANT_ID}/oidc/callback`
+        `http://localhost:10000/api/global/auth/${TENANT_ID}/oidc/callback`,
+        expect.any(Function)
       )
     })
 

package/src/environment.js

@@ -19,7 +19,7 @@ if (!LOADED && isDev() && !isTest()) {
 module.exports = {
   NODE_ENV: process.env.NODE_ENV,
   SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
-  PORT: process.env.PORT,
+  PORT: process.env.PORT || process.env.WORKER_PORT,
   CLUSTER_PORT: process.env.CLUSTER_PORT,
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,

package/src/index.ts

@@ -2,6 +2,7 @@
 import { Scope } from "@sentry/node"
 import { Event } from "@sentry/types/dist/event"
 import Application from "koa"
+import { bootstrap } from "global-agent"
 
 const env = require("./environment")
 import db from "./db"
@@ -17,6 +18,9 @@ const api = require("./api")
 const redis = require("./utilities/redis")
 const Sentry = require("@sentry/node")
 
+// this will setup http and https proxies form env variables
+bootstrap()
+
 const app: Application = new Koa()
 
 app.keys = ["secret", "key"]