@budibase/worker 1.0.123-alpha.0 → 1.0.123

package/nodemon.json

@@ -1,5 +1,5 @@
 {
-  "watch": ["src", "../backend-core", "../../../budibase-pro/packages/pro"],
+  "watch": ["src", "../backend-core"],
   "ext": "js,ts,json",
   "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js"],
   "exec": "ts-node src/index.ts"

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "1.0.123-alpha.0",
+  "version": "1.0.123",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -31,17 +31,14 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "^1.0.123-alpha.0",
-    "@budibase/pro": "1.0.105-alpha.43",
-    "@budibase/string-templates": "^1.0.123-alpha.0",
+    "@budibase/backend-core": "^1.0.123",
+    "@budibase/string-templates": "^1.0.123",
     "@koa/router": "^8.0.0",
-    "@sentry/node": "6.17.7",
+    "@sentry/node": "^6.0.0",
     "@techpass/passport-openidconnect": "^0.3.0",
-    "@types/global-agent": "^2.1.1",
     "aws-sdk": "^2.811.0",
     "bcryptjs": "^2.4.3",
     "dotenv": "^8.2.0",
-    "global-agent": "^3.0.0",
     "got": "^11.8.1",
     "joi": "^17.4.0",
     "koa": "^2.7.0",
@@ -66,9 +63,7 @@
     "@types/jest": "^26.0.23",
     "@types/koa": "^2.13.3",
     "@types/koa-router": "^7.4.2",
-    "@types/koa__router": "^8.0.11",
     "@types/node": "^15.12.4",
-    "@typescript-eslint/parser": "5.12.0",
     "copyfiles": "^2.4.1",
     "eslint": "^6.8.0",
     "jest": "^27.0.5",
@@ -79,7 +74,7 @@
     "supertest": "^6.1.3",
     "ts-jest": "^27.0.3",
     "ts-node": "^10.0.0",
-    "typescript": "4.5.5",
+    "typescript": "4.3.5",
     "update-dotenv": "^1.1.1"
   },
   "jest": {
@@ -89,5 +84,5 @@
       "./scripts/jestSetup.js"
     ]
   },
-  "gitHead": "94632aed5c844bb780b9a81238207b4965c40a54"
+  "gitHead": "db99c31b53d0dbbb788335126e45d1cc2f39fbf0"
 }

package/src/api/controllers/global/{auth.ts → auth.js}

@@ -21,9 +21,8 @@ const {
   isMultiTenant,
 } = require("@budibase/backend-core/tenancy")
 const env = require("../../../environment")
-import { users } from "@budibase/pro"
 
-const ssoCallbackUrl = async (config: any, type: any) => {
+const ssoCallbackUrl = async (config, type) => {
   // incase there is a callback URL from before
   if (config && config.callbackURL) {
     return config.callbackURL
@@ -43,15 +42,15 @@ const ssoCallbackUrl = async (config: any, type: any) => {
   return `${publicConfig.platformUrl}${callbackUrl}`
 }
 
-export const googleCallbackUrl = async (config: any) => {
+exports.googleCallbackUrl = async config => {
   return ssoCallbackUrl(config, "google")
 }
 
-export const oidcCallbackUrl = async (config: any) => {
+exports.oidcCallbackUrl = async config => {
   return ssoCallbackUrl(config, "oidc")
 }
 
-async function authInternal(ctx: any, user: any, err = null, info = null) {
+async function authInternal(ctx, user, err = null, info = null) {
   if (err) {
     console.error("Authentication error", err)
     return ctx.throw(403, info ? info : "Unauthorized")
@@ -72,23 +71,20 @@ async function authInternal(ctx: any, user: any, err = null, info = null) {
   }
 }
 
-export const authenticate = async (ctx: any, next: any) => {
-  return passport.authenticate(
-    "local",
-    async (err: any, user: any, info: any) => {
-      await authInternal(ctx, user, err, info)
-      ctx.status = 200
-    }
-  )(ctx, next)
+exports.authenticate = async (ctx, next) => {
+  return passport.authenticate("local", async (err, user, info) => {
+    await authInternal(ctx, user, err, info)
+    ctx.status = 200
+  })(ctx, next)
 }
 
-export const setInitInfo = (ctx: any) => {
+exports.setInitInfo = ctx => {
   const initInfo = ctx.request.body
   setCookie(ctx, initInfo, Cookies.Init)
   ctx.status = 200
 }
 
-export const getInitInfo = (ctx: any) => {
+exports.getInitInfo = ctx => {
   try {
     ctx.body = getCookie(ctx, Cookies.Init) || {}
   } catch (err) {
@@ -100,7 +96,7 @@ export const getInitInfo = (ctx: any) => {
 /**
  * Reset the user password, used as part of a forgotten password flow.
  */
-export const reset = async (ctx: any) => {
+exports.reset = async ctx => {
   const { email } = ctx.request.body
   const configured = await isEmailConfigured()
   if (!configured) {
@@ -130,7 +126,7 @@ export const reset = async (ctx: any) => {
 /**
  * Perform the user password update if the provided reset code is valid.
  */
-export const resetUpdate = async (ctx: any) => {
+exports.resetUpdate = async ctx => {
   const { resetCode, password } = ctx.request.body
   try {
     const { userId } = await checkResetPasswordCode(resetCode)
@@ -146,14 +142,14 @@ export const resetUpdate = async (ctx: any) => {
   }
 }
 
-export const logout = async (ctx: any) => {
+exports.logout = async ctx => {
   if (ctx.user && ctx.user._id) {
     await platformLogout({ ctx, userId: ctx.user._id })
   }
   ctx.body = { message: "User logged out." }
 }
 
-export const datasourcePreAuth = async (ctx: any, next: any) => {
+exports.datasourcePreAuth = async (ctx, next) => {
   const provider = ctx.params.provider
   const middleware = require(`@budibase/backend-core/middleware`)
   const handler = middleware.datasource[provider]
@@ -171,7 +167,7 @@ export const datasourcePreAuth = async (ctx: any, next: any) => {
   return handler.preAuth(passport, ctx, next)
 }
 
-export const datasourceAuth = async (ctx: any, next: any) => {
+exports.datasourceAuth = async (ctx, next) => {
   const authStateCookie = getCookie(ctx, Cookies.DatasourceAuth)
   const provider = authStateCookie.provider
   const middleware = require(`@budibase/backend-core/middleware`)
@@ -183,7 +179,7 @@ export const datasourceAuth = async (ctx: any, next: any) => {
  * The initial call that google authentication makes to take you to the google login screen.
  * On a successful login, you will be redirected to the googleAuth callback route.
  */
-export const googlePreAuth = async (ctx: any, next: any) => {
+exports.googlePreAuth = async (ctx, next) => {
   const db = getGlobalDB()
 
   const config = await core.db.getScopedConfig(db, {
@@ -191,14 +187,14 @@ export const googlePreAuth = async (ctx: any, next: any) => {
     workspace: ctx.query.workspace,
   })
   let callbackUrl = await exports.googleCallbackUrl(config)
-  const strategy = await google.strategyFactory(config, callbackUrl, users.save)
+  const strategy = await google.strategyFactory(config, callbackUrl)
 
   return passport.authenticate(strategy, {
     scope: ["profile", "email"],
   })(ctx, next)
 }
 
-export const googleAuth = async (ctx: any, next: any) => {
+exports.googleAuth = async (ctx, next) => {
   const db = getGlobalDB()
 
   const config = await core.db.getScopedConfig(db, {
@@ -206,12 +202,12 @@ export const googleAuth = async (ctx: any, next: any) => {
     workspace: ctx.query.workspace,
   })
   const callbackUrl = await exports.googleCallbackUrl(config)
-  const strategy = await google.strategyFactory(config, callbackUrl, users.save)
+  const strategy = await google.strategyFactory(config, callbackUrl)
 
   return passport.authenticate(
     strategy,
     { successRedirect: "/", failureRedirect: "/error" },
-    async (err: any, user: any, info: any) => {
+    async (err, user, info) => {
       await authInternal(ctx, user, err, info)
 
       ctx.redirect("/")
@@ -219,24 +215,24 @@ export const googleAuth = async (ctx: any, next: any) => {
   )(ctx, next)
 }
 
-async function oidcStrategyFactory(ctx: any, configId: any) {
+async function oidcStrategyFactory(ctx, configId) {
   const db = getGlobalDB()
   const config = await core.db.getScopedConfig(db, {
     type: Configs.OIDC,
     group: ctx.query.group,
   })
 
-  const chosenConfig = config.configs.filter((c: any) => c.uuid === configId)[0]
+  const chosenConfig = config.configs.filter(c => c.uuid === configId)[0]
   let callbackUrl = await exports.oidcCallbackUrl(chosenConfig)
 
-  return oidc.strategyFactory(chosenConfig, callbackUrl, users.save)
+  return oidc.strategyFactory(chosenConfig, callbackUrl)
 }
 
 /**
  * The initial call that OIDC authentication makes to take you to the configured OIDC login screen.
  * On a successful login, you will be redirected to the oidcAuth callback route.
  */
-export const oidcPreAuth = async (ctx: any, next: any) => {
+exports.oidcPreAuth = async (ctx, next) => {
   const { configId } = ctx.params
   const strategy = await oidcStrategyFactory(ctx, configId)
 
@@ -248,14 +244,14 @@ export const oidcPreAuth = async (ctx: any, next: any) => {
   })(ctx, next)
 }
 
-export const oidcAuth = async (ctx: any, next: any) => {
+exports.oidcAuth = async (ctx, next) => {
   const configId = getCookie(ctx, Cookies.OIDC_CONFIG)
   const strategy = await oidcStrategyFactory(ctx, configId)
 
   return passport.authenticate(
     strategy,
     { successRedirect: "/", failureRedirect: "/error" },
-    async (err: any, user: any, info: any) => {
+    async (err, user, info) => {
       await authInternal(ctx, user, err, info)
 
       ctx.redirect("/")

package/src/api/controllers/global/self.js

@@ -15,7 +15,6 @@ const { encrypt } = require("@budibase/backend-core/encryption")
 const { newid } = require("@budibase/backend-core/utils")
 const { getUser } = require("../../utilities")
 const { Cookies } = require("@budibase/backend-core/constants")
-const { featureFlags } = require("@budibase/backend-core")
 
 function newApiKey() {
   return encrypt(`${getTenantId()}${SEPARATOR}${newid()}`)
@@ -69,29 +68,6 @@ const checkCurrentApp = ctx => {
   }
 }
 
-/**
- * Add the attributes that are session based to the current user.
- */
-const addSessionAttributesToUser = ctx => {
-  ctx.body.account = ctx.user.account
-  ctx.body.license = ctx.user.license
-  ctx.body.budibaseAccess = ctx.user.budibaseAccess
-  ctx.body.accountPortalAccess = ctx.user.accountPortalAccess
-  ctx.body.csrfToken = ctx.user.csrfToken
-}
-
-/**
- * Remove the attributes that are session based from the current user,
- * so that stale values are not written to the db
- */
-const removeSessionAttributesFromUser = ctx => {
-  delete ctx.request.body.csrfToken
-  delete ctx.request.body.account
-  delete ctx.request.body.accountPortalAccess
-  delete ctx.request.body.budibaseAccess
-  delete ctx.request.body.license
-}
-
 exports.getSelf = async ctx => {
   if (!ctx.user) {
     ctx.throw(403, "User not logged in")
@@ -105,12 +81,11 @@ exports.getSelf = async ctx => {
 
   // get the main body of the user
   ctx.body = await getUser(userId)
-
-  // add the feature flags for this tenant
-  const tenantId = getTenantId()
-  ctx.body.featureFlags = featureFlags.getTenantFeatureFlags(tenantId)
-
-  addSessionAttributesToUser(ctx)
+  // forward session information not found in db
+  ctx.body.account = ctx.user.account
+  ctx.body.budibaseAccess = ctx.user.budibaseAccess
+  ctx.body.accountPortalAccess = ctx.user.accountPortalAccess
+  ctx.body.csrfToken = ctx.user.csrfToken
 }
 
 exports.updateSelf = async ctx => {
@@ -129,8 +104,8 @@ exports.updateSelf = async ctx => {
   // don't allow sending up an ID/Rev, always use the existing one
   delete ctx.request.body._id
   delete ctx.request.body._rev
-  removeSessionAttributesFromUser(ctx)
-
+  // don't allow setting the csrf token
+  delete ctx.request.body.csrfToken
   const response = await db.put({
     ...user,
     ...ctx.request.body,

package/src/api/controllers/global/{users.ts → users.js}

@@ -1,11 +1,15 @@
 const {
   getGlobalUserParams,
   StaticDatabases,
+  generateNewUsageQuotaDoc,
 } = require("@budibase/backend-core/db")
-const { getGlobalUserByEmail } = require("@budibase/backend-core/utils")
-import { EmailTemplatePurpose } from "../../../constants"
-import { checkInviteCode } from "../../../utilities/redis"
-import { sendEmail } from "../../../utilities/email"
+const {
+  getGlobalUserByEmail,
+  saveUser,
+} = require("@budibase/backend-core/utils")
+const { EmailTemplatePurpose } = require("../../../constants")
+const { checkInviteCode } = require("../../../utilities/redis")
+const { sendEmail } = require("../../../utilities/email")
 const { user: userCache } = require("@budibase/backend-core/cache")
 const { invalidateSessions } = require("@budibase/backend-core/sessions")
 const accounts = require("@budibase/backend-core/accounts")
@@ -17,28 +21,26 @@ const {
   doesTenantExist,
 } = require("@budibase/backend-core/tenancy")
 const { removeUserFromInfoDB } = require("@budibase/backend-core/deprovision")
-import env from "../../../environment"
-import { syncUserInApps } from "../../../utilities/appService"
-import { quotas, users } from "@budibase/pro"
-const { errors } = require("@budibase/backend-core")
-import { allUsers, getUser } from "../../utilities"
+const env = require("../../../environment")
+const { syncUserInApps } = require("../../../utilities/appService")
+const { allUsers, getUser } = require("../../utilities")
 
-export const save = async (ctx: any) => {
+exports.save = async ctx => {
   try {
-    const user: any = await users.save(ctx.request.body, getTenantId())
+    const user = await saveUser(ctx.request.body, getTenantId())
     // let server know to sync user
     await syncUserInApps(user._id)
     ctx.body = user
-  } catch (err: any) {
+  } catch (err) {
     ctx.throw(err.status || 400, err)
   }
 }
 
-const parseBooleanParam = (param: any) => {
+const parseBooleanParam = param => {
   return !(param && param === "false")
 }
 
-export const adminUser = async (ctx: any) => {
+exports.adminUser = async ctx => {
   const { email, password, tenantId } = ctx.request.body
 
   // account portal sends a pre-hashed password - honour param to prevent double hashing
@@ -50,7 +52,7 @@ export const adminUser = async (ctx: any) => {
     ctx.throw(403, "Organisation already exists.")
   }
 
-  const response = await doWithGlobalDB(tenantId, async (db: any) => {
+  const response = await doWithGlobalDB(tenantId, async db => {
     const response = await db.allDocs(
       getGlobalUserParams(null, {
         include_docs: true,
@@ -67,12 +69,12 @@ export const adminUser = async (ctx: any) => {
       } catch (err) {
         // don't worry about errors
       }
-      await db.put(quotas.generateNewQuotaUsage())
+      await db.put(generateNewUsageQuotaDoc())
     }
     return response
   })
 
-  if (response.rows.some((row: any) => row.doc.admin)) {
+  if (response.rows.some(row => row.doc.admin)) {
     ctx.throw(
       403,
       "You cannot initialise once an global user has been created."
@@ -93,13 +95,13 @@ export const adminUser = async (ctx: any) => {
     tenantId,
   }
   try {
-    ctx.body = await users.save(user, tenantId, hashPassword, requirePassword)
-  } catch (err: any) {
+    ctx.body = await saveUser(user, tenantId, hashPassword, requirePassword)
+  } catch (err) {
     ctx.throw(err.status || 400, err)
   }
 }
 
-export const destroy = async (ctx: any) => {
+exports.destroy = async ctx => {
   const db = getGlobalDB()
   const dbUser = await db.get(ctx.params.id)
 
@@ -118,7 +120,6 @@ export const destroy = async (ctx: any) => {
 
   await removeUserFromInfoDB(dbUser)
   await db.remove(dbUser._id, dbUser._rev)
-  await quotas.removeUser(dbUser)
   await userCache.invalidateUser(dbUser._id)
   await invalidateSessions(dbUser._id)
   // let server know to sync user
@@ -129,23 +130,23 @@ export const destroy = async (ctx: any) => {
 }
 
 // called internally by app server user fetch
-export const fetch = async (ctx: any) => {
-  const all = await allUsers()
+exports.fetch = async ctx => {
+  const users = await allUsers(ctx)
   // user hashed password shouldn't ever be returned
-  for (let user of all) {
+  for (let user of users) {
     if (user) {
       delete user.password
     }
   }
-  ctx.body = all
+  ctx.body = users
 }
 
 // called internally by app server user find
-export const find = async (ctx: any) => {
+exports.find = async ctx => {
   ctx.body = await getUser(ctx.params.id)
 }
 
-export const tenantUserLookup = async (ctx: any) => {
+exports.tenantUserLookup = async ctx => {
   const id = ctx.params.id
   const user = await getTenantUser(id)
   if (user) {
@@ -155,7 +156,7 @@ export const tenantUserLookup = async (ctx: any) => {
   }
 }
 
-export const invite = async (ctx: any) => {
+exports.invite = async ctx => {
   let { email, userInfo } = ctx.request.body
   const existing = await getGlobalUserByEmail(email)
   if (existing) {
@@ -165,22 +166,21 @@ export const invite = async (ctx: any) => {
     userInfo = {}
   }
   userInfo.tenantId = getTenantId()
-  const opts: any = {
+  await sendEmail(email, EmailTemplatePurpose.INVITATION, {
     subject: "{{ company }} platform invitation",
     info: userInfo,
-  }
-  await sendEmail(email, EmailTemplatePurpose.INVITATION, opts)
+  })
   ctx.body = {
     message: "Invitation has been sent.",
   }
 }
 
-export const inviteAccept = async (ctx: any) => {
+exports.inviteAccept = async ctx => {
   const { inviteCode, password, firstName, lastName } = ctx.request.body
   try {
     // info is an extension of the user object that was stored by global
-    const { email, info }: any = await checkInviteCode(inviteCode)
-    ctx.body = await users.save(
+    const { email, info } = await checkInviteCode(inviteCode)
+    ctx.body = await saveUser(
       {
         firstName,
         lastName,
@@ -190,11 +190,7 @@ export const inviteAccept = async (ctx: any) => {
       },
       info.tenantId
     )
-  } catch (err: any) {
-    if (err.code === errors.codes.USAGE_LIMIT_EXCEEDED) {
-      // explicitly re-throw limit exceeded errors
-      ctx.throw(400, err)
-    }
+  } catch (err) {
     ctx.throw(400, "Unable to create new user, invitation invalid.")
   }
 }

package/src/api/index.js

@@ -8,8 +8,6 @@ const {
   buildTenancyMiddleware,
   buildCsrfMiddleware,
 } = require("@budibase/backend-core/auth")
-const { middleware: pro } = require("@budibase/pro")
-const { errors } = require("@budibase/backend-core")
 
 const PUBLIC_ENDPOINTS = [
   // old deprecated endpoints kept for backwards compat
@@ -100,7 +98,6 @@ router
   .use(buildAuthMiddleware(PUBLIC_ENDPOINTS))
   .use(buildTenancyMiddleware(PUBLIC_ENDPOINTS, NO_TENANCY_ENDPOINTS))
   .use(buildCsrfMiddleware({ noCsrfPatterns: NO_CSRF_ENDPOINTS }))
-  .use(pro.licensing())
   // for now no public access is allowed to worker (bar health check)
   .use((ctx, next) => {
     if (ctx.publicEndpoint) {
@@ -113,18 +110,16 @@ router
   })
   .use(auditLog)
 
-// error handling middleware - TODO: This could be moved to backend-core
+// error handling middleware
 router.use(async (ctx, next) => {
   try {
     await next()
   } catch (err) {
     ctx.log.error(err)
     ctx.status = err.status || err.statusCode || 500
-    const error = errors.getPublicError(err)
     ctx.body = {
       message: err.message,
       status: ctx.status,
-      error,
     }
   }
 })

package/src/api/routes/index.js

@@ -10,7 +10,6 @@ const environmentRoutes = require("./system/environment")
 const tenantsRoutes = require("./system/tenants")
 const statusRoutes = require("./system/status")
 const selfRoutes = require("./global/self")
-const licenseRoutes = require("./global/license")
 
 exports.routes = [
   configRoutes,
@@ -25,5 +24,4 @@ exports.routes = [
   environmentRoutes,
   statusRoutes,
   selfRoutes,
-  licenseRoutes,
 ]

package/src/api/routes/tests/auth.spec.js

@@ -75,8 +75,7 @@ describe("/api/global/auth", () => {
     afterEach(() => {
       expect(strategyFactory).toBeCalledWith(
         chosenConfig, 
-        `http://localhost:10000/api/global/auth/${TENANT_ID}/oidc/callback`,
-        expect.any(Function)
+        `http://localhost:10000/api/global/auth/${TENANT_ID}/oidc/callback`
       )
     })
 

package/src/environment.js

@@ -19,7 +19,7 @@ if (!LOADED && isDev() && !isTest()) {
 module.exports = {
   NODE_ENV: process.env.NODE_ENV,
   SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
-  PORT: process.env.PORT || process.env.WORKER_PORT,
+  PORT: process.env.PORT,
   CLUSTER_PORT: process.env.CLUSTER_PORT,
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,

package/src/index.ts

@@ -2,7 +2,6 @@
 import { Scope } from "@sentry/node"
 import { Event } from "@sentry/types/dist/event"
 import Application from "koa"
-import { bootstrap } from "global-agent"
 
 const env = require("./environment")
 import db from "./db"
@@ -18,9 +17,6 @@ const api = require("./api")
 const redis = require("./utilities/redis")
 const Sentry = require("@sentry/node")
 
-// this will setup http and https proxies form env variables
-bootstrap()
-
 const app: Application = new Koa()
 
 app.keys = ["secret", "key"]

package/src/api/controllers/global/license.ts

@@ -1,30 +0,0 @@
-import { licensing, quotas } from "@budibase/pro"
-
-export const activate = async (ctx: any) => {
-  const { licenseKey } = ctx.request.body
-  if (!licenseKey) {
-    ctx.throw(400, "licenseKey is required")
-  }
-
-  await licensing.activateLicenseKey(licenseKey)
-  ctx.status = 200
-}
-
-export const refresh = async (ctx: any) => {
-  await licensing.cache.refresh()
-  ctx.status = 200
-}
-
-export const getInfo = async (ctx: any) => {
-  const licenseInfo = await licensing.getLicenseInfo()
-  if (licenseInfo) {
-    licenseInfo.licenseKey = "*"
-    ctx.body = licenseInfo
-  }
-  ctx.status = 200
-}
-
-export const getQuotaUsage = async (ctx: any) => {
-  const usage = await quotas.getQuotaUsage()
-  ctx.body = usage
-}

package/src/api/routes/global/license.ts

@@ -1,12 +0,0 @@
-import Router from "@koa/router"
-import * as controller from "../../controllers/global/license"
-
-const router = new Router()
-
-router
-  .post("/api/global/license/activate", controller.activate)
-  .post("/api/global/license/refresh", controller.refresh)
-  .get("/api/global/license/info", controller.getInfo)
-  .get("/api/global/license/usage", controller.getQuotaUsage)
-
-export = router