@budibase/worker 1.0.182 → 1.0.185-alpha.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "1.0.182",
+  "version": "1.0.185-alpha.0",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -32,9 +32,9 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "^1.0.182",
-    "@budibase/pro": "1.0.181",
-    "@budibase/string-templates": "^1.0.182",
+    "@budibase/backend-core": "^1.0.185-alpha.0",
+    "@budibase/pro": "1.0.184",
+    "@budibase/string-templates": "^1.0.185-alpha.0",
     "@koa/router": "^8.0.0",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "^0.3.0",
@@ -90,5 +90,5 @@
       "./scripts/jestSetup.js"
     ]
   },
-  "gitHead": "1fb604e6019a0dc345128344f2622de87a3fc64a"
+  "gitHead": "ef65c070fb9f06bdd953bb26c08f2715143ff853"
 }

package/scripts/load/heavyUse.js

@@ -0,0 +1,172 @@
+const fetch = require("node-fetch")
+
+const MAX_RUNTIME_SEC = 600
+const HOST = "http://localhost:10000"
+const TENANT_ID = "default"
+const RATE_MS = 500
+
+let API_KEY = process.argv[2]
+let STATS = {
+  iterations: 0,
+  error: 0,
+  success: 0,
+}
+
+if (!API_KEY) {
+  console.error("Must specify API key as first run command!")
+  process.exit(-1)
+}
+
+const USERS = [
+  {
+    email: "loadtest1@test.com",
+    password: "test",
+  },
+  {
+    email: "loadtest2@test.com",
+    password: "test",
+  },
+  {
+    email: "loadtest3@test.com",
+    password: "test",
+  },
+  {
+    email: "loadtest4@test.com",
+    password: "test",
+  },
+  {
+    email: "loadtest5@test.com",
+    password: "test",
+  },
+  {
+    email: "loadtest6@test.com",
+    password: "test",
+  },
+  {
+    email: "loadtest7@test.com",
+    password: "test",
+  },
+]
+
+const REQUESTS = [
+  {
+    endpoint: `/api/global/self`,
+    method: "GET",
+  },
+]
+
+function timeout() {
+  return new Promise(resolve => {
+    setTimeout(() => {
+      resolve()
+    }, MAX_RUNTIME_SEC * 1000)
+  })
+}
+
+async function preTest() {
+  // check if the user exists or not
+  const response = await fetch(`${HOST}/api/global/users`, {
+    method: "GET",
+    headers: {
+      "x-budibase-api-key": API_KEY,
+    },
+  })
+  if (response.status !== 200) {
+    throw new Error("Unable to retrieve users")
+  }
+  const users = await response.json()
+  for (let user of USERS) {
+    if (users.find(u => u.email === user.email)) {
+      continue
+    }
+    const response = await fetch(`${HOST}/api/global/users`, {
+      method: "POST",
+      headers: {
+        "x-budibase-api-key": API_KEY,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        ...user,
+        roles: {},
+        status: "active",
+      }),
+    })
+    if (response.status !== 200) {
+      throw new Error(
+        `Unable to create user ${user.email}, reason: ${await response.text()}`
+      )
+    }
+  }
+}
+
+async function requests(user) {
+  let response = await fetch(`${HOST}/api/global/auth/${TENANT_ID}/login`, {
+    method: "POST",
+    body: JSON.stringify({
+      username: user.email,
+      password: user.password,
+    }),
+    headers: {
+      "Content-Type": "application/json",
+    },
+  })
+  // unable to login
+  if (response.status !== 200) {
+    STATS.error++
+    return
+  } else {
+    STATS.success++
+  }
+  const cookie = response.headers.get("set-cookie")
+  let promises = []
+  for (let request of REQUESTS) {
+    const headers = {
+      cookie,
+    }
+    if (request.body) {
+      headers["Content-Type"] = "application/json"
+    }
+    promises.push(
+      fetch(`${HOST}${request.endpoint}`, {
+        method: request.method,
+        headers: {
+          cookie,
+        },
+      })
+    )
+  }
+  const responses = await Promise.all(promises)
+  for (let resp of responses) {
+    if (resp.status !== 200) {
+      console.error(await resp.text())
+      STATS.error++
+    } else {
+      STATS.success++
+    }
+  }
+}
+
+async function run() {
+  await preTest()
+  setInterval(async () => {
+    let promises = []
+    for (let user of USERS) {
+      promises.push(requests(user))
+    }
+    await Promise.all(promises)
+    console.log(
+      `Iteration ${STATS.iterations++} - errors: ${STATS.error}, success: ${
+        STATS.success
+      }`
+    )
+  }, RATE_MS)
+  await timeout()
+  console.log(
+    `Max runtime of ${MAX_RUNTIME_SEC} seconds has been reached - stopping.`
+  )
+  process.exit(0)
+}
+
+run().catch(err => {
+  console.error("Failed to run - ", err)
+})

package/scripts/load/users.js

@@ -0,0 +1,97 @@
+// get the JWT secret etc
+require("../../src/environment")
+require("@budibase/backend-core").init()
+const {
+  getProdAppID,
+  generateGlobalUserID,
+} = require("@budibase/backend-core/db")
+const { doInTenant, getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { internalSaveUser } = require("@budibase/backend-core/utils")
+const { publicApiUserFix } = require("../../src/utilities/users")
+const { hash } = require("@budibase/backend-core/utils")
+
+const USER_LOAD_NUMBER = 10000
+const BATCH_SIZE = 200
+const PASSWORD = "test"
+const TENANT_ID = "default"
+
+const APP_ID = process.argv[2]
+
+const words = [
+  "test",
+  "testing",
+  "budi",
+  "mail",
+  "age",
+  "risk",
+  "load",
+  "uno",
+  "arm",
+  "leg",
+  "pen",
+  "glass",
+  "box",
+  "chicken",
+  "bottle",
+]
+
+if (!APP_ID) {
+  console.error("Must supply app ID as first CLI option!")
+  process.exit(-1)
+}
+
+const WORD_1 = words[Math.floor(Math.random() * words.length)]
+const WORD_2 = words[Math.floor(Math.random() * words.length)]
+let HASHED_PASSWORD
+
+function generateUser(count) {
+  return {
+    _id: generateGlobalUserID(),
+    password: HASHED_PASSWORD,
+    email: `${WORD_1}${count}@${WORD_2}.com`,
+    roles: {
+      [getProdAppID(APP_ID)]: "BASIC",
+    },
+    status: "active",
+    forceResetPassword: false,
+    firstName: "John",
+    lastName: "Smith",
+  }
+}
+
+async function run() {
+  HASHED_PASSWORD = await hash(PASSWORD)
+  return doInTenant(TENANT_ID, async () => {
+    const db = getGlobalDB()
+    for (let i = 0; i < USER_LOAD_NUMBER; i += BATCH_SIZE) {
+      let userSavePromises = []
+      for (let j = 0; j < BATCH_SIZE; j++) {
+        // like the public API
+        const ctx = publicApiUserFix({
+          request: {
+            body: generateUser(i + j),
+          },
+        })
+        userSavePromises.push(
+          internalSaveUser(ctx.request.body, TENANT_ID, {
+            hashPassword: false,
+            requirePassword: true,
+            bulkCreate: true,
+          })
+        )
+      }
+      const users = await Promise.all(userSavePromises)
+      await db.bulkDocs(users)
+      console.log(`${i + BATCH_SIZE} users have been created.`)
+    }
+  })
+}
+
+run()
+  .then(() => {
+    console.log(`Generated ${USER_LOAD_NUMBER} users!`)
+  })
+  .catch(err => {
+    console.error("Failed for reason: ", err)
+    process.exit(-1)
+  })

package/src/api/controllers/global/license.ts

@@ -25,6 +25,5 @@ export const getInfo = async (ctx: any) => {
 }
 
 export const getQuotaUsage = async (ctx: any) => {
-  const usage = await quotas.getQuotaUsage()
-  ctx.body = usage
+  ctx.body = await quotas.getQuotaUsage()
 }

package/src/api/controllers/global/self.js

@@ -75,8 +75,8 @@ const checkCurrentApp = ctx => {
 const addSessionAttributesToUser = ctx => {
   ctx.body.account = ctx.user.account
   ctx.body.license = ctx.user.license
-  ctx.body.budibaseAccess = ctx.user.budibaseAccess
-  ctx.body.accountPortalAccess = ctx.user.accountPortalAccess
+  ctx.body.budibaseAccess = !!ctx.user.budibaseAccess
+  ctx.body.accountPortalAccess = !!ctx.user.accountPortalAccess
   ctx.body.csrfToken = ctx.user.csrfToken
 }
 

package/src/environment.js

@@ -27,33 +27,40 @@ function parseIntSafe(number) {
 }
 
 module.exports = {
-  NODE_ENV: process.env.NODE_ENV,
-  SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
-  PORT: process.env.PORT || process.env.WORKER_PORT,
-  CLUSTER_PORT: process.env.CLUSTER_PORT,
+  // auth
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
-  MINIO_URL: process.env.MINIO_URL,
-  COUCH_DB_URL: process.env.COUCH_DB_URL,
-  LOG_LEVEL: process.env.LOG_LEVEL,
   JWT_SECRET: process.env.JWT_SECRET,
   SALT_ROUNDS: process.env.SALT_ROUNDS,
-  REDIS_URL: process.env.REDIS_URL,
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
+  COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
+  // urls
+  MINIO_URL: process.env.MINIO_URL,
+  COUCH_DB_URL: process.env.COUCH_DB_URL,
+  REDIS_URL: process.env.REDIS_URL,
+  ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
+  PLATFORM_URL: process.env.PLATFORM_URL,
+  APPS_URL: process.env.APPS_URL,
+  // ports
+  PORT: process.env.PORT || process.env.WORKER_PORT,
+  CLUSTER_PORT: process.env.CLUSTER_PORT,
+  // flags
+  NODE_ENV: process.env.NODE_ENV,
+  SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
+  LOG_LEVEL: process.env.LOG_LEVEL,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
-  ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
   SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED,
+  DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
+  // smtp
   SMTP_USER: process.env.SMTP_USER,
   SMTP_PASSWORD: process.env.SMTP_PASSWORD,
   SMTP_HOST: process.env.SMTP_HOST,
   SMTP_PORT: process.env.SMTP_PORT,
   SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS,
-  PLATFORM_URL: process.env.PLATFORM_URL,
-  COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
+  // other
   CHECKLIST_CACHE_TTL: parseIntSafe(process.env.CHECKLIST_CACHE_TTL) || 3600,
-  APPS_URL: process.env.APPS_URL,
   _set(key, value) {
     process.env[key] = value
     module.exports[key] = value