@budibase/server 2.1.32-alpha.8 → 2.1.32

package/src/utilities/fileSystem/clientLibrary.js

@@ -1,7 +1,7 @@
 const { join } = require("path")
 const { ObjectStoreBuckets } = require("../../constants")
 const fs = require("fs")
-const { objectStore } = require("@budibase/backend-core")
+const { upload, retrieveToTmp, streamUpload } = require("./utilities")
 const { resolve } = require("../centralPath")
 const env = require("../../environment")
 const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
@@ -38,13 +38,13 @@ exports.backupClientLibrary = async appId => {
   let tmpManifestPath
   try {
     // Try to load the manifest from the new file location
-    tmpManifestPath = await objectStore.retrieveToTmp(
+    tmpManifestPath = await retrieveToTmp(
       ObjectStoreBuckets.APPS,
       join(appId, "manifest.json")
     )
   } catch (error) {
     // Fallback to loading it from the old location for old apps
-    tmpManifestPath = await objectStore.retrieveToTmp(
+    tmpManifestPath = await retrieveToTmp(
       ObjectStoreBuckets.APPS,
       join(
         appId,
@@ -58,19 +58,19 @@ exports.backupClientLibrary = async appId => {
   }
 
   // Copy existing client lib to tmp
-  const tmpClientPath = await objectStore.retrieveToTmp(
+  const tmpClientPath = await retrieveToTmp(
     ObjectStoreBuckets.APPS,
     join(appId, "budibase-client.js")
   )
 
   // Upload manifest and client library as backups
-  const manifestUpload = objectStore.upload({
+  const manifestUpload = upload({
     bucket: ObjectStoreBuckets.APPS,
     filename: join(appId, "manifest.json.bak"),
     path: tmpManifestPath,
     type: "application/json",
   })
-  const clientUpload = objectStore.upload({
+  const clientUpload = upload({
     bucket: ObjectStoreBuckets.APPS,
     filename: join(appId, "budibase-client.js.bak"),
     path: tmpClientPath,
@@ -99,7 +99,7 @@ exports.updateClientLibrary = async appId => {
   }
 
   // Upload latest manifest and client library
-  const manifestUpload = objectStore.streamUpload(
+  const manifestUpload = streamUpload(
     ObjectStoreBuckets.APPS,
     join(appId, "manifest.json"),
     fs.createReadStream(manifest),
@@ -107,7 +107,7 @@ exports.updateClientLibrary = async appId => {
       ContentType: "application/json",
     }
   )
-  const clientUpload = objectStore.streamUpload(
+  const clientUpload = streamUpload(
     ObjectStoreBuckets.APPS,
     join(appId, "budibase-client.js"),
     fs.createReadStream(client),
@@ -126,25 +126,25 @@ exports.updateClientLibrary = async appId => {
  */
 exports.revertClientLibrary = async appId => {
   // Copy backups manifest to tmp directory
-  const tmpManifestPath = await objectStore.retrieveToTmp(
+  const tmpManifestPath = await retrieveToTmp(
     ObjectStoreBuckets.APPS,
     join(appId, "manifest.json.bak")
   )
 
   // Copy backup client lib to tmp
-  const tmpClientPath = await objectStore.retrieveToTmp(
+  const tmpClientPath = await retrieveToTmp(
     ObjectStoreBuckets.APPS,
     join(appId, "budibase-client.js.bak")
   )
 
   // Upload backups as new versions
-  const manifestUpload = objectStore.upload({
+  const manifestUpload = upload({
     bucket: ObjectStoreBuckets.APPS,
     filename: join(appId, "manifest.json"),
     path: tmpManifestPath,
     type: "application/json",
   })
-  const clientUpload = objectStore.upload({
+  const clientUpload = upload({
     bucket: ObjectStoreBuckets.APPS,
     filename: join(appId, "budibase-client.js"),
     path: tmpClientPath,

package/src/utilities/fileSystem/index.js

@@ -2,13 +2,7 @@ const { budibaseTempDir } = require("../budibaseDir")
 const fs = require("fs")
 const { join } = require("path")
 const uuid = require("uuid/v4")
-const { context, objectStore } = require("@budibase/backend-core")
 const { ObjectStoreBuckets } = require("../../constants")
-const { updateClientLibrary } = require("./clientLibrary")
-const { checkSlashesInUrl } = require("../")
-const env = require("../../environment")
-const tar = require("tar")
-const fetch = require("node-fetch")
 const {
   upload,
   retrieve,
@@ -17,7 +11,13 @@ const {
   downloadTarball,
   downloadTarballDirect,
   deleteFiles,
-} = objectStore
+} = require("./utilities")
+const { updateClientLibrary } = require("./clientLibrary")
+const { checkSlashesInUrl } = require("../")
+const env = require("../../environment")
+const { getAppId } = require("@budibase/backend-core/context")
+const tar = require("tar")
+const fetch = require("node-fetch")
 
 const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
 const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
@@ -165,7 +165,7 @@ exports.downloadTemplate = async (type, name) => {
  * Retrieves component libraries from object store (or tmp symlink if in local)
  */
 exports.getComponentLibraryManifest = async library => {
-  const appId = context.getAppId()
+  const appId = getAppId()
   const filename = "manifest.json"
   /* istanbul ignore next */
   // when testing in cypress and so on we need to get the package

package/src/utilities/fileSystem/utilities.js

@@ -0,0 +1,36 @@
+const {
+  ObjectStore,
+  makeSureBucketExists,
+  upload,
+  deleteFiles,
+  streamUpload,
+  retrieve,
+  retrieveToTmp,
+  retrieveDirectory,
+  deleteFolder,
+  uploadDirectory,
+  downloadTarball,
+  downloadTarballDirect,
+} = require("@budibase/backend-core/objectStore")
+
+/***********************************
+ *              NOTE               *
+ * This file purely exists so that *
+ *  the object store functionality *
+ *   can easily be mocked out of   *
+ *  the server without mocking the *
+ *      entire core library.       *
+ ***********************************/
+
+exports.ObjectStore = ObjectStore
+exports.makeSureBucketExists = makeSureBucketExists
+exports.upload = upload
+exports.streamUpload = streamUpload
+exports.retrieve = retrieve
+exports.retrieveToTmp = retrieveToTmp
+exports.retrieveDirectory = retrieveDirectory
+exports.deleteFolder = deleteFolder
+exports.uploadDirectory = uploadDirectory
+exports.downloadTarball = downloadTarball
+exports.downloadTarballDirect = downloadTarballDirect
+exports.deleteFiles = deleteFiles

package/src/utilities/global.js

@@ -0,0 +1,127 @@
+const {
+  getMultiIDParams,
+  getGlobalIDFromUserMetadataID,
+} = require("../db/utils")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
+const { getProdAppID } = require("@budibase/backend-core/db")
+const { getGlobalUserParams } = require("@budibase/backend-core/db")
+const { user: userCache } = require("@budibase/backend-core/cache")
+const {
+  getGlobalDB,
+  isUserInAppTenant,
+} = require("@budibase/backend-core/tenancy")
+const env = require("../environment")
+const { getAppId } = require("@budibase/backend-core/context")
+const { groups } = require("@budibase/pro")
+
+exports.updateAppRole = (user, { appId } = {}) => {
+  appId = appId || getAppId()
+
+  if (!user || !user.roles) {
+    return user
+  }
+  // if in an multi-tenancy environment make sure roles are never updated
+  if (env.MULTI_TENANCY && !isUserInAppTenant(appId, user)) {
+    delete user.builder
+    delete user.admin
+    user.roleId = BUILTIN_ROLE_IDS.PUBLIC
+    return user
+  }
+  // always use the deployed app
+  user.roleId = user.roles[getProdAppID(appId)]
+  // if a role wasn't found then either set as admin (builder) or public (everyone else)
+  if (!user.roleId && user.builder && user.builder.global) {
+    user.roleId = BUILTIN_ROLE_IDS.ADMIN
+  } else if (!user.roleId && !user?.userGroups?.length) {
+    user.roleId = BUILTIN_ROLE_IDS.PUBLIC
+  } else if (user?.userGroups?.length) {
+    user.roleId = null
+  }
+
+  delete user.roles
+  return user
+}
+
+async function checkGroupRoles(user, { appId } = {}) {
+  if (user.roleId && user.roleId !== BUILTIN_ROLE_IDS.PUBLIC) {
+    return user
+  }
+  user.roleId = await groups.getGroupRoleId(user, appId)
+  return user
+}
+
+async function processUser(user, { appId } = {}) {
+  if (user) {
+    delete user.password
+  }
+  user = await exports.updateAppRole(user, { appId })
+  if (!user.roleId && user?.userGroups?.length) {
+    user = await checkGroupRoles(user, { appId })
+  }
+
+  return user
+}
+
+exports.getCachedSelf = async (ctx, appId) => {
+  // this has to be tenant aware, can't depend on the context to find it out
+  // running some middlewares before the tenancy causes context to break
+  const user = await userCache.getUser(ctx.user._id)
+  return processUser(user, { appId })
+}
+
+exports.getRawGlobalUser = async userId => {
+  const db = getGlobalDB()
+  return db.get(getGlobalIDFromUserMetadataID(userId))
+}
+
+exports.getGlobalUser = async userId => {
+  const appId = getAppId()
+  let user = await exports.getRawGlobalUser(userId)
+  return processUser(user, { appId })
+}
+
+exports.getGlobalUsers = async (users = null) => {
+  const appId = getAppId()
+  const db = getGlobalDB()
+  let globalUsers
+  if (users) {
+    const globalIds = users.map(user => getGlobalIDFromUserMetadataID(user._id))
+    globalUsers = (await db.allDocs(getMultiIDParams(globalIds))).rows.map(
+      row => row.doc
+    )
+  } else {
+    globalUsers = (
+      await db.allDocs(
+        getGlobalUserParams(null, {
+          include_docs: true,
+        })
+      )
+    ).rows.map(row => row.doc)
+  }
+  globalUsers = globalUsers
+    .filter(user => user != null)
+    .map(user => {
+      delete user.password
+      delete user.forceResetPassword
+      return user
+    })
+  if (!appId) {
+    return globalUsers
+  }
+
+  return globalUsers.map(user => exports.updateAppRole(user))
+}
+
+exports.getGlobalUsersFromMetadata = async users => {
+  const globalUsers = await exports.getGlobalUsers(users)
+  return users.map(user => {
+    const globalUser = globalUsers.find(
+      globalUser => globalUser && user._id.includes(globalUser._id)
+    )
+    return {
+      ...globalUser,
+      // doing user second overwrites the id and rev (always metadata)
+      ...user,
+    }
+  })
+}

package/src/utilities/{index.ts → index.js}

@@ -1,20 +1,17 @@
-import env from "../environment"
-import { OBJ_STORE_DIRECTORY } from "../constants"
-import { objectStore, context } from "@budibase/backend-core"
-import { generateMetadataID } from "../db/utils"
-import { Document } from "@budibase/types"
-import stream from "stream"
-const Readable = stream.Readable
-
-export function wait(ms: number) {
-  return new Promise(resolve => setTimeout(resolve, ms))
-}
+const env = require("../environment")
+const { OBJ_STORE_DIRECTORY } = require("../constants")
+const { sanitizeKey } = require("@budibase/backend-core/objectStore")
+const { generateMetadataID } = require("../db/utils")
+const Readable = require("stream").Readable
+const { getAppDB } = require("@budibase/backend-core/context")
+
+exports.wait = ms => new Promise(resolve => setTimeout(resolve, ms))
 
-export const isDev = env.isDev
+exports.isDev = env.isDev
 
-export const NUMBER_REGEX = /^[+-]?([0-9]*[.])?[0-9]+$/g
+exports.NUMBER_REGEX = /^[+-]?([0-9]*[.])?[0-9]+$/g
 
-export function removeFromArray(array: any[], element: any) {
+exports.removeFromArray = (array, element) => {
   const index = array.indexOf(element)
   if (index !== -1) {
     array.splice(index, 1)
@@ -28,7 +25,7 @@ export function removeFromArray(array: any[], element: any) {
  * @param {string} url The URL to test and remove any extra double slashes.
  * @return {string} The updated url.
  */
-export function checkSlashesInUrl(url: string) {
+exports.checkSlashesInUrl = url => {
   return url.replace(/(https?:\/\/)|(\/)+/g, "$1$2")
 }
 
@@ -36,7 +33,7 @@ export function checkSlashesInUrl(url: string) {
  * Gets the address of the object store, depending on whether self hosted or in cloud.
  * @return {string} The base URL of the object store (MinIO or S3).
  */
-export function objectStoreUrl() {
+exports.objectStoreUrl = () => {
   if (env.SELF_HOSTED || env.MINIO_URL) {
     // can use a relative url for this as all goes through the proxy (this is hosted in minio)
     return OBJ_STORE_DIRECTORY
@@ -55,9 +52,9 @@ export function objectStoreUrl() {
  * @return {string} The URL to be inserted into appPackage response or server rendered
  * app index file.
  */
-export function clientLibraryPath(appId: string, version: string) {
+exports.clientLibraryPath = (appId, version) => {
   if (env.isProd()) {
-    let url = `${objectStoreUrl()}/${objectStore.sanitizeKey(
+    let url = `${exports.objectStoreUrl()}/${sanitizeKey(
       appId
     )}/budibase-client.js`
 
@@ -71,19 +68,18 @@ export function clientLibraryPath(appId: string, version: string) {
   }
 }
 
-export function attachmentsRelativeURL(attachmentKey: string) {
-  return checkSlashesInUrl(`${objectStoreUrl()}/${attachmentKey}`)
+exports.attachmentsRelativeURL = attachmentKey => {
+  return exports.checkSlashesInUrl(
+    `${exports.objectStoreUrl()}/${attachmentKey}`
+  )
 }
 
-export async function updateEntityMetadata(
-  type: string,
-  entityId: string,
-  updateFn: any
-) {
-  const db = context.getAppDB()
+exports.updateEntityMetadata = async (type, entityId, updateFn) => {
+  const db = getAppDB()
   const id = generateMetadataID(type, entityId)
   // read it to see if it exists, we'll overwrite it no matter what
-  let rev, metadata: Document
+  let rev,
+    metadata = {}
   try {
     const oldMetadata = await db.get(id)
     rev = oldMetadata._rev
@@ -104,18 +100,14 @@ export async function updateEntityMetadata(
   }
 }
 
-export async function saveEntityMetadata(
-  type: string,
-  entityId: string,
-  metadata: Document
-) {
-  return updateEntityMetadata(type, entityId, () => {
+exports.saveEntityMetadata = async (type, entityId, metadata) => {
+  return exports.updateEntityMetadata(type, entityId, () => {
     return metadata
   })
 }
 
-export async function deleteEntityMetadata(type: string, entityId: string) {
-  const db = context.getAppDB()
+exports.deleteEntityMetadata = async (type, entityId) => {
+  const db = getAppDB()
   const id = generateMetadataID(type, entityId)
   let rev
   try {
@@ -131,7 +123,7 @@ export async function deleteEntityMetadata(type: string, entityId: string) {
   }
 }
 
-export function escapeDangerousCharacters(string: string) {
+exports.escapeDangerousCharacters = string => {
   return string
     .replace(/[\\]/g, "\\\\")
     .replace(/[\b]/g, "\\b")
@@ -141,7 +133,7 @@ export function escapeDangerousCharacters(string: string) {
     .replace(/[\t]/g, "\\t")
 }
 
-export function stringToReadStream(string: string) {
+exports.stringToReadStream = string => {
   return new Readable({
     read() {
       this.push(string)
@@ -150,7 +142,7 @@ export function stringToReadStream(string: string) {
   })
 }
 
-export function formatBytes(bytes: string) {
+exports.formatBytes = bytes => {
   const units = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]
   const byteIncrements = 1024
   let unit = 0
@@ -161,7 +153,7 @@ export function formatBytes(bytes: string) {
   return `${size.toFixed(size < 10 && unit > 0 ? 1 : 0)}${units[unit]}`
 }
 
-export function convertBookmark(bookmark: string) {
+exports.convertBookmark = bookmark => {
   const IS_NUMBER = /^\d+\.?\d*$/
   if (typeof bookmark === "string" && bookmark.match(IS_NUMBER)) {
     return parseFloat(bookmark)
@@ -169,7 +161,7 @@ export function convertBookmark(bookmark: string) {
   return bookmark
 }
 
-export function isQsTrue(param: string) {
+exports.isQsTrue = param => {
   if (typeof param === "string") {
     return param.toLowerCase() === "true"
   } else {

package/src/utilities/{redis.ts → redis.js}

@@ -1,47 +1,46 @@
-import { redis } from "@budibase/backend-core"
-import { getGlobalIDFromUserMetadataID } from "../db/utils"
-import { ContextUser } from "@budibase/types"
+const { Client, utils } = require("@budibase/backend-core/redis")
+const { getGlobalIDFromUserMetadataID } = require("../db/utils")
 
 const APP_DEV_LOCK_SECONDS = 600
 const AUTOMATION_TEST_FLAG_SECONDS = 60
-let devAppClient: any, debounceClient: any, flagClient: any
+let devAppClient, debounceClient, flagClient
 
 // we init this as we want to keep the connection open all the time
 // reduces the performance hit
-export async function init() {
-  devAppClient = new redis.Client(redis.utils.Databases.DEV_LOCKS)
-  debounceClient = new redis.Client(redis.utils.Databases.DEBOUNCE)
-  flagClient = new redis.Client(redis.utils.Databases.FLAGS)
+exports.init = async () => {
+  devAppClient = new Client(utils.Databases.DEV_LOCKS)
+  debounceClient = new Client(utils.Databases.DEBOUNCE)
+  flagClient = new Client(utils.Databases.FLAGS)
   await devAppClient.init()
   await debounceClient.init()
   await flagClient.init()
 }
 
-export async function shutdown() {
+exports.shutdown = async () => {
   if (devAppClient) await devAppClient.finish()
   if (debounceClient) await debounceClient.finish()
   if (flagClient) await flagClient.finish()
   console.log("Redis shutdown")
 }
 
-export async function doesUserHaveLock(devAppId: string, user: ContextUser) {
+exports.doesUserHaveLock = async (devAppId, user) => {
   const value = await devAppClient.get(devAppId)
   if (!value) {
     return true
   }
   // make sure both IDs are global
   const expected = getGlobalIDFromUserMetadataID(value._id)
-  const userId = getGlobalIDFromUserMetadataID(user._id!)
+  const userId = getGlobalIDFromUserMetadataID(user._id)
   return expected === userId
 }
 
-export async function getLocksById(appIds: string[]) {
+exports.getLocksById = async appIds => {
   return await devAppClient.bulkGet(appIds)
 }
 
-export async function updateLock(devAppId: string, user: ContextUser) {
+exports.updateLock = async (devAppId, user) => {
   // make sure always global user ID
-  const globalId = getGlobalIDFromUserMetadataID(user._id!)
+  const globalId = getGlobalIDFromUserMetadataID(user._id)
   const inputUser = {
     ...user,
     userId: globalId,
@@ -52,35 +51,35 @@ export async function updateLock(devAppId: string, user: ContextUser) {
   await devAppClient.store(devAppId, inputUser, APP_DEV_LOCK_SECONDS)
 }
 
-export async function clearLock(devAppId: string, user: ContextUser) {
+exports.clearLock = async (devAppId, user) => {
   const value = await devAppClient.get(devAppId)
   if (!value) {
     return
   }
-  const userId = getGlobalIDFromUserMetadataID(user._id!)
+  const userId = getGlobalIDFromUserMetadataID(user._id)
   if (value._id !== userId) {
     throw "User does not hold lock, cannot clear it."
   }
   await devAppClient.delete(devAppId)
 }
 
-export async function checkDebounce(id: string) {
+exports.checkDebounce = async id => {
   return debounceClient.get(id)
 }
 
-export async function setDebounce(id: string, seconds: number) {
+exports.setDebounce = async (id, seconds) => {
   await debounceClient.store(id, "debouncing", seconds)
 }
 
-export async function setTestFlag(id: string) {
+exports.setTestFlag = async id => {
   await flagClient.store(id, { testing: true }, AUTOMATION_TEST_FLAG_SECONDS)
 }
 
-export async function checkTestFlag(id: string) {
+exports.checkTestFlag = async id => {
   const flag = await flagClient.get(id)
   return !!(flag && flag.testing)
 }
 
-export async function clearTestFlag(id: string) {
+exports.clearTestFlag = async id => {
   await devAppClient.delete(id)
 }

package/src/utilities/routing/index.js

@@ -0,0 +1,25 @@
+const { createRoutingView } = require("../../db/views/staticViews")
+const { ViewName, getQueryIndex, UNICODE_MAX } = require("../../db/utils")
+const { getAppDB } = require("@budibase/backend-core/context")
+
+exports.getRoutingInfo = async () => {
+  const db = getAppDB()
+  try {
+    const allRouting = await db.query(getQueryIndex(ViewName.ROUTING), {
+      startKey: "",
+      endKey: UNICODE_MAX,
+    })
+    return allRouting.rows.map(row => row.value)
+  } catch (err) {
+    // check if the view doesn't exist, it should for all new instances
+    /* istanbul ignore next */
+    if (err != null && err.name === "not_found") {
+      await createRoutingView()
+      return exports.getRoutingInfo()
+    } else {
+      throw err
+    }
+  }
+}
+
+exports.createRoutingView = createRoutingView