@budibase/server 2.1.40-alpha.5 → 2.1.41

package/src/utilities/fileSystem/{index.ts → index.js}

@@ -1,17 +1,26 @@
-import { budibaseTempDir } from "../budibaseDir"
-import fs from "fs"
-import { join } from "path"
-import { context, objectStore } from "@budibase/backend-core"
-import { ObjectStoreBuckets } from "../../constants"
-import { updateClientLibrary } from "./clientLibrary"
-import { checkSlashesInUrl } from "../"
-import env from "../../environment"
-import fetch from "node-fetch"
+const { budibaseTempDir } = require("../budibaseDir")
+const fs = require("fs")
+const { join } = require("path")
 const uuid = require("uuid/v4")
+const { ObjectStoreBuckets } = require("../../constants")
+const {
+  upload,
+  retrieve,
+  retrieveToTmp,
+  deleteFolder,
+  downloadTarball,
+  downloadTarballDirect,
+  deleteFiles,
+} = require("./utilities")
+const { updateClientLibrary } = require("./clientLibrary")
+const { checkSlashesInUrl } = require("../")
+const env = require("../../environment")
+const { getAppId } = require("@budibase/backend-core/context")
 const tar = require("tar")
+const fetch = require("node-fetch")
 
-export const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
-export const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
+const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
+const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
 const DATASOURCE_PATH = join(budibaseTempDir(), "datasource")
 
 /**
@@ -25,14 +34,14 @@ const DATASOURCE_PATH = join(budibaseTempDir(), "datasource")
 /**
  * Upon first startup of instance there may not be everything we need in tmp directory, set it up.
  */
-export function init() {
+exports.init = () => {
   const tempDir = budibaseTempDir()
   if (!fs.existsSync(tempDir)) {
     // some test cases fire this quickly enough that
     // synchronous cases can end up here at the same time
     try {
       fs.mkdirSync(tempDir)
-    } catch (err: any) {
+    } catch (err) {
       if (!err || err.code !== "EEXIST") {
         throw err
       }
@@ -72,7 +81,7 @@ exports.checkDevelopmentEnvironment = () => {
  * @param {string} path The path to the handlebars file which is to be loaded.
  * @returns {string} The loaded handlebars file as a string - loaded as utf8.
  */
-export function loadHandlebarsFile(path: string) {
+exports.loadHandlebarsFile = path => {
   return fs.readFileSync(path, "utf8")
 }
 
@@ -80,19 +89,15 @@ export function loadHandlebarsFile(path: string) {
  * When return a file from the API need to write the file to the system temporarily so we
  * can create a read stream to send.
  * @param {string} contents the contents of the file which is to be returned from the API.
- * @param {string} encoding the encoding of the file to return (utf8 default)
  * @return {Object} the read stream which can be put into the koa context body.
  */
-export function apiFileReturn(
-  contents: string,
-  encoding: BufferEncoding = "utf8"
-) {
+exports.apiFileReturn = contents => {
   const path = join(budibaseTempDir(), uuid())
-  fs.writeFileSync(path, contents, { encoding })
-  return fs.createReadStream(path, { encoding })
+  fs.writeFileSync(path, contents)
+  return fs.createReadStream(path)
 }
 
-export function streamFile(path: string) {
+exports.streamFile = path => {
   return fs.createReadStream(path)
 }
 
@@ -101,7 +106,7 @@ export function streamFile(path: string) {
  * @param {string} fileContents contents which will be written to a temp file.
  * @return {string} the path to the temp file.
  */
-export function storeTempFile(fileContents: string) {
+exports.storeTempFile = fileContents => {
   const path = join(budibaseTempDir(), uuid())
   fs.writeFileSync(path, fileContents)
   return path
@@ -111,7 +116,7 @@ export function storeTempFile(fileContents: string) {
  * Utility function for getting a file read stream - a simple in memory buffered read
  * stream doesn't work for pouchdb.
  */
-export function stringToFileStream(contents: string) {
+exports.stringToFileStream = contents => {
   const path = exports.storeTempFile(contents)
   return fs.createReadStream(path)
 }
@@ -120,7 +125,7 @@ export function stringToFileStream(contents: string) {
  * Creates a temp file and returns it from the API.
  * @param {string} fileContents the contents to be returned in file.
  */
-export function sendTempFile(fileContents: string) {
+exports.sendTempFile = fileContents => {
   const path = exports.storeTempFile(fileContents)
   return fs.createReadStream(path)
 }
@@ -130,7 +135,7 @@ export function sendTempFile(fileContents: string) {
  * @param {string} appId The ID of the app which is being created.
  * @return {Promise<void>} once promise completes app resources should be ready in object store.
  */
-export async function createApp(appId: string) {
+exports.createApp = async appId => {
   await updateClientLibrary(appId)
 }
 
@@ -139,8 +144,8 @@ export async function createApp(appId: string) {
  * @param {string} appId The ID of the app which is being deleted.
  * @return {Promise<void>} once promise completes the app resources will be removed from object store.
  */
-export async function deleteApp(appId: string) {
-  await objectStore.deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`)
+exports.deleteApp = async appId => {
+  await deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`)
 }
 
 /**
@@ -149,22 +154,18 @@ export async function deleteApp(appId: string) {
  * @param name
  * @return {Promise<*>}
  */
-export async function downloadTemplate(type: string, name: string) {
+exports.downloadTemplate = async (type, name) => {
   const DEFAULT_TEMPLATES_BUCKET =
     "prod-budi-templates.s3-eu-west-1.amazonaws.com"
   const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz`
-  return objectStore.downloadTarball(
-    templateUrl,
-    ObjectStoreBuckets.TEMPLATES,
-    type
-  )
+  return downloadTarball(templateUrl, ObjectStoreBuckets.TEMPLATES, type)
 }
 
 /**
  * Retrieves component libraries from object store (or tmp symlink if in local)
  */
-export async function getComponentLibraryManifest(library: string) {
-  const appId = context.getAppId()
+exports.getComponentLibraryManifest = async library => {
+  const appId = getAppId()
   const filename = "manifest.json"
   /* istanbul ignore next */
   // when testing in cypress and so on we need to get the package
@@ -181,16 +182,12 @@ export async function getComponentLibraryManifest(library: string) {
     return require(path)
   }
 
-  if (!appId) {
-    throw new Error("No app ID found - cannot get component libraries")
-  }
-
   let resp
   let path
   try {
     // Try to load the manifest from the new file location
     path = join(appId, filename)
-    resp = await objectStore.retrieve(ObjectStoreBuckets.APPS, path)
+    resp = await retrieve(ObjectStoreBuckets.APPS, path)
   } catch (error) {
     console.error(
       `component-manifest-objectstore=failed appId=${appId} path=${path}`,
@@ -198,7 +195,7 @@ export async function getComponentLibraryManifest(library: string) {
     )
     // Fallback to loading it from the old location for old apps
     path = join(appId, "node_modules", library, "package", filename)
-    resp = await objectStore.retrieve(ObjectStoreBuckets.APPS, path)
+    resp = await retrieve(ObjectStoreBuckets.APPS, path)
   }
   if (typeof resp !== "string") {
     resp = resp.toString("utf8")
@@ -210,17 +207,14 @@ export async function getComponentLibraryManifest(library: string) {
  * All file reads come through here just to make sure all of them make sense
  * allows a centralised location to check logic is all good.
  */
-export function readFileSync(
-  filepath: string,
-  options: BufferEncoding = "utf8"
-) {
-  return fs.readFileSync(filepath, { encoding: options })
+exports.readFileSync = (filepath, options = "utf8") => {
+  return fs.readFileSync(filepath, options)
 }
 
 /**
  * Given a set of app IDs makes sure file system is cleared of any of their temp info.
  */
-export function cleanup(appIds: string[]) {
+exports.cleanup = appIds => {
   for (let appId of appIds) {
     const path = join(budibaseTempDir(), appId)
     if (fs.existsSync(path)) {
@@ -229,7 +223,7 @@ export function cleanup(appIds: string[]) {
   }
 }
 
-export function createTempFolder(item: string) {
+const createTempFolder = item => {
   const path = join(budibaseTempDir(), item)
   try {
     // remove old tmp directories automatically - don't combine
@@ -237,22 +231,24 @@ export function createTempFolder(item: string) {
       fs.rmSync(path, { recursive: true, force: true })
     }
     fs.mkdirSync(path)
-  } catch (err: any) {
+  } catch (err) {
     throw new Error(`Path cannot be created: ${err.message}`)
   }
 
   return path
 }
+exports.createTempFolder = createTempFolder
 
-export async function extractTarball(fromFilePath: string, toPath: string) {
+const extractTarball = async (fromFilePath, toPath) => {
   await tar.extract({
     file: fromFilePath,
     C: toPath,
   })
 }
+exports.extractTarball = extractTarball
 
-export async function getPluginMetadata(path: string) {
-  let metadata: { schema?: any; package?: any } = {}
+const getPluginMetadata = async path => {
+  let metadata = {}
   try {
     const pkg = fs.readFileSync(join(path, "package.json"), "utf8")
     const schema = fs.readFileSync(join(path, "schema.json"), "utf8")
@@ -269,7 +265,7 @@ export async function getPluginMetadata(path: string) {
         "package.json is missing one of 'name', 'version' or 'description'."
       )
     }
-  } catch (err: any) {
+  } catch (err) {
     throw new Error(
       `Unable to process schema.json/package.json in plugin. ${err.message}`
     )
@@ -277,12 +273,9 @@ export async function getPluginMetadata(path: string) {
 
   return { metadata, directory: path }
 }
+exports.getPluginMetadata = getPluginMetadata
 
-export async function getDatasourcePlugin(
-  name: string,
-  url: string,
-  hash: string
-) {
+exports.getDatasourcePlugin = async (name, url, hash) => {
   if (!fs.existsSync(DATASOURCE_PATH)) {
     fs.mkdirSync(DATASOURCE_PATH)
   }
@@ -318,7 +311,7 @@ export async function getDatasourcePlugin(
 /**
  * Find for a file recursively from start path applying filter, return first match
  */
-export function findFileRec(startPath: string, filter: any) {
+exports.findFileRec = (startPath, filter) => {
   if (!fs.existsSync(startPath)) {
     return
   }
@@ -339,10 +332,21 @@ export function findFileRec(startPath: string, filter: any) {
 /**
  * Remove a folder which is not empty from the file system
  */
-export function deleteFolderFileSystem(path: string) {
+exports.deleteFolderFileSystem = path => {
   if (!fs.existsSync(path)) {
     return
   }
 
   fs.rmSync(path, { recursive: true, force: true })
 }
+
+/**
+ * Full function definition for below can be found in the utilities.
+ */
+exports.upload = upload
+exports.retrieve = retrieve
+exports.retrieveToTmp = retrieveToTmp
+exports.deleteFiles = deleteFiles
+exports.downloadTarballDirect = downloadTarballDirect
+exports.TOP_LEVEL_PATH = TOP_LEVEL_PATH
+exports.NODE_MODULES_PATH = NODE_MODULES_PATH

package/src/utilities/fileSystem/{processor.ts → processor.js}

@@ -1,19 +1,21 @@
-import jimp from "jimp"
+const jimp = require("jimp")
 
 const FORMATS = {
   IMAGES: ["png", "jpg", "jpeg", "gif", "bmp", "tiff"],
 }
 
-function processImage(file: { path: string }) {
+function processImage(file) {
   // this will overwrite the temp file
   return jimp.read(file.path).then(img => {
     return img.resize(300, jimp.AUTO).write(file.path)
   })
 }
 
-export async function process(file: { extension: string; path: string }) {
+async function process(file) {
   if (FORMATS.IMAGES.includes(file.extension.toLowerCase())) {
     await processImage(file)
   }
   return file
 }
+
+exports.process = process

package/src/utilities/fileSystem/utilities.js

@@ -0,0 +1,36 @@
+const {
+  ObjectStore,
+  makeSureBucketExists,
+  upload,
+  deleteFiles,
+  streamUpload,
+  retrieve,
+  retrieveToTmp,
+  retrieveDirectory,
+  deleteFolder,
+  uploadDirectory,
+  downloadTarball,
+  downloadTarballDirect,
+} = require("@budibase/backend-core/objectStore")
+
+/***********************************
+ *              NOTE               *
+ * This file purely exists so that *
+ *  the object store functionality *
+ *   can easily be mocked out of   *
+ *  the server without mocking the *
+ *      entire core library.       *
+ ***********************************/
+
+exports.ObjectStore = ObjectStore
+exports.makeSureBucketExists = makeSureBucketExists
+exports.upload = upload
+exports.streamUpload = streamUpload
+exports.retrieve = retrieve
+exports.retrieveToTmp = retrieveToTmp
+exports.retrieveDirectory = retrieveDirectory
+exports.deleteFolder = deleteFolder
+exports.uploadDirectory = uploadDirectory
+exports.downloadTarball = downloadTarball
+exports.downloadTarballDirect = downloadTarballDirect
+exports.deleteFiles = deleteFiles

package/src/utilities/global.js

@@ -0,0 +1,127 @@
+const {
+  getMultiIDParams,
+  getGlobalIDFromUserMetadataID,
+} = require("../db/utils")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
+const { getProdAppID } = require("@budibase/backend-core/db")
+const { getGlobalUserParams } = require("@budibase/backend-core/db")
+const { user: userCache } = require("@budibase/backend-core/cache")
+const {
+  getGlobalDB,
+  isUserInAppTenant,
+} = require("@budibase/backend-core/tenancy")
+const env = require("../environment")
+const { getAppId } = require("@budibase/backend-core/context")
+const { groups } = require("@budibase/pro")
+
+exports.updateAppRole = (user, { appId } = {}) => {
+  appId = appId || getAppId()
+
+  if (!user || !user.roles) {
+    return user
+  }
+  // if in an multi-tenancy environment make sure roles are never updated
+  if (env.MULTI_TENANCY && !isUserInAppTenant(appId, user)) {
+    delete user.builder
+    delete user.admin
+    user.roleId = BUILTIN_ROLE_IDS.PUBLIC
+    return user
+  }
+  // always use the deployed app
+  user.roleId = user.roles[getProdAppID(appId)]
+  // if a role wasn't found then either set as admin (builder) or public (everyone else)
+  if (!user.roleId && user.builder && user.builder.global) {
+    user.roleId = BUILTIN_ROLE_IDS.ADMIN
+  } else if (!user.roleId && !user?.userGroups?.length) {
+    user.roleId = BUILTIN_ROLE_IDS.PUBLIC
+  } else if (user?.userGroups?.length) {
+    user.roleId = null
+  }
+
+  delete user.roles
+  return user
+}
+
+async function checkGroupRoles(user, { appId } = {}) {
+  if (user.roleId && user.roleId !== BUILTIN_ROLE_IDS.PUBLIC) {
+    return user
+  }
+  user.roleId = await groups.getGroupRoleId(user, appId)
+  return user
+}
+
+async function processUser(user, { appId } = {}) {
+  if (user) {
+    delete user.password
+  }
+  user = await exports.updateAppRole(user, { appId })
+  if (!user.roleId && user?.userGroups?.length) {
+    user = await checkGroupRoles(user, { appId })
+  }
+
+  return user
+}
+
+exports.getCachedSelf = async (ctx, appId) => {
+  // this has to be tenant aware, can't depend on the context to find it out
+  // running some middlewares before the tenancy causes context to break
+  const user = await userCache.getUser(ctx.user._id)
+  return processUser(user, { appId })
+}
+
+exports.getRawGlobalUser = async userId => {
+  const db = getGlobalDB()
+  return db.get(getGlobalIDFromUserMetadataID(userId))
+}
+
+exports.getGlobalUser = async userId => {
+  const appId = getAppId()
+  let user = await exports.getRawGlobalUser(userId)
+  return processUser(user, { appId })
+}
+
+exports.getGlobalUsers = async (users = null) => {
+  const appId = getAppId()
+  const db = getGlobalDB()
+  let globalUsers
+  if (users) {
+    const globalIds = users.map(user => getGlobalIDFromUserMetadataID(user._id))
+    globalUsers = (await db.allDocs(getMultiIDParams(globalIds))).rows.map(
+      row => row.doc
+    )
+  } else {
+    globalUsers = (
+      await db.allDocs(
+        getGlobalUserParams(null, {
+          include_docs: true,
+        })
+      )
+    ).rows.map(row => row.doc)
+  }
+  globalUsers = globalUsers
+    .filter(user => user != null)
+    .map(user => {
+      delete user.password
+      delete user.forceResetPassword
+      return user
+    })
+  if (!appId) {
+    return globalUsers
+  }
+
+  return globalUsers.map(user => exports.updateAppRole(user))
+}
+
+exports.getGlobalUsersFromMetadata = async users => {
+  const globalUsers = await exports.getGlobalUsers(users)
+  return users.map(user => {
+    const globalUser = globalUsers.find(
+      globalUser => globalUser && user._id.includes(globalUser._id)
+    )
+    return {
+      ...globalUser,
+      // doing user second overwrites the id and rev (always metadata)
+      ...user,
+    }
+  })
+}

package/src/utilities/{index.ts → index.js}

@@ -1,20 +1,17 @@
-import env from "../environment"
-import { OBJ_STORE_DIRECTORY } from "../constants"
-import { objectStore, context } from "@budibase/backend-core"
-import { generateMetadataID } from "../db/utils"
-import { Document } from "@budibase/types"
-import stream from "stream"
-const Readable = stream.Readable
-
-export function wait(ms: number) {
-  return new Promise(resolve => setTimeout(resolve, ms))
-}
+const env = require("../environment")
+const { OBJ_STORE_DIRECTORY } = require("../constants")
+const { sanitizeKey } = require("@budibase/backend-core/objectStore")
+const { generateMetadataID } = require("../db/utils")
+const Readable = require("stream").Readable
+const { getAppDB } = require("@budibase/backend-core/context")
+
+exports.wait = ms => new Promise(resolve => setTimeout(resolve, ms))
 
-export const isDev = env.isDev
+exports.isDev = env.isDev
 
-export const NUMBER_REGEX = /^[+-]?([0-9]*[.])?[0-9]+$/g
+exports.NUMBER_REGEX = /^[+-]?([0-9]*[.])?[0-9]+$/g
 
-export function removeFromArray(array: any[], element: any) {
+exports.removeFromArray = (array, element) => {
   const index = array.indexOf(element)
   if (index !== -1) {
     array.splice(index, 1)
@@ -28,7 +25,7 @@ export function removeFromArray(array: any[], element: any) {
  * @param {string} url The URL to test and remove any extra double slashes.
  * @return {string} The updated url.
  */
-export function checkSlashesInUrl(url: string) {
+exports.checkSlashesInUrl = url => {
   return url.replace(/(https?:\/\/)|(\/)+/g, "$1$2")
 }
 
@@ -36,7 +33,7 @@ export function checkSlashesInUrl(url: string) {
  * Gets the address of the object store, depending on whether self hosted or in cloud.
  * @return {string} The base URL of the object store (MinIO or S3).
  */
-export function objectStoreUrl() {
+exports.objectStoreUrl = () => {
   if (env.SELF_HOSTED || env.MINIO_URL) {
     // can use a relative url for this as all goes through the proxy (this is hosted in minio)
     return OBJ_STORE_DIRECTORY
@@ -55,9 +52,9 @@ export function objectStoreUrl() {
  * @return {string} The URL to be inserted into appPackage response or server rendered
  * app index file.
  */
-export function clientLibraryPath(appId: string, version: string) {
+exports.clientLibraryPath = (appId, version) => {
   if (env.isProd()) {
-    let url = `${objectStoreUrl()}/${objectStore.sanitizeKey(
+    let url = `${exports.objectStoreUrl()}/${sanitizeKey(
       appId
     )}/budibase-client.js`
 
@@ -71,19 +68,18 @@ export function clientLibraryPath(appId: string, version: string) {
   }
 }
 
-export function attachmentsRelativeURL(attachmentKey: string) {
-  return checkSlashesInUrl(`${objectStoreUrl()}/${attachmentKey}`)
+exports.attachmentsRelativeURL = attachmentKey => {
+  return exports.checkSlashesInUrl(
+    `${exports.objectStoreUrl()}/${attachmentKey}`
+  )
 }
 
-export async function updateEntityMetadata(
-  type: string,
-  entityId: string,
-  updateFn: any
-) {
-  const db = context.getAppDB()
+exports.updateEntityMetadata = async (type, entityId, updateFn) => {
+  const db = getAppDB()
   const id = generateMetadataID(type, entityId)
   // read it to see if it exists, we'll overwrite it no matter what
-  let rev, metadata: Document
+  let rev,
+    metadata = {}
   try {
     const oldMetadata = await db.get(id)
     rev = oldMetadata._rev
@@ -104,18 +100,14 @@ export async function updateEntityMetadata(
   }
 }
 
-export async function saveEntityMetadata(
-  type: string,
-  entityId: string,
-  metadata: Document
-) {
-  return updateEntityMetadata(type, entityId, () => {
+exports.saveEntityMetadata = async (type, entityId, metadata) => {
+  return exports.updateEntityMetadata(type, entityId, () => {
     return metadata
   })
 }
 
-export async function deleteEntityMetadata(type: string, entityId: string) {
-  const db = context.getAppDB()
+exports.deleteEntityMetadata = async (type, entityId) => {
+  const db = getAppDB()
   const id = generateMetadataID(type, entityId)
   let rev
   try {
@@ -131,7 +123,7 @@ export async function deleteEntityMetadata(type: string, entityId: string) {
   }
 }
 
-export function escapeDangerousCharacters(string: string) {
+exports.escapeDangerousCharacters = string => {
   return string
     .replace(/[\\]/g, "\\\\")
     .replace(/[\b]/g, "\\b")
@@ -141,7 +133,7 @@ export function escapeDangerousCharacters(string: string) {
     .replace(/[\t]/g, "\\t")
 }
 
-export function stringToReadStream(string: string) {
+exports.stringToReadStream = string => {
   return new Readable({
     read() {
       this.push(string)
@@ -150,7 +142,7 @@ export function stringToReadStream(string: string) {
   })
 }
 
-export function formatBytes(bytes: string) {
+exports.formatBytes = bytes => {
   const units = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]
   const byteIncrements = 1024
   let unit = 0
@@ -161,7 +153,7 @@ export function formatBytes(bytes: string) {
   return `${size.toFixed(size < 10 && unit > 0 ? 1 : 0)}${units[unit]}`
 }
 
-export function convertBookmark(bookmark: string) {
+exports.convertBookmark = bookmark => {
   const IS_NUMBER = /^\d+\.?\d*$/
   if (typeof bookmark === "string" && bookmark.match(IS_NUMBER)) {
     return parseFloat(bookmark)
@@ -169,7 +161,7 @@ export function convertBookmark(bookmark: string) {
   return bookmark
 }
 
-export function isQsTrue(param: string) {
+exports.isQsTrue = param => {
   if (typeof param === "string") {
     return param.toLowerCase() === "true"
   } else {

package/src/utilities/{plugins.ts → plugins.js}

@@ -1,9 +1,8 @@
-import env from "../environment"
-import { plugins as ProPlugins } from "@budibase/pro"
-import { objectStore } from "@budibase/backend-core"
-import { Plugin } from "@budibase/types"
+const env = require("../environment")
+const { plugins: ProPlugins } = require("@budibase/pro")
+const { objectStore } = require("@budibase/backend-core")
 
-export function enrichPluginURLs(plugins: Plugin[]) {
+exports.enrichPluginURLs = plugins => {
   if (!plugins || !plugins.length) {
     return []
   }