@budibase/server 0.0.32 → 0.0.999-alpha.30

package/src/utilities/fileSystem/filesystem.ts

@@ -0,0 +1,168 @@
+import { PathLike } from "fs"
+import fs from "fs"
+import { budibaseTempDir } from "../budibaseDir"
+import { join } from "path"
+import env from "../../environment"
+import tar from "tar"
+const uuid = require("uuid/v4")
+
+export const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
+
+/**
+ * Upon first startup of instance there may not be everything we need in tmp directory, set it up.
+ */
+export const init = () => {
+  const tempDir = budibaseTempDir()
+  if (!fs.existsSync(tempDir)) {
+    // some test cases fire this quickly enough that
+    // synchronous cases can end up here at the same time
+    try {
+      fs.mkdirSync(tempDir)
+    } catch (err: any) {
+      if (!err || err.code !== "EEXIST") {
+        throw err
+      }
+    }
+  }
+}
+
+/**
+ * Checks if the system is currently in development mode and if it is makes sure
+ * everything required to function is ready.
+ */
+export const checkDevelopmentEnvironment = () => {
+  if (!env.isDev() || env.isTest()) {
+    return
+  }
+  if (!fs.existsSync(budibaseTempDir())) {
+    fs.mkdirSync(budibaseTempDir())
+  }
+  let error
+  if (!fs.existsSync(join(process.cwd(), ".env"))) {
+    error = "Must run via yarn once to generate environment."
+  }
+  if (error) {
+    console.error(error)
+    process.exit(-1)
+  }
+}
+
+/**
+ * Used to retrieve a handlebars file from the system which will be used as a template.
+ * This is allowable as the template handlebars files should be static and identical across
+ * the cluster.
+ * @param {string} path The path to the handlebars file which is to be loaded.
+ * @returns {string} The loaded handlebars file as a string - loaded as utf8.
+ */
+export const loadHandlebarsFile = (path: PathLike) => {
+  return fs.readFileSync(path, "utf8")
+}
+
+/**
+ * When return a file from the API need to write the file to the system temporarily so we
+ * can create a read stream to send.
+ * @param {string} contents the contents of the file which is to be returned from the API.
+ * @return {Object} the read stream which can be put into the koa context body.
+ */
+export const apiFileReturn = (contents: any) => {
+  const path = join(budibaseTempDir(), uuid())
+  fs.writeFileSync(path, contents)
+  return fs.createReadStream(path)
+}
+
+export const streamFile = (path: string) => {
+  return fs.createReadStream(path)
+}
+
+/**
+ * Writes the provided contents to a temporary file, which can be used briefly.
+ * @param {string} fileContents contents which will be written to a temp file.
+ * @return {string} the path to the temp file.
+ */
+export const storeTempFile = (fileContents: any) => {
+  const path = join(budibaseTempDir(), uuid())
+  fs.writeFileSync(path, fileContents)
+  return path
+}
+
+/**
+ * Utility function for getting a file read stream - a simple in memory buffered read
+ * stream doesn't work for pouchdb.
+ */
+export const stringToFileStream = (contents: any) => {
+  const path = storeTempFile(contents)
+  return fs.createReadStream(path)
+}
+
+/**
+ * Creates a temp file and returns it from the API.
+ * @param {string} fileContents the contents to be returned in file.
+ */
+export const sendTempFile = (fileContents: any) => {
+  const path = storeTempFile(fileContents)
+  return fs.createReadStream(path)
+}
+
+/**
+ * All file reads come through here just to make sure all of them make sense
+ * allows a centralised location to check logic is all good.
+ */
+export const readFileSync = (filepath: PathLike, options = "utf8") => {
+  // @ts-ignore
+  return fs.readFileSync(filepath, options)
+}
+
+export const createTempFolder = (item: any) => {
+  const path = join(budibaseTempDir(), item)
+  try {
+    // remove old tmp directories automatically - don't combine
+    if (fs.existsSync(path)) {
+      fs.rmSync(path, { recursive: true, force: true })
+    }
+    fs.mkdirSync(path)
+  } catch (err: any) {
+    throw new Error(`Path cannot be created: ${err.message}`)
+  }
+
+  return path
+}
+
+export const extractTarball = async (fromFilePath: string, toPath: string) => {
+  await tar.extract({
+    file: fromFilePath,
+    C: toPath,
+  })
+}
+
+/**
+ * Find for a file recursively from start path applying filter, return first match
+ */
+export const findFileRec = (startPath: PathLike, filter: string): any => {
+  if (!fs.existsSync(startPath)) {
+    return
+  }
+
+  const files = fs.readdirSync(startPath)
+  for (let i = 0, len = files.length; i < len; i++) {
+    // @ts-ignore
+    const filename = join(startPath, files[i])
+    const stat = fs.lstatSync(filename)
+
+    if (stat.isDirectory()) {
+      return findFileRec(filename, filter)
+    } else if (filename.endsWith(filter)) {
+      return filename
+    }
+  }
+}
+
+/**
+ * Remove a folder which is not empty from the file system
+ */
+export const deleteFolderFileSystem = (path: PathLike) => {
+  if (!fs.existsSync(path)) {
+    return
+  }
+
+  fs.rmSync(path, { recursive: true, force: true })
+}

package/src/utilities/fileSystem/index.ts

@@ -0,0 +1,5 @@
+export * from "./app"
+export * from "./clientLibrary"
+export * from "./filesystem"
+export * from "./plugin"
+export * from "./template"

package/src/utilities/fileSystem/plugin.ts

@@ -0,0 +1,73 @@
+import { Plugin } from "@budibase/types"
+import { budibaseTempDir } from "../budibaseDir"
+import fs from "fs"
+import { join } from "path"
+import { objectStore } from "@budibase/backend-core"
+
+const DATASOURCE_PATH = join(budibaseTempDir(), "datasource")
+const AUTOMATION_PATH = join(budibaseTempDir(), "automation")
+
+export const getPluginMetadata = async (path: string) => {
+  let metadata: any = {}
+  try {
+    const pkg = fs.readFileSync(join(path, "package.json"), "utf8")
+    const schema = fs.readFileSync(join(path, "schema.json"), "utf8")
+
+    metadata.schema = JSON.parse(schema)
+    metadata.package = JSON.parse(pkg)
+
+    if (
+      !metadata.package.name ||
+      !metadata.package.version ||
+      !metadata.package.description
+    ) {
+      throw new Error(
+        "package.json is missing one of 'name', 'version' or 'description'."
+      )
+    }
+  } catch (err: any) {
+    throw new Error(
+      `Unable to process schema.json/package.json in plugin. ${err.message}`
+    )
+  }
+
+  return { metadata, directory: path }
+}
+
+async function getPluginImpl(path: string, plugin: Plugin) {
+  const hash = plugin.schema?.hash
+  if (!fs.existsSync(path)) {
+    fs.mkdirSync(path)
+  }
+  const filename = join(path, plugin.name)
+  const metadataName = `${filename}.bbmetadata`
+  if (fs.existsSync(filename)) {
+    const currentHash = fs.readFileSync(metadataName, "utf8")
+    // if hash is the same return the file, otherwise remove it and re-download
+    if (currentHash === hash) {
+      return require(filename)
+    } else {
+      console.log(`Updating plugin: ${plugin.name}`)
+      delete require.cache[require.resolve(filename)]
+      fs.unlinkSync(filename)
+    }
+  }
+  const pluginKey = objectStore.getPluginJSKey(plugin)
+  const pluginJs = await objectStore.retrieve(
+    objectStore.ObjectStoreBuckets.PLUGINS,
+    pluginKey
+  )
+
+  fs.writeFileSync(filename, pluginJs)
+  fs.writeFileSync(metadataName, hash)
+
+  return require(filename)
+}
+
+export const getDatasourcePlugin = async (plugin: Plugin) => {
+  return getPluginImpl(DATASOURCE_PATH, plugin)
+}
+
+export const getAutomationPlugin = async (plugin: Plugin) => {
+  return getPluginImpl(AUTOMATION_PATH, plugin)
+}

package/src/utilities/fileSystem/processor.ts

@@ -0,0 +1,19 @@
+import jimp from "jimp"
+
+const FORMATS = {
+  IMAGES: ["png", "jpg", "jpeg", "gif", "bmp", "tiff"],
+}
+
+function processImage(file: { path: string }) {
+  // this will overwrite the temp file
+  return jimp.read(file.path).then(img => {
+    return img.resize(300, jimp.AUTO).write(file.path)
+  })
+}
+
+export async function process(file: { extension: string; path: string }) {
+  if (FORMATS.IMAGES.includes(file.extension.toLowerCase())) {
+    await processImage(file)
+  }
+  return file
+}

package/src/utilities/fileSystem/template.ts

@@ -0,0 +1,36 @@
+import fs from "fs"
+import { join } from "path"
+import { ObjectStoreBuckets } from "../../constants"
+import { objectStore } from "@budibase/backend-core"
+
+/**
+ * This function manages temporary template files which are stored by Koa.
+ * @param {Object} template The template object retrieved from the Koa context object.
+ * @returns {Object} Returns an fs read stream which can be loaded into the database.
+ */
+export const getTemplateStream = async (template: any) => {
+  if (template.file) {
+    return fs.createReadStream(template.file.path)
+  } else {
+    const [type, name] = template.key.split("/")
+    const tmpPath = await downloadTemplate(type, name)
+    return fs.createReadStream(join(tmpPath, name, "db", "dump.txt"))
+  }
+}
+
+/**
+ * Retrieves a template and pipes it to minio as well as making it available temporarily.
+ * @param {string} type The type of template which is to be retrieved.
+ * @param name
+ * @return {Promise<*>}
+ */
+export const downloadTemplate = async (type: string, name: string) => {
+  const DEFAULT_TEMPLATES_BUCKET =
+    "prod-budi-templates.s3-eu-west-1.amazonaws.com"
+  const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz`
+  return objectStore.downloadTarball(
+    templateUrl,
+    ObjectStoreBuckets.TEMPLATES,
+    type
+  )
+}

package/src/utilities/global.ts

@@ -0,0 +1,153 @@
+import { getMultiIDParams, getGlobalIDFromUserMetadataID } from "../db/utils"
+import {
+  roles,
+  db as dbCore,
+  cache,
+  tenancy,
+  context,
+} from "@budibase/backend-core"
+import env from "../environment"
+import { groups } from "@budibase/pro"
+import { UserCtx, ContextUser, User, UserGroup } from "@budibase/types"
+import { global } from "yargs"
+
+export function updateAppRole(
+  user: ContextUser,
+  { appId }: { appId?: string } = {}
+) {
+  appId = appId || context.getAppId()
+
+  if (!user || (!user.roles && !user.userGroups)) {
+    return user
+  }
+  // if in an multi-tenancy environment make sure roles are never updated
+  if (env.MULTI_TENANCY && appId && !tenancy.isUserInAppTenant(appId, user)) {
+    delete user.builder
+    delete user.admin
+    user.roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
+    return user
+  }
+  // always use the deployed app
+  if (appId && user.roles) {
+    user.roleId = user.roles[dbCore.getProdAppID(appId)]
+  }
+  // if a role wasn't found then either set as admin (builder) or public (everyone else)
+  if (!user.roleId && user.builder && user.builder.global) {
+    user.roleId = roles.BUILTIN_ROLE_IDS.ADMIN
+  } else if (!user.roleId && !user?.userGroups?.length) {
+    user.roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
+  }
+
+  delete user.roles
+  return user
+}
+
+async function checkGroupRoles(
+  user: ContextUser,
+  opts: { appId?: string; groups?: UserGroup[] } = {}
+) {
+  if (user.roleId && user.roleId !== roles.BUILTIN_ROLE_IDS.PUBLIC) {
+    return user
+  }
+  if (opts.appId) {
+    user.roleId = await groups.getGroupRoleId(user as User, opts.appId, {
+      groups: opts.groups,
+    })
+  }
+  // final fallback, simply couldn't find a role - user must be public
+  if (!user.roleId) {
+    user.roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
+  }
+  return user
+}
+
+export async function processUser(
+  user: ContextUser,
+  opts: { appId?: string; groups?: UserGroup[] } = {}
+) {
+  if (user) {
+    delete user.password
+  }
+  const appId = opts.appId || context.getAppId()
+  user = updateAppRole(user, { appId })
+  if (!user.roleId && user?.userGroups?.length) {
+    user = await checkGroupRoles(user, { appId, groups: opts?.groups })
+  }
+
+  return user
+}
+
+export async function getCachedSelf(ctx: UserCtx, appId: string) {
+  // this has to be tenant aware, can't depend on the context to find it out
+  // running some middlewares before the tenancy causes context to break
+  const user = await cache.user.getUser(ctx.user?._id!)
+  return processUser(user, { appId })
+}
+
+export async function getRawGlobalUser(userId: string) {
+  const db = tenancy.getGlobalDB()
+  return db.get(getGlobalIDFromUserMetadataID(userId))
+}
+
+export async function getGlobalUser(userId: string) {
+  const appId = context.getAppId()
+  let user = await getRawGlobalUser(userId)
+  return processUser(user, { appId })
+}
+
+export async function getGlobalUsers(
+  userIds?: string[],
+  opts?: { noProcessing?: boolean }
+) {
+  const appId = context.getAppId()
+  const db = tenancy.getGlobalDB()
+  let globalUsers
+  if (userIds) {
+    globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map(
+      row => row.doc
+    )
+  } else {
+    globalUsers = (
+      await db.allDocs(
+        dbCore.getGlobalUserParams(null, {
+          include_docs: true,
+        })
+      )
+    ).rows.map(row => row.doc)
+  }
+  globalUsers = globalUsers
+    .filter(user => user != null)
+    .map(user => {
+      delete user.password
+      delete user.forceResetPassword
+      return user
+    })
+  if (!appId) {
+    return globalUsers
+  }
+
+  if (opts?.noProcessing) {
+    return globalUsers
+  } else {
+    // pass in the groups, meaning we don't actually need to retrieve them for
+    // each user individually
+    const allGroups = await groups.fetch()
+    return Promise.all(
+      globalUsers.map(user => processUser(user, { groups: allGroups }))
+    )
+  }
+}
+
+export async function getGlobalUsersFromMetadata(users: ContextUser[]) {
+  const globalUsers = await getGlobalUsers(users.map(user => user._id!))
+  return users.map(user => {
+    const globalUser = globalUsers.find(
+      globalUser => globalUser && user._id?.includes(globalUser._id)
+    )
+    return {
+      ...globalUser,
+      // doing user second overwrites the id and rev (always metadata)
+      ...user,
+    }
+  })
+}

package/src/utilities/index.ts

@@ -0,0 +1,134 @@
+import env from "../environment"
+import { context } from "@budibase/backend-core"
+import { generateMetadataID } from "../db/utils"
+import { Document } from "@budibase/types"
+import stream from "stream"
+const Readable = stream.Readable
+
+export function wait(ms: number) {
+  return new Promise(resolve => setTimeout(resolve, ms))
+}
+
+export const isDev = env.isDev
+
+export const NUMBER_REGEX = /^[+-]?([0-9]*[.])?[0-9]+$/g
+
+export function removeFromArray(array: any[], element: any) {
+  const index = array.indexOf(element)
+  if (index !== -1) {
+    array.splice(index, 1)
+  }
+  return array
+}
+
+/**
+ * Makes sure that a URL has the correct number of slashes, while maintaining the
+ * http(s):// double slashes.
+ * @param {string} url The URL to test and remove any extra double slashes.
+ * @return {string} The updated url.
+ */
+export function checkSlashesInUrl(url: string) {
+  return url.replace(/(https?:\/\/)|(\/)+/g, "$1$2")
+}
+
+export async function updateEntityMetadata(
+  type: string,
+  entityId: string,
+  updateFn: any
+) {
+  const db = context.getAppDB()
+  const id = generateMetadataID(type, entityId)
+  // read it to see if it exists, we'll overwrite it no matter what
+  let rev, metadata: Document
+  try {
+    const oldMetadata = await db.get(id)
+    rev = oldMetadata._rev
+    metadata = updateFn(oldMetadata)
+  } catch (err) {
+    rev = null
+    metadata = updateFn({})
+  }
+  metadata._id = id
+  if (rev) {
+    metadata._rev = rev
+  }
+  const response = await db.put(metadata)
+  return {
+    ...metadata,
+    _id: id,
+    _rev: response.rev,
+  }
+}
+
+export async function saveEntityMetadata(
+  type: string,
+  entityId: string,
+  metadata: Document
+) {
+  return updateEntityMetadata(type, entityId, () => {
+    return metadata
+  })
+}
+
+export async function deleteEntityMetadata(type: string, entityId: string) {
+  const db = context.getAppDB()
+  const id = generateMetadataID(type, entityId)
+  let rev
+  try {
+    const metadata = await db.get(id)
+    if (metadata) {
+      rev = metadata._rev
+    }
+  } catch (err) {
+    // don't need to error if it doesn't exist
+  }
+  if (id && rev) {
+    await db.remove(id, rev)
+  }
+}
+
+export function escapeDangerousCharacters(string: string) {
+  return string
+    .replace(/[\\]/g, "\\\\")
+    .replace(/[\b]/g, "\\b")
+    .replace(/[\f]/g, "\\f")
+    .replace(/[\n]/g, "\\n")
+    .replace(/[\r]/g, "\\r")
+    .replace(/[\t]/g, "\\t")
+}
+
+export function stringToReadStream(string: string) {
+  return new Readable({
+    read() {
+      this.push(string)
+      this.push(null)
+    },
+  })
+}
+
+export function formatBytes(bytes: string) {
+  const units = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]
+  const byteIncrements = 1024
+  let unit = 0
+  let size = parseInt(bytes, 10) || 0
+  while (size >= byteIncrements && ++unit) {
+    size /= byteIncrements
+  }
+  return `${size.toFixed(size < 10 && unit > 0 ? 1 : 0)}${units[unit]}`
+}
+
+export function convertBookmark(bookmark: string) {
+  const IS_NUMBER = /^\d+\.?\d*$/
+  if (typeof bookmark === "string" && bookmark.match(IS_NUMBER)) {
+    return parseFloat(bookmark)
+  }
+  return bookmark
+}
+
+export function isQsTrue(param: string) {
+  if (typeof param === "string") {
+    return param.toLowerCase() === "true"
+  } else {
+    return param === true
+  }
+}

package/src/utilities/redis.ts

@@ -0,0 +1,88 @@
+import { redis } from "@budibase/backend-core"
+import { getGlobalIDFromUserMetadataID } from "../db/utils"
+import { ContextUser } from "@budibase/types"
+
+const APP_DEV_LOCK_SECONDS = 600
+const AUTOMATION_TEST_FLAG_SECONDS = 60
+let devAppClient: any, debounceClient: any, flagClient: any
+
+// we init this as we want to keep the connection open all the time
+// reduces the performance hit
+export async function init() {
+  devAppClient = new redis.Client(redis.utils.Databases.DEV_LOCKS)
+  debounceClient = new redis.Client(redis.utils.Databases.DEBOUNCE)
+  flagClient = new redis.Client(redis.utils.Databases.FLAGS)
+  await devAppClient.init()
+  await debounceClient.init()
+  await flagClient.init()
+}
+
+export async function shutdown() {
+  if (devAppClient) await devAppClient.finish()
+  if (debounceClient) await debounceClient.finish()
+  if (flagClient) await flagClient.finish()
+  // shutdown core clients
+  await redis.clients.shutdown()
+  console.log("Redis shutdown")
+}
+
+export async function doesUserHaveLock(devAppId: string, user: ContextUser) {
+  const value = await devAppClient.get(devAppId)
+  if (!value) {
+    return true
+  }
+  // make sure both IDs are global
+  const expected = getGlobalIDFromUserMetadataID(value._id)
+  const userId = getGlobalIDFromUserMetadataID(user._id!)
+  return expected === userId
+}
+
+export async function getLocksById(appIds: string[]) {
+  return await devAppClient.bulkGet(appIds)
+}
+
+export async function updateLock(devAppId: string, user: ContextUser) {
+  // make sure always global user ID
+  const globalId = getGlobalIDFromUserMetadataID(user._id!)
+  const inputUser = {
+    ...user,
+    userId: globalId,
+    _id: globalId,
+    lockedAt: new Date().getTime(),
+  }
+
+  await devAppClient.store(devAppId, inputUser, APP_DEV_LOCK_SECONDS)
+}
+
+export async function clearLock(devAppId: string, user: ContextUser) {
+  const value = await devAppClient.get(devAppId)
+  if (!value) {
+    return
+  }
+  const userId = getGlobalIDFromUserMetadataID(user._id!)
+  if (value._id !== userId) {
+    throw "User does not hold lock, cannot clear it."
+  }
+  await devAppClient.delete(devAppId)
+}
+
+export async function checkDebounce(id: string) {
+  return debounceClient.get(id)
+}
+
+export async function setDebounce(id: string, seconds: number) {
+  await debounceClient.store(id, "debouncing", seconds)
+}
+
+export async function setTestFlag(id: string) {
+  await flagClient.store(id, { testing: true }, AUTOMATION_TEST_FLAG_SECONDS)
+}
+
+export async function checkTestFlag(id: string) {
+  const flag = await flagClient.get(id)
+  return !!(flag && flag.testing)
+}
+
+export async function clearTestFlag(id: string) {
+  await devAppClient.delete(id)
+}

package/src/utilities/retry.ts

@@ -0,0 +1,18 @@
+export async function retry<T extends (...arg0: any[]) => any>(
+  fn: T,
+  maxTry: number = 5,
+  retryCount = 1
+): Promise<Awaited<ReturnType<T>>> {
+  const currRetry = typeof retryCount === "number" ? retryCount : 1
+  try {
+    const result = await fn()
+    return result
+  } catch (e) {
+    console.log(`Retry ${currRetry} failed.`)
+    if (currRetry > maxTry) {
+      console.log(`All ${maxTry} retry attempts exhausted`)
+      throw e
+    }
+    return retry(fn, maxTry, currRetry + 1)
+  }
+}