@budibase/server 1.0.49-alpha.6 → 1.0.49

package/src/threads/automation.js

@@ -5,11 +5,11 @@ const automationUtils = require("../automations/automationUtils")
 const AutomationEmitter = require("../events/AutomationEmitter")
 const { processObject } = require("@budibase/string-templates")
 const { DEFAULT_TENANT_ID } = require("@budibase/backend-core/constants")
+const CouchDB = require("../db")
 const { DocumentTypes, isDevAppID } = require("../db/utils")
 const { doInTenant } = require("@budibase/backend-core/tenancy")
 const usage = require("../utilities/usageQuota")
 const { definitions: triggerDefs } = require("../automations/triggerInfo")
-const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
 
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
 const CRON_STEP_ID = triggerDefs.CRON.stepId
@@ -59,10 +59,11 @@ class Orchestrator {
   }
 
   async getApp() {
+    const appId = this._appId
     if (this._app) {
       return this._app
     }
-    const db = getAppDB()
+    const db = new CouchDB(appId)
     this._app = await db.get(DocumentTypes.APP_METADATA)
     return this._app
   }
@@ -130,19 +131,16 @@ class Orchestrator {
 }
 
 module.exports = (input, callback) => {
-  const appId = input.data.event.appId
-  doInAppContext(appId, () => {
-    const automationOrchestrator = new Orchestrator(
-      input.data.automation,
-      input.data.event
-    )
-    automationOrchestrator
-      .execute()
-      .then(response => {
-        callback(null, response)
-      })
-      .catch(err => {
-        callback(err)
-      })
-  })
+  const automationOrchestrator = new Orchestrator(
+    input.data.automation,
+    input.data.event
+  )
+  automationOrchestrator
+    .execute()
+    .then(response => {
+      callback(null, response)
+    })
+    .catch(err => {
+      callback(err)
+    })
 }

package/src/threads/query.js

@@ -3,13 +3,11 @@ threadUtils.threadSetup()
 const ScriptRunner = require("../utilities/scriptRunner")
 const { integrations } = require("../integrations")
 const { processStringSync } = require("@budibase/string-templates")
-const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
-
-const IS_TRIPLE_BRACE = new RegExp(/^{{3}.*}{3}$/)
-const IS_HANDLEBARS = new RegExp(/^{{2}.*}{2}$/)
+const CouchDB = require("../db")
 
 class QueryRunner {
   constructor(input, flags = { noRecursiveQuery: false }) {
+    this.appId = input.appId
     this.datasource = input.datasource
     this.queryVerb = input.queryVerb
     this.fields = input.fields
@@ -77,11 +75,6 @@ class QueryRunner {
       return this.execute()
     }
 
-    // check for undefined response
-    if (!rows) {
-      rows = []
-    }
-
     // needs to an array for next step
     if (!Array.isArray(rows)) {
       rows = [rows]
@@ -103,11 +96,12 @@ class QueryRunner {
   }
 
   async runAnotherQuery(queryId, parameters) {
-    const db = getAppDB()
+    const db = new CouchDB(this.appId)
     const query = await db.get(queryId)
     const datasource = await db.get(query.datasourceId)
     return new QueryRunner(
       {
+        appId: this.appId,
         datasource,
         queryVerb: query.queryVerb,
         fields: query.fields,
@@ -188,12 +182,8 @@ class QueryRunner {
         enrichedQuery[key] = this.enrichQueryFields(fields[key], parameters)
       } else if (typeof fields[key] === "string") {
         // enrich string value as normal
-        let value = fields[key]
-        // add triple brace to avoid escaping e.g. '=' in cookie header
-        if (IS_HANDLEBARS.test(value) && !IS_TRIPLE_BRACE.test(value)) {
-          value = `{${value}}`
-        }
-        enrichedQuery[key] = processStringSync(value, parameters, {
+        enrichedQuery[key] = processStringSync(fields[key], parameters, {
+          noEscaping: true,
           noHelpers: true,
         })
       } else {
@@ -221,14 +211,12 @@ class QueryRunner {
 }
 
 module.exports = (input, callback) => {
-  doInAppContext(input.appId, () => {
-    const Runner = new QueryRunner(input)
-    Runner.execute()
-      .then(response => {
-        callback(null, response)
-      })
-      .catch(err => {
-        callback(err)
-      })
-  })
+  const Runner = new QueryRunner(input)
+  Runner.execute()
+    .then(response => {
+      callback(null, response)
+    })
+    .catch(err => {
+      callback(err)
+    })
 }

package/src/utilities/fileSystem/index.js

@@ -1,4 +1,5 @@
 const { budibaseTempDir } = require("../budibaseDir")
+const { isDev } = require("../index")
 const fs = require("fs")
 const { join } = require("path")
 const uuid = require("uuid/v4")
@@ -19,7 +20,6 @@ const {
   LINK_USER_METADATA_PREFIX,
 } = require("../../db/utils")
 const MemoryStream = require("memorystream")
-const { getAppId } = require("@budibase/backend-core/context")
 
 const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
 const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
@@ -51,7 +51,7 @@ exports.init = () => {
  * everything required to function is ready.
  */
 exports.checkDevelopmentEnvironment = () => {
-  if (!env.isDev() || env.isTest()) {
+  if (!isDev()) {
     return
   }
   if (!fs.existsSync(budibaseTempDir())) {
@@ -251,8 +251,7 @@ exports.downloadTemplate = async (type, name) => {
 /**
  * Retrieves component libraries from object store (or tmp symlink if in local)
  */
-exports.getComponentLibraryManifest = async library => {
-  const appId = getAppId()
+exports.getComponentLibraryManifest = async (appId, library) => {
   const filename = "manifest.json"
   /* istanbul ignore next */
   // when testing in cypress and so on we need to get the package

package/src/utilities/global.js

@@ -3,7 +3,7 @@ const {
   getGlobalIDFromUserMetadataID,
 } = require("../db/utils")
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
-const { getProdAppID } = require("@budibase/backend-core/db")
+const { getDeployedAppID } = require("@budibase/backend-core/db")
 const { getGlobalUserParams } = require("@budibase/backend-core/db")
 const { user: userCache } = require("@budibase/backend-core/cache")
 const {
@@ -11,10 +11,8 @@ const {
   isUserInAppTenant,
 } = require("@budibase/backend-core/tenancy")
 const env = require("../environment")
-const { getAppId } = require("@budibase/backend-core/context")
 
-exports.updateAppRole = (user, { appId } = {}) => {
-  appId = appId || getAppId()
+exports.updateAppRole = (appId, user) => {
   if (!user || !user.roles) {
     return user
   }
@@ -26,7 +24,7 @@ exports.updateAppRole = (user, { appId } = {}) => {
     return user
   }
   // always use the deployed app
-  user.roleId = user.roles[getProdAppID(appId)]
+  user.roleId = user.roles[getDeployedAppID(appId)]
   // if a role wasn't found then either set as admin (builder) or public (everyone else)
   if (!user.roleId && user.builder && user.builder.global) {
     user.roleId = BUILTIN_ROLE_IDS.ADMIN
@@ -37,18 +35,18 @@ exports.updateAppRole = (user, { appId } = {}) => {
   return user
 }
 
-function processUser(user, { appId } = {}) {
+function processUser(appId, user) {
   if (user) {
     delete user.password
   }
-  return exports.updateAppRole(user, { appId })
+  return exports.updateAppRole(appId, user)
 }
 
 exports.getCachedSelf = async (ctx, appId) => {
   // this has to be tenant aware, can't depend on the context to find it out
   // running some middlewares before the tenancy causes context to break
   const user = await userCache.getUser(ctx.user._id)
-  return processUser(user, { appId })
+  return processUser(appId, user)
 }
 
 exports.getRawGlobalUser = async userId => {
@@ -56,13 +54,12 @@ exports.getRawGlobalUser = async userId => {
   return db.get(getGlobalIDFromUserMetadataID(userId))
 }
 
-exports.getGlobalUser = async userId => {
+exports.getGlobalUser = async (appId, userId) => {
   let user = await exports.getRawGlobalUser(userId)
-  return processUser(user)
+  return processUser(appId, user)
 }
 
-exports.getGlobalUsers = async (users = null) => {
-  const appId = getAppId()
+exports.getGlobalUsers = async (appId = null, users = null) => {
   const db = getGlobalDB()
   let globalUsers
   if (users) {
@@ -89,11 +86,11 @@ exports.getGlobalUsers = async (users = null) => {
   if (!appId) {
     return globalUsers
   }
-  return globalUsers.map(user => exports.updateAppRole(user))
+  return globalUsers.map(user => exports.updateAppRole(appId, user))
 }
 
-exports.getGlobalUsersFromMetadata = async users => {
-  const globalUsers = await exports.getGlobalUsers(users)
+exports.getGlobalUsersFromMetadata = async (appId, users) => {
+  const globalUsers = await exports.getGlobalUsers(appId, users)
   return users.map(user => {
     const globalUser = globalUsers.find(
       globalUser => globalUser && user._id.includes(globalUser._id)

package/src/utilities/index.js

@@ -1,9 +1,9 @@
 const env = require("../environment")
 const { OBJ_STORE_DIRECTORY } = require("../constants")
 const { sanitizeKey } = require("@budibase/backend-core/objectStore")
+const CouchDB = require("../db")
 const { generateMetadataID } = require("../db/utils")
 const Readable = require("stream").Readable
-const { getAppDB } = require("@budibase/backend-core/context")
 
 const BB_CDN = "https://cdn.budi.live"
 
@@ -73,8 +73,8 @@ exports.attachmentsRelativeURL = attachmentKey => {
   )
 }
 
-exports.updateEntityMetadata = async (type, entityId, updateFn) => {
-  const db = getAppDB()
+exports.updateEntityMetadata = async (appId, type, entityId, updateFn) => {
+  const db = new CouchDB(appId)
   const id = generateMetadataID(type, entityId)
   // read it to see if it exists, we'll overwrite it no matter what
   let rev,
@@ -99,14 +99,14 @@ exports.updateEntityMetadata = async (type, entityId, updateFn) => {
   }
 }
 
-exports.saveEntityMetadata = async (type, entityId, metadata) => {
-  return exports.updateEntityMetadata(type, entityId, () => {
+exports.saveEntityMetadata = async (appId, type, entityId, metadata) => {
+  return exports.updateEntityMetadata(appId, type, entityId, () => {
     return metadata
   })
 }
 
-exports.deleteEntityMetadata = async (type, entityId) => {
-  const db = getAppDB()
+exports.deleteEntityMetadata = async (appId, type, entityId) => {
+  const db = new CouchDB(appId)
   const id = generateMetadataID(type, entityId)
   let rev
   try {
@@ -141,6 +141,16 @@ exports.stringToReadStream = string => {
   })
 }
 
+exports.doesDatabaseExist = async dbName => {
+  try {
+    const db = new CouchDB(dbName, { skip_setup: true })
+    const info = await db.info()
+    return info && !info.error
+  } catch (err) {
+    return false
+  }
+}
+
 exports.formatBytes = bytes => {
   const units = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]
   const byteIncrements = 1024

package/src/utilities/routing/index.js

@@ -1,9 +1,9 @@
+const CouchDB = require("../../db")
 const { createRoutingView } = require("../../db/views/staticViews")
 const { ViewNames, getQueryIndex, UNICODE_MAX } = require("../../db/utils")
-const { getAppDB } = require("@budibase/backend-core/context")
 
-exports.getRoutingInfo = async () => {
-  const db = getAppDB()
+exports.getRoutingInfo = async appId => {
+  const db = new CouchDB(appId)
   try {
     const allRouting = await db.query(getQueryIndex(ViewNames.ROUTING), {
       startKey: "",
@@ -14,8 +14,8 @@ exports.getRoutingInfo = async () => {
     // check if the view doesn't exist, it should for all new instances
     /* istanbul ignore next */
     if (err != null && err.name === "not_found") {
-      await createRoutingView()
-      return exports.getRoutingInfo()
+      await createRoutingView(appId)
+      return exports.getRoutingInfo(appId)
     } else {
       throw err
     }

package/src/utilities/rowProcessor/index.js

@@ -7,10 +7,10 @@ const { deleteFiles } = require("../../utilities/fileSystem/utilities")
 const { ObjectStoreBuckets } = require("../../constants")
 const {
   isProdAppID,
-  getProdAppID,
+  getDeployedAppID,
   dbExists,
 } = require("@budibase/backend-core/db")
-const { getAppId } = require("@budibase/backend-core/context")
+const CouchDB = require("../../db")
 
 const BASE_AUTO_ID = 1
 
@@ -180,8 +180,6 @@ function processAutoColumn(
 }
 exports.processAutoColumn = processAutoColumn
 
-exports.processFormulas = processFormulas
-
 /**
  * This will coerce a value to the correct types based on the type transform map
  * @param {object} row The value to coerce
@@ -231,12 +229,11 @@ exports.inputProcessing = (
       }
       continue
     }
-    // remove any formula values, they are to be generated
+    // specific case to delete formula values if they get saved
+    // type coercion cannot completely remove the field, so have to do it here
     if (field.type === FieldTypes.FORMULA) {
       delete clonedRow[key]
-    }
-    // otherwise coerce what is there to correct types
-    else {
+    } else {
       clonedRow[key] = exports.coerce(value, field.type)
     }
   }
@@ -253,23 +250,30 @@ exports.inputProcessing = (
 /**
  * This function enriches the input rows with anything they are supposed to contain, for example
  * link records or attachment links.
+ * @param {object} ctx the request which is looking for enriched rows.
  * @param {object} table the table from which these rows came from originally, this is used to determine
  * the schema of the rows and then enrich.
  * @param {object[]|object} rows the rows which are to be enriched.
  * @param {object} opts used to set some options for the output, such as disabling relationship squashing.
  * @returns {object[]|object} the enriched rows will be returned.
  */
-exports.outputProcessing = async (table, rows, opts = { squash: true }) => {
+exports.outputProcessing = async (
+  ctx,
+  table,
+  rows,
+  opts = { squash: true }
+) => {
+  const appId = ctx.appId
   let wasArray = true
   if (!(rows instanceof Array)) {
     rows = [rows]
     wasArray = false
   }
   // attach any linked row information
-  let enriched = await linkRows.attachFullLinkedDocs(table, rows)
+  let enriched = await linkRows.attachFullLinkedDocs(ctx, table, rows)
 
   // process formulas
-  enriched = processFormulas(table, enriched, { dynamic: true })
+  enriched = processFormulas(table, enriched)
 
   // update the attachments URL depending on hosting
   for (let [property, column] of Object.entries(table.schema)) {
@@ -285,27 +289,29 @@ exports.outputProcessing = async (table, rows, opts = { squash: true }) => {
     }
   }
   if (opts.squash) {
-    enriched = await linkRows.squashLinksToPrimaryDisplay(table, enriched)
+    enriched = await linkRows.squashLinksToPrimaryDisplay(
+      appId,
+      table,
+      enriched
+    )
   }
   return wasArray ? enriched : enriched[0]
 }
 
 /**
  * Clean up any attachments that were attached to a row.
+ * @param {string} appId The ID of the app from which a row is being deleted.
  * @param {object} table The table from which a row is being removed.
  * @param {any} row optional - the row being removed.
  * @param {any} rows optional - if multiple rows being deleted can do this in bulk.
  * @param {any} oldRow optional - if updating a row this will determine the difference.
- * @param {any} oldTable optional - if updating a table, can supply the old table to look for
- * deleted attachment columns.
  * @return {Promise<void>} When all attachments have been removed this will return.
  */
-exports.cleanupAttachments = async (table, { row, rows, oldRow, oldTable }) => {
-  const appId = getAppId()
+exports.cleanupAttachments = async (appId, table, { row, rows, oldRow }) => {
   if (!isProdAppID(appId)) {
-    const prodAppId = getProdAppID(appId)
+    const prodAppId = getDeployedAppID(appId)
     // if prod exists, then don't allow deleting
-    const exists = await dbExists(prodAppId)
+    const exists = await dbExists(CouchDB, prodAppId)
     if (exists) {
       return
     }
@@ -316,16 +322,12 @@ exports.cleanupAttachments = async (table, { row, rows, oldRow, oldTable }) => {
       files = files.concat(row[key].map(attachment => attachment.key))
     }
   }
-  const schemaToUse = oldTable ? oldTable.schema : table.schema
-  for (let [key, schema] of Object.entries(schemaToUse)) {
+  for (let [key, schema] of Object.entries(table.schema)) {
     if (schema.type !== FieldTypes.ATTACHMENT) {
       continue
     }
-    // old table had this column, new table doesn't - delete it
-    if (oldTable && !table.schema[key]) {
-      rows.forEach(row => addFiles(row, key))
-    } else if (oldRow && row) {
-      // if updating, need to manage the differences
+    // if updating, need to manage the differences
+    if (oldRow && row) {
       files = files.concat(getRemovedAttachmentKeys(oldRow, row, key))
     } else if (row) {
       addFiles(row, key)

package/src/utilities/rowProcessor/utils.js

@@ -1,39 +1,23 @@
-const { FieldTypes, FormulaTypes } = require("../../constants")
+const { FieldTypes } = require("../../constants")
 const { processStringSync } = require("@budibase/string-templates")
 
 /**
  * Looks through the rows provided and finds formulas - which it then processes.
  */
-exports.processFormulas = (
-  table,
-  rows,
-  { dynamic, contextRows } = { dynamic: true }
-) => {
+exports.processFormulas = (table, rows) => {
   const single = !Array.isArray(rows)
   if (single) {
     rows = [rows]
-    contextRows = contextRows ? [contextRows] : contextRows
   }
   for (let [column, schema] of Object.entries(table.schema)) {
-    const isStatic = schema.formulaType === FormulaTypes.STATIC
-    if (
-      schema.type !== FieldTypes.FORMULA ||
-      (dynamic && isStatic) ||
-      (!dynamic && !isStatic)
-    ) {
+    if (schema.type !== FieldTypes.FORMULA) {
       continue
     }
     // iterate through rows and process formula
-    for (let i = 0; i < rows.length; i++) {
-      if (schema.formula) {
-        let row = rows[i]
-        let context = contextRows ? contextRows[i] : row
-        rows[i] = {
-          ...row,
-          [column]: processStringSync(schema.formula, context),
-        }
-      }
-    }
+    rows = rows.map(row => ({
+      ...row,
+      [column]: processStringSync(schema.formula, row),
+    }))
   }
   return single ? rows[0] : rows
 }

package/src/utilities/usageQuota.js

@@ -0,0 +1,73 @@
+const env = require("../environment")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const {
+  StaticDatabases,
+  generateNewUsageQuotaDoc,
+} = require("@budibase/backend-core/db")
+
+function getNewQuotaReset() {
+  return Date.now() + 2592000000
+}
+
+exports.Properties = {
+  ROW: "rows",
+  UPLOAD: "storage",
+  VIEW: "views",
+  USER: "users",
+  AUTOMATION: "automationRuns",
+  APPS: "apps",
+  EMAILS: "emails",
+}
+
+async function getUsageQuotaDoc(db) {
+  let quota
+  try {
+    quota = await db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota)
+  } catch (err) {
+    // doc doesn't exist. Create it
+    quota = await db.post(generateNewUsageQuotaDoc())
+  }
+
+  return quota
+}
+
+/**
+ * Given a specified tenantId this will add to the usage object for the specified property.
+ * @param {string} property The property which is to be added to (within the nested usageQuota object).
+ * @param {number} usage The amount (this can be negative) to adjust the number by.
+ * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
+ * also been reset after this call.
+ */
+exports.update = async (property, usage) => {
+  if (!env.USE_QUOTAS) {
+    return
+  }
+
+  try {
+    const db = getGlobalDB()
+    const quota = await getUsageQuotaDoc(db)
+
+    // Check if the quota needs reset
+    if (Date.now() >= quota.quotaReset) {
+      quota.quotaReset = getNewQuotaReset()
+      for (let prop of Object.keys(quota.usageQuota)) {
+        quota.usageQuota[prop] = 0
+      }
+    }
+
+    // increment the quota
+    quota.usageQuota[property] += usage
+
+    if (quota.usageQuota[property] > quota.usageLimits[property]) {
+      throw new Error(
+        `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
+      )
+    }
+
+    // update the usage quotas
+    await db.put(quota)
+  } catch (err) {
+    console.error(`Error updating usage quotas for ${property}`, err)
+    throw err
+  }
+}

package/src/utilities/users.js

@@ -1,13 +1,13 @@
+const CouchDB = require("../db")
 const { InternalTables } = require("../db/utils")
 const { getGlobalUser } = require("../utilities/global")
-const { getAppDB } = require("@budibase/backend-core/context")
 
 exports.getFullUser = async (ctx, userId) => {
-  const global = await getGlobalUser(userId)
+  const global = await getGlobalUser(ctx.appId, userId)
   let metadata
   try {
     // this will throw an error if the db doesn't exist, or there is no appId
-    const db = getAppDB()
+    const db = new CouchDB(ctx.appId)
     metadata = await db.get(userId)
   } catch (err) {
     // it is fine if there is no user metadata, just remove global db info

package/src/utilities/workerRequests.js

@@ -1,7 +1,7 @@
 const fetch = require("node-fetch")
 const env = require("../environment")
 const { checkSlashesInUrl } = require("./index")
-const { getProdAppID } = require("@budibase/backend-core/db")
+const { getDeployedAppID } = require("@budibase/backend-core/db")
 const { updateAppRole } = require("./global")
 const { Headers } = require("@budibase/backend-core/constants")
 const { getTenantId, isTenantIdSet } = require("@budibase/backend-core/tenancy")
@@ -58,6 +58,29 @@ exports.sendSmtpEmail = async (to, from, subject, contents, automation) => {
   return response.json()
 }
 
+exports.getDeployedApps = async () => {
+  try {
+    const response = await fetch(
+      checkSlashesInUrl(env.WORKER_URL + `/api/apps`),
+      request(null, {
+        method: "GET",
+      })
+    )
+    const json = await response.json()
+    const apps = {}
+    for (let [key, value] of Object.entries(json)) {
+      if (value.url) {
+        value.url = value.url.toLowerCase()
+        apps[key.toLowerCase()] = value
+      }
+    }
+    return apps
+  } catch (err) {
+    // error, cannot determine deployed apps, don't stop app creation - sort this later
+    return {}
+  }
+}
+
 exports.getGlobalSelf = async (ctx, appId = null) => {
   const endpoint = `/api/global/users/self`
   const response = await fetch(
@@ -70,15 +93,15 @@ exports.getGlobalSelf = async (ctx, appId = null) => {
   }
   let json = await response.json()
   if (appId) {
-    json = updateAppRole(json)
+    json = updateAppRole(appId, json)
   }
   return json
 }
 
 exports.removeAppFromUserRoles = async (ctx, appId) => {
-  const prodAppId = getProdAppID(appId)
+  const deployedAppId = getDeployedAppID(appId)
   const response = await fetch(
-    checkSlashesInUrl(env.WORKER_URL + `/api/global/roles/${prodAppId}`),
+    checkSlashesInUrl(env.WORKER_URL + `/api/global/roles/${deployedAppId}`),
     request(ctx, {
       method: "DELETE",
     })