@budibase/server 1.0.49 → 1.0.50-alpha.0

package/src/utilities/rowProcessor/index.js

@@ -7,10 +7,10 @@ const { deleteFiles } = require("../../utilities/fileSystem/utilities")
 const { ObjectStoreBuckets } = require("../../constants")
 const {
   isProdAppID,
-  getDeployedAppID,
+  getProdAppID,
   dbExists,
 } = require("@budibase/backend-core/db")
-const CouchDB = require("../../db")
+const { getAppId } = require("@budibase/backend-core/context")
 
 const BASE_AUTO_ID = 1
 
@@ -180,6 +180,8 @@ function processAutoColumn(
 }
 exports.processAutoColumn = processAutoColumn
 
+exports.processFormulas = processFormulas
+
 /**
  * This will coerce a value to the correct types based on the type transform map
  * @param {object} row The value to coerce
@@ -229,11 +231,12 @@ exports.inputProcessing = (
       }
       continue
     }
-    // specific case to delete formula values if they get saved
-    // type coercion cannot completely remove the field, so have to do it here
+    // remove any formula values, they are to be generated
     if (field.type === FieldTypes.FORMULA) {
       delete clonedRow[key]
-    } else {
+    }
+    // otherwise coerce what is there to correct types
+    else {
       clonedRow[key] = exports.coerce(value, field.type)
     }
   }
@@ -250,30 +253,23 @@ exports.inputProcessing = (
 /**
  * This function enriches the input rows with anything they are supposed to contain, for example
  * link records or attachment links.
- * @param {object} ctx the request which is looking for enriched rows.
  * @param {object} table the table from which these rows came from originally, this is used to determine
  * the schema of the rows and then enrich.
  * @param {object[]|object} rows the rows which are to be enriched.
  * @param {object} opts used to set some options for the output, such as disabling relationship squashing.
  * @returns {object[]|object} the enriched rows will be returned.
  */
-exports.outputProcessing = async (
-  ctx,
-  table,
-  rows,
-  opts = { squash: true }
-) => {
-  const appId = ctx.appId
+exports.outputProcessing = async (table, rows, opts = { squash: true }) => {
   let wasArray = true
   if (!(rows instanceof Array)) {
     rows = [rows]
     wasArray = false
   }
   // attach any linked row information
-  let enriched = await linkRows.attachFullLinkedDocs(ctx, table, rows)
+  let enriched = await linkRows.attachFullLinkedDocs(table, rows)
 
   // process formulas
-  enriched = processFormulas(table, enriched)
+  enriched = processFormulas(table, enriched, { dynamic: true })
 
   // update the attachments URL depending on hosting
   for (let [property, column] of Object.entries(table.schema)) {
@@ -289,29 +285,27 @@ exports.outputProcessing = async (
     }
   }
   if (opts.squash) {
-    enriched = await linkRows.squashLinksToPrimaryDisplay(
-      appId,
-      table,
-      enriched
-    )
+    enriched = await linkRows.squashLinksToPrimaryDisplay(table, enriched)
   }
   return wasArray ? enriched : enriched[0]
 }
 
 /**
  * Clean up any attachments that were attached to a row.
- * @param {string} appId The ID of the app from which a row is being deleted.
  * @param {object} table The table from which a row is being removed.
  * @param {any} row optional - the row being removed.
  * @param {any} rows optional - if multiple rows being deleted can do this in bulk.
  * @param {any} oldRow optional - if updating a row this will determine the difference.
+ * @param {any} oldTable optional - if updating a table, can supply the old table to look for
+ * deleted attachment columns.
  * @return {Promise<void>} When all attachments have been removed this will return.
  */
-exports.cleanupAttachments = async (appId, table, { row, rows, oldRow }) => {
+exports.cleanupAttachments = async (table, { row, rows, oldRow, oldTable }) => {
+  const appId = getAppId()
   if (!isProdAppID(appId)) {
-    const prodAppId = getDeployedAppID(appId)
+    const prodAppId = getProdAppID(appId)
     // if prod exists, then don't allow deleting
-    const exists = await dbExists(CouchDB, prodAppId)
+    const exists = await dbExists(prodAppId)
     if (exists) {
       return
     }
@@ -322,12 +316,16 @@ exports.cleanupAttachments = async (appId, table, { row, rows, oldRow }) => {
       files = files.concat(row[key].map(attachment => attachment.key))
     }
   }
-  for (let [key, schema] of Object.entries(table.schema)) {
+  const schemaToUse = oldTable ? oldTable.schema : table.schema
+  for (let [key, schema] of Object.entries(schemaToUse)) {
     if (schema.type !== FieldTypes.ATTACHMENT) {
       continue
     }
-    // if updating, need to manage the differences
-    if (oldRow && row) {
+    // old table had this column, new table doesn't - delete it
+    if (oldTable && !table.schema[key]) {
+      rows.forEach(row => addFiles(row, key))
+    } else if (oldRow && row) {
+      // if updating, need to manage the differences
       files = files.concat(getRemovedAttachmentKeys(oldRow, row, key))
     } else if (row) {
       addFiles(row, key)

package/src/utilities/rowProcessor/utils.js

@@ -1,23 +1,39 @@
-const { FieldTypes } = require("../../constants")
+const { FieldTypes, FormulaTypes } = require("../../constants")
 const { processStringSync } = require("@budibase/string-templates")
 
 /**
  * Looks through the rows provided and finds formulas - which it then processes.
  */
-exports.processFormulas = (table, rows) => {
+exports.processFormulas = (
+  table,
+  rows,
+  { dynamic, contextRows } = { dynamic: true }
+) => {
   const single = !Array.isArray(rows)
   if (single) {
     rows = [rows]
+    contextRows = contextRows ? [contextRows] : contextRows
   }
   for (let [column, schema] of Object.entries(table.schema)) {
-    if (schema.type !== FieldTypes.FORMULA) {
+    const isStatic = schema.formulaType === FormulaTypes.STATIC
+    if (
+      schema.type !== FieldTypes.FORMULA ||
+      (dynamic && isStatic) ||
+      (!dynamic && !isStatic)
+    ) {
       continue
     }
     // iterate through rows and process formula
-    rows = rows.map(row => ({
-      ...row,
-      [column]: processStringSync(schema.formula, row),
-    }))
+    for (let i = 0; i < rows.length; i++) {
+      if (schema.formula) {
+        let row = rows[i]
+        let context = contextRows ? contextRows[i] : row
+        rows[i] = {
+          ...row,
+          [column]: processStringSync(schema.formula, context),
+        }
+      }
+    }
   }
   return single ? rows[0] : rows
 }

package/src/utilities/tests/usageQuota/usageQuota.spec.js

@@ -0,0 +1,72 @@
+const getTenantId = jest.fn()
+jest.mock("@budibase/backend-core/tenancy", () => ({
+  getTenantId
+}))
+const usageQuota = require("../../usageQuota")
+const env = require("../../../environment")
+
+class TestConfiguration {
+  constructor() {
+    this.enableQuotas()
+  }
+
+  enableQuotas = () => {
+    env.USE_QUOTAS = 1
+  }
+
+  disableQuotas = () => {
+    env.USE_QUOTAS = null
+  }
+
+  setTenantId = (tenantId) => {
+    getTenantId.mockReturnValue(tenantId)
+  } 
+
+  setExcludedTenants = (tenants) => {
+    env.EXCLUDE_QUOTAS_TENANTS = tenants
+  }
+
+  reset = () => {
+    this.disableQuotas()
+    this.setExcludedTenants(null)
+  }
+}
+
+describe("usageQuota", () => {
+  let config
+
+  beforeEach(() => {
+    config = new TestConfiguration()
+  })
+
+  afterEach(() => {
+    config.reset()
+  })
+
+  describe("useQuotas", () => {
+    it("works when no settings have been provided", () => {
+      config.reset()
+      expect(usageQuota.useQuotas()).toBe(false)
+    })
+    it("honours USE_QUOTAS setting", () => {
+      config.disableQuotas()
+      expect(usageQuota.useQuotas()).toBe(false)
+
+      config.enableQuotas()
+      expect(usageQuota.useQuotas()).toBe(true)
+    })
+    it("honours EXCLUDE_QUOTAS_TENANTS setting", () => {
+      config.setTenantId("test")
+
+      // tenantId is in the list
+      config.setExcludedTenants("test, test2, test2")
+      expect(usageQuota.useQuotas()).toBe(false)
+      config.setExcludedTenants("test,test2,test2")
+      expect(usageQuota.useQuotas()).toBe(false)
+
+      // tenantId is not in the list
+      config.setTenantId("other")
+      expect(usageQuota.useQuotas()).toBe(true)
+    })
+  })
+})

package/src/utilities/usageQuota/index.js

@@ -0,0 +1,93 @@
+const env = require("../../environment")
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
+const {
+  StaticDatabases,
+  generateNewUsageQuotaDoc,
+} = require("@budibase/backend-core/db")
+
+exports.useQuotas = () => {
+  // check if quotas are enabled
+  if (env.USE_QUOTAS) {
+    // check if there are any tenants without limits
+    if (env.EXCLUDE_QUOTAS_TENANTS) {
+      const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.replace(
+        /\s/g,
+        ""
+      ).split(",")
+      const tenantId = getTenantId()
+      if (excludedTenants.includes(tenantId)) {
+        return false
+      }
+    }
+    return true
+  }
+  return false
+}
+
+exports.Properties = {
+  ROW: "rows",
+  UPLOAD: "storage", // doesn't work yet
+  VIEW: "views", // doesn't work yet
+  USER: "users", // doesn't work yet
+  AUTOMATION: "automationRuns", // doesn't work yet
+  APPS: "apps",
+  EMAILS: "emails", // doesn't work yet
+}
+
+exports.getUsageQuotaDoc = async db => {
+  let quota
+  try {
+    quota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota)
+  } catch (err) {
+    // doc doesn't exist. Create it
+    quota = generateNewUsageQuotaDoc()
+    const response = await db.put(quota)
+    quota._rev = response.rev
+  }
+
+  return quota
+}
+
+/**
+ * Given a specified tenantId this will add to the usage object for the specified property.
+ * @param {string} property The property which is to be added to (within the nested usageQuota object).
+ * @param {number} usage The amount (this can be negative) to adjust the number by.
+ * @param {object} opts optional - options such as dryRun, to check what update will do.
+ * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
+ * also been reset after this call.
+ */
+exports.update = async (property, usage, opts = { dryRun: false }) => {
+  if (!exports.useQuotas()) {
+    return
+  }
+
+  try {
+    const db = getGlobalDB()
+    const quota = await exports.getUsageQuotaDoc(db)
+
+    // increment the quota
+    quota.usageQuota[property] += usage
+
+    if (
+      quota.usageQuota[property] > quota.usageLimits[property] &&
+      usage > 0 // allow for decrementing usage when the quota is already exceeded
+    ) {
+      throw new Error(
+        `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
+      )
+    }
+
+    if (quota.usageQuota[property] < 0) {
+      // never go negative if the quota has previously been exceeded
+      quota.usageQuota[property] = 0
+    }
+
+    // update the usage quotas
+    if (!opts.dryRun) {
+      await db.put(quota)
+    }
+  } catch (err) {
+    console.error(`Error updating usage quotas for ${property}`, err)
+    throw err
+  }
+}

package/src/utilities/usageQuota/rows.js

@@ -0,0 +1,75 @@
+const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils")
+const CouchDB = require("../../db")
+const { isDevAppID, getDevelopmentAppID } = require("@budibase/backend-core/db")
+
+const ROW_EXCLUSIONS = [USER_METDATA_PREFIX]
+
+const getAppPairs = appIds => {
+  // collect the app ids into dev / prod pairs
+  // keyed by the dev app id
+  const pairs = {}
+  for (let appId of appIds) {
+    const devId = getDevelopmentAppID(appId)
+    if (!pairs[devId]) {
+      pairs[devId] = {}
+    }
+    if (isDevAppID(appId)) {
+      pairs[devId].devId = appId
+    } else {
+      pairs[devId].prodId = appId
+    }
+  }
+  return pairs
+}
+
+const getAppRows = async appId => {
+  // need to specify the app ID, as this is used for different apps in one call
+  const appDb = new CouchDB(appId)
+  const response = await appDb.allDocs(
+    getRowParams(null, null, {
+      include_docs: false,
+    })
+  )
+  return response.rows
+    .map(r => r.id)
+    .filter(id => {
+      for (let exclusion of ROW_EXCLUSIONS) {
+        if (id.startsWith(exclusion)) {
+          return false
+        }
+      }
+      return true
+    })
+}
+
+/**
+ * Return a set of all rows in the given app ids.
+ * The returned rows will be unique on a per dev/prod app basis.
+ * Rows duplicates may exist across apps due to data import so they are not filtered out.
+ */
+exports.getUniqueRows = async appIds => {
+  let uniqueRows = []
+  const pairs = getAppPairs(appIds)
+
+  for (let pair of Object.values(pairs)) {
+    let appRows = []
+    for (let appId of [pair.devId, pair.prodId]) {
+      if (!appId) {
+        continue
+      }
+      try {
+        appRows.push(await getAppRows(appId))
+      } catch (e) {
+        console.error(e)
+        // don't error out if we can't count the app rows, just continue
+      }
+    }
+
+    // ensure uniqueness on a per app pair basis
+    // this can't be done on all rows because app import results in
+    // duplicate row ids across apps
+    uniqueRows = uniqueRows.concat(...new Set(appRows))
+  }
+
+  return uniqueRows
+}

package/src/utilities/usageQuota/usageQuoteReset.js

@@ -0,0 +1,18 @@
+// UNUSED CODE
+// Preserved for future use
+
+/* eslint-disable no-unused-vars */
+
+function getNewQuotaReset() {
+  return Date.now() + 2592000000
+}
+
+function resetQuotasIfRequired(quota) {
+  // Check if the quota needs reset
+  if (Date.now() >= quota.quotaReset) {
+    quota.quotaReset = getNewQuotaReset()
+    for (let prop of Object.keys(quota.usageQuota)) {
+      quota.usageQuota[prop] = 0
+    }
+  }
+}

package/src/utilities/users.js

@@ -1,13 +1,13 @@
-const CouchDB = require("../db")
 const { InternalTables } = require("../db/utils")
 const { getGlobalUser } = require("../utilities/global")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 exports.getFullUser = async (ctx, userId) => {
-  const global = await getGlobalUser(ctx.appId, userId)
+  const global = await getGlobalUser(userId)
   let metadata
   try {
     // this will throw an error if the db doesn't exist, or there is no appId
-    const db = new CouchDB(ctx.appId)
+    const db = getAppDB()
     metadata = await db.get(userId)
   } catch (err) {
     // it is fine if there is no user metadata, just remove global db info

package/src/utilities/workerRequests.js

@@ -1,7 +1,7 @@
 const fetch = require("node-fetch")
 const env = require("../environment")
 const { checkSlashesInUrl } = require("./index")
-const { getDeployedAppID } = require("@budibase/backend-core/db")
+const { getProdAppID } = require("@budibase/backend-core/db")
 const { updateAppRole } = require("./global")
 const { Headers } = require("@budibase/backend-core/constants")
 const { getTenantId, isTenantIdSet } = require("@budibase/backend-core/tenancy")
@@ -58,29 +58,6 @@ exports.sendSmtpEmail = async (to, from, subject, contents, automation) => {
   return response.json()
 }
 
-exports.getDeployedApps = async () => {
-  try {
-    const response = await fetch(
-      checkSlashesInUrl(env.WORKER_URL + `/api/apps`),
-      request(null, {
-        method: "GET",
-      })
-    )
-    const json = await response.json()
-    const apps = {}
-    for (let [key, value] of Object.entries(json)) {
-      if (value.url) {
-        value.url = value.url.toLowerCase()
-        apps[key.toLowerCase()] = value
-      }
-    }
-    return apps
-  } catch (err) {
-    // error, cannot determine deployed apps, don't stop app creation - sort this later
-    return {}
-  }
-}
-
 exports.getGlobalSelf = async (ctx, appId = null) => {
   const endpoint = `/api/global/users/self`
   const response = await fetch(
@@ -93,15 +70,15 @@ exports.getGlobalSelf = async (ctx, appId = null) => {
   }
   let json = await response.json()
   if (appId) {
-    json = updateAppRole(appId, json)
+    json = updateAppRole(json)
   }
   return json
 }
 
 exports.removeAppFromUserRoles = async (ctx, appId) => {
-  const deployedAppId = getDeployedAppID(appId)
+  const prodAppId = getProdAppID(appId)
   const response = await fetch(
-    checkSlashesInUrl(env.WORKER_URL + `/api/global/roles/${deployedAppId}`),
+    checkSlashesInUrl(env.WORKER_URL + `/api/global/roles/${prodAppId}`),
     request(ctx, {
       method: "DELETE",
     })

package/tsconfig.json

@@ -12,7 +12,8 @@
     "incremental": true
   },
   "include": [
-    "./src/**/*"
+    "./src/**/*",
+    "./src/module.d.ts"
   ],
   "exclude": [
     "node_modules",