@budibase/server 1.0.46-alpha.2 → 1.0.46-alpha.3

package/src/migrations/usageQuotas/syncRows.js

@@ -0,0 +1,21 @@
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
+const { getAllApps } = require("@budibase/backend-core/db")
+const CouchDB = require("../../db")
+const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
+const { getUniqueRows } = require("../../utilities/usageQuota/rows")
+
+exports.run = async () => {
+  const db = getGlobalDB()
+  // get all rows in all apps
+  const allApps = await getAllApps(CouchDB, { all: true })
+  const appIds = allApps ? allApps.map(app => app.appId) : []
+  const rows = await getUniqueRows(appIds)
+  const rowCount = rows ? rows.length : 0
+
+  // sync row count
+  const tenantId = getTenantId()
+  console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.rows = rowCount
+  await db.put(usageDoc)
+}

package/src/utilities/tests/usageQuota/usageQuota.spec.js

@@ -0,0 +1,72 @@
+const getTenantId = jest.fn()
+jest.mock("@budibase/backend-core/tenancy", () => ({
+  getTenantId
+}))
+const usageQuota = require("../../usageQuota")
+const env = require("../../../environment")
+
+class TestConfiguration {
+  constructor() {
+    this.enableQuotas()
+  }
+
+  enableQuotas = () => {
+    env.USE_QUOTAS = 1
+  }
+
+  disableQuotas = () => {
+    env.USE_QUOTAS = null
+  }
+
+  setTenantId = (tenantId) => {
+    getTenantId.mockReturnValue(tenantId)
+  } 
+
+  setExcludedTenants = (tenants) => {
+    env.EXCLUDE_QUOTAS_TENANTS = tenants
+  }
+
+  reset = () => {
+    this.disableQuotas()
+    this.setExcludedTenants(null)
+  }
+}
+
+describe("usageQuota", () => {
+  let config
+
+  beforeEach(() => {
+    config = new TestConfiguration()
+  })
+
+  afterEach(() => {
+    config.reset()
+  })
+
+  describe("useQuotas", () => {
+    it("works when no settings have been provided", () => {
+      config.reset()
+      expect(usageQuota.useQuotas()).toBe(false)
+    })
+    it("honours USE_QUOTAS setting", () => {
+      config.disableQuotas()
+      expect(usageQuota.useQuotas()).toBe(false)
+
+      config.enableQuotas()
+      expect(usageQuota.useQuotas()).toBe(true)
+    })
+    it("honours EXCLUDE_QUOTAS_TENANTS setting", () => {
+      config.setTenantId("test")
+
+      // tenantId is in the list
+      config.setExcludedTenants("test, test2, test2")
+      expect(usageQuota.useQuotas()).toBe(false)
+      config.setExcludedTenants("test,test2,test2")
+      expect(usageQuota.useQuotas()).toBe(false)
+
+      // tenantId is not in the list
+      config.setTenantId("other")
+      expect(usageQuota.useQuotas()).toBe(true)
+    })
+  })
+})

package/src/utilities/usageQuota/index.js

@@ -0,0 +1,92 @@
+const env = require("../../environment")
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
+const {
+  StaticDatabases,
+  generateNewUsageQuotaDoc,
+} = require("@budibase/backend-core/db")
+
+exports.useQuotas = () => {
+  // check if quotas are enabled
+  if (env.USE_QUOTAS) {
+    // check if there are any tenants without limits
+    if (env.EXCLUDE_QUOTAS_TENANTS) {
+      const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.replace(
+        /\s/g,
+        ""
+      ).split(",")
+      const tenantId = getTenantId()
+      if (excludedTenants.includes(tenantId)) {
+        return false
+      }
+    }
+    return true
+  }
+  return false
+}
+
+exports.Properties = {
+  ROW: "rows",
+  UPLOAD: "storage", // doesn't work yet
+  VIEW: "views", // doesn't work yet
+  USER: "users", // doesn't work yet
+  AUTOMATION: "automationRuns", // doesn't work yet
+  APPS: "apps",
+  EMAILS: "emails", // doesn't work yet
+}
+
+exports.getUsageQuotaDoc = async db => {
+  let quota
+  try {
+    quota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota)
+  } catch (err) {
+    // doc doesn't exist. Create it
+    quota = generateNewUsageQuotaDoc()
+    const response = await db.put(quota)
+    quota._rev = response.rev
+  }
+
+  return quota
+}
+
+/**
+ * Given a specified tenantId this will add to the usage object for the specified property.
+ * @param {string} property The property which is to be added to (within the nested usageQuota object).
+ * @param {number} usage The amount (this can be negative) to adjust the number by.
+ * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
+ * also been reset after this call.
+ */
+exports.update = async (property, usage, opts = { dryRun: false }) => {
+  if (!exports.useQuotas()) {
+    return
+  }
+
+  try {
+    const db = getGlobalDB()
+    const quota = await exports.getUsageQuotaDoc(db)
+
+    // increment the quota
+    quota.usageQuota[property] += usage
+
+    if (
+      quota.usageQuota[property] > quota.usageLimits[property] &&
+      usage > 0 // allow for decrementing usage when the quota is already exceeded
+    ) {
+      throw new Error(
+        `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
+      )
+    }
+
+    if (quota.usageQuota[property] < 0) {
+      // never go negative if the quota has previously been exceeded
+      quota.usageQuota[property] = 0
+    }
+
+    // update the usage quotas
+    if (!opts.dryRun) {
+      await db.put(quota)
+    }
+  } catch (err) {
+    console.error(`Error updating usage quotas for ${property}`, err)
+    throw err
+  }
+}

package/src/utilities/usageQuota/rows.js

@@ -0,0 +1,74 @@
+const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils")
+const CouchDB = require("../../db")
+const { isDevAppID, getDevelopmentAppID } = require("@budibase/backend-core/db")
+
+const ROW_EXCLUSIONS = [USER_METDATA_PREFIX]
+
+const getAppPairs = appIds => {
+  // collect the app ids into dev / prod pairs
+  // keyed by the dev app id
+  const pairs = {}
+  for (let appId of appIds) {
+    const devId = getDevelopmentAppID(appId)
+    if (!pairs[devId]) {
+      pairs[devId] = {}
+    }
+    if (isDevAppID(appId)) {
+      pairs[devId].devId = appId
+    } else {
+      pairs[devId].prodId = appId
+    }
+  }
+  return pairs
+}
+
+const getAppRows = async appId => {
+  const appDb = new CouchDB(appId)
+  const response = await appDb.allDocs(
+    getRowParams(null, null, {
+      include_docs: false,
+    })
+  )
+  return response.rows
+    .map(r => r.id)
+    .filter(id => {
+      for (let exclusion of ROW_EXCLUSIONS) {
+        if (id.startsWith(exclusion)) {
+          return false
+        }
+      }
+      return true
+    })
+}
+
+/**
+ * Return a set of all rows in the given app ids.
+ * The returned rows will be unique on a per dev/prod app basis.
+ * Rows duplicates may exist across apps due to data import so they are not filtered out.
+ */
+exports.getUniqueRows = async appIds => {
+  let uniqueRows = []
+  const pairs = getAppPairs(appIds)
+
+  for (let pair of Object.values(pairs)) {
+    let appRows = []
+    for (let appId of [pair.devId, pair.prodId]) {
+      if (!appId) {
+        continue
+      }
+      try {
+        appRows.push(await getAppRows(appId))
+      } catch (e) {
+        console.error(e)
+        // don't error out if we can't count the app rows, just continue
+      }
+    }
+
+    // ensure uniqueness on a per app pair basis
+    // this can't be done on all rows because app import results in
+    // duplicate row ids across apps
+    uniqueRows = uniqueRows.concat(...new Set(appRows))
+  }
+
+  return uniqueRows
+}

package/src/utilities/usageQuota/usageQuoteReset.js

@@ -0,0 +1,18 @@
+// UNUSED CODE
+// Preserved for future use
+
+/* eslint-disable no-unused-vars */
+
+function getNewQuotaReset() {
+  return Date.now() + 2592000000
+}
+
+function resetQuotasIfRequired(quota) {
+  // Check if the quota needs reset
+  if (Date.now() >= quota.quotaReset) {
+    quota.quotaReset = getNewQuotaReset()
+    for (let prop of Object.keys(quota.usageQuota)) {
+      quota.usageQuota[prop] = 0
+    }
+  }
+}

package/src/utilities/usageQuota.js

@@ -1,73 +0,0 @@
-const env = require("../environment")
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const {
-  StaticDatabases,
-  generateNewUsageQuotaDoc,
-} = require("@budibase/backend-core/db")
-
-function getNewQuotaReset() {
-  return Date.now() + 2592000000
-}
-
-exports.Properties = {
-  ROW: "rows",
-  UPLOAD: "storage",
-  VIEW: "views",
-  USER: "users",
-  AUTOMATION: "automationRuns",
-  APPS: "apps",
-  EMAILS: "emails",
-}
-
-async function getUsageQuotaDoc(db) {
-  let quota
-  try {
-    quota = await db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota)
-  } catch (err) {
-    // doc doesn't exist. Create it
-    quota = await db.post(generateNewUsageQuotaDoc())
-  }
-
-  return quota
-}
-
-/**
- * Given a specified tenantId this will add to the usage object for the specified property.
- * @param {string} property The property which is to be added to (within the nested usageQuota object).
- * @param {number} usage The amount (this can be negative) to adjust the number by.
- * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
- * also been reset after this call.
- */
-exports.update = async (property, usage) => {
-  if (!env.USE_QUOTAS) {
-    return
-  }
-
-  try {
-    const db = getGlobalDB()
-    const quota = await getUsageQuotaDoc(db)
-
-    // Check if the quota needs reset
-    if (Date.now() >= quota.quotaReset) {
-      quota.quotaReset = getNewQuotaReset()
-      for (let prop of Object.keys(quota.usageQuota)) {
-        quota.usageQuota[prop] = 0
-      }
-    }
-
-    // increment the quota
-    quota.usageQuota[property] += usage
-
-    if (quota.usageQuota[property] > quota.usageLimits[property]) {
-      throw new Error(
-        `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
-      )
-    }
-
-    // update the usage quotas
-    await db.put(quota)
-  } catch (err) {
-    console.error(`Error updating usage quotas for ${property}`, err)
-    throw err
-  }
-}