@budibase/server 1.0.46-alpha.2 → 1.0.46-alpha.6

package/src/middleware/usageQuota.js

@@ -1,14 +1,11 @@
 const CouchDB = require("../db")
 const usageQuota = require("../utilities/usageQuota")
-const env = require("../environment")
-const { getTenantId } = require("@budibase/backend-core/tenancy")
+const { getUniqueRows } = require("../utilities/usageQuota/rows")
 const {
   isExternalTable,
   isRowId: isExternalRowId,
 } = require("../integrations/utils")
-
-// tenants without limits
-const EXCLUDED_TENANTS = ["bb", "default", "bbtest", "bbstaging"]
+const migration = require("../migrations/usageQuotas")
 
 // currently only counting new writes and deletes
 const METHOD_MAP = {
@@ -18,13 +15,13 @@ const METHOD_MAP = {
 
 const DOMAIN_MAP = {
   rows: usageQuota.Properties.ROW,
-  upload: usageQuota.Properties.UPLOAD,
-  views: usageQuota.Properties.VIEW,
-  users: usageQuota.Properties.USER,
+  // upload: usageQuota.Properties.UPLOAD,            // doesn't work yet
+  // views: usageQuota.Properties.VIEW,               // doesn't work yet
+  // users: usageQuota.Properties.USER,               // doesn't work yet
   applications: usageQuota.Properties.APPS,
   // this will not be updated by endpoint calls
   // instead it will be updated by triggerInfo
-  automationRuns: usageQuota.Properties.AUTOMATION,
+  // automationRuns: usageQuota.Properties.AUTOMATION, // doesn't work yet
 }
 
 function getProperty(url) {
@@ -36,10 +33,7 @@ function getProperty(url) {
 }
 
 module.exports = async (ctx, next) => {
-  const tenantId = getTenantId()
-
-  // if in development or a self hosted cloud usage quotas should not be executed
-  if (env.isDev() || env.SELF_HOSTED || EXCLUDED_TENANTS.includes(tenantId)) {
+  if (!usageQuota.useQuotas()) {
     return next()
   }
 
@@ -80,9 +74,93 @@ module.exports = async (ctx, next) => {
     usage = files.map(file => file.size).reduce((total, size) => total + size)
   }
   try {
-    await usageQuota.update(property, usage)
-    return next()
+    await migration.run()
+    await performRequest(ctx, next, property, usage)
   } catch (err) {
     ctx.throw(400, err)
   }
 }
+
+const performRequest = async (ctx, next, property, usage) => {
+  const usageContext = {
+    skipNext: false,
+    skipUsage: false,
+    [usageQuota.Properties.APPS]: {},
+  }
+
+  if (usage === -1) {
+    if (PRE_DELETE[property]) {
+      await PRE_DELETE[property](ctx, usageContext)
+    }
+  } else {
+    if (PRE_CREATE[property]) {
+      await PRE_CREATE[property](ctx, usageContext)
+    }
+  }
+
+  // run the request
+  if (!usageContext.skipNext) {
+    await usageQuota.update(property, usage, { dryRun: true })
+    await next()
+  }
+
+  if (usage === -1) {
+    if (POST_DELETE[property]) {
+      await POST_DELETE[property](ctx, usageContext)
+    }
+  } else {
+    if (POST_CREATE[property]) {
+      await POST_CREATE[property](ctx, usageContext)
+    }
+  }
+
+  // update the usage
+  if (!usageContext.skipUsage) {
+    await usageQuota.update(property, usage)
+  }
+}
+
+const appPreDelete = async (ctx, usageContext) => {
+  if (ctx.query.unpublish) {
+    // don't run usage decrement for unpublish
+    usageContext.skipUsage = true
+    return
+  }
+
+  // store the row count to delete
+  const rows = await getUniqueRows([ctx.appId])
+  if (rows.length) {
+    usageContext[usageQuota.Properties.APPS] = { rowCount: rows.length }
+  }
+}
+
+const appPostDelete = async (ctx, usageContext) => {
+  // delete the app rows from usage
+  const rowCount = usageContext[usageQuota.Properties.APPS].rowCount
+  if (rowCount) {
+    await usageQuota.update(usageQuota.Properties.ROW, -rowCount)
+  }
+}
+
+const appPostCreate = async ctx => {
+  // app import & template creation
+  if (ctx.request.body.useTemplate === "true") {
+    const rows = await getUniqueRows([ctx.response.body.appId])
+    const rowCount = rows ? rows.length : 0
+    await usageQuota.update(usageQuota.Properties.ROW, rowCount)
+  }
+}
+
+const PRE_DELETE = {
+  [usageQuota.Properties.APPS]: appPreDelete,
+}
+
+const POST_DELETE = {
+  [usageQuota.Properties.APPS]: appPostDelete,
+}
+
+const PRE_CREATE = {}
+
+const POST_CREATE = {
+  [usageQuota.Properties.APPS]: appPostCreate,
+}

package/src/migrations/tests/usageQuotas/index.spec.js

@@ -0,0 +1,27 @@
+const env = require("../../../environment")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+
+const syncApps = jest.fn()
+const syncRows = jest.fn()
+
+jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) )
+jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) )
+
+const migrations = require("../../usageQuotas")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)  
+
+  it("runs the required migrations", async () => {
+    await migrations.run()
+    expect(syncApps).toHaveBeenCalledTimes(1)
+    expect(syncRows).toHaveBeenCalledTimes(1)
+  })
+})

package/src/migrations/tests/usageQuotas/syncApps.spec.js

@@ -0,0 +1,37 @@
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
+const syncApps = require("../../usageQuotas/syncApps")
+const env = require("../../../environment")
+
+describe("syncApps", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    // create the usage quota doc and mock usages
+    const db = getGlobalDB()
+    await getUsageQuotaDoc(db)
+    await update(Properties.APPS, 3)
+
+    let usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.apps).toEqual(3)
+
+    // create an extra app to test the migration
+    await config.createApp("quota-test")
+
+    // migrate
+    await syncApps.run()
+
+    // assert the migration worked 
+    usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.apps).toEqual(2)
+  })
+})
+

package/src/migrations/tests/usageQuotas/syncRows.spec.js

@@ -0,0 +1,43 @@
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
+const syncRows = require("../../usageQuotas/syncRows")
+const env = require("../../../environment")
+
+describe("syncRows", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    // create the usage quota doc and mock usages
+    const db = getGlobalDB()
+    await getUsageQuotaDoc(db)
+    await update(Properties.ROW, 300)
+
+    let usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.rows).toEqual(300)
+
+    // app 1
+    await config.createTable()
+    await config.createRow()
+    // app 2
+    await config.createApp("second-app")
+    await config.createTable()
+    await config.createRow()
+    await config.createRow()
+    
+    // migrate
+    await syncRows.run()
+
+    // assert the migration worked 
+    usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.rows).toEqual(3)
+  })
+})
+

package/src/migrations/usageQuotas/index.js

@@ -0,0 +1,24 @@
+const {
+  MIGRATIONS,
+  MIGRATION_DBS,
+  migrateIfRequired,
+} = require("@budibase/backend-core/migrations")
+const { useQuotas } = require("../../utilities/usageQuota")
+const syncApps = require("./syncApps")
+const syncRows = require("./syncRows")
+
+exports.run = async () => {
+  if (!useQuotas()) {
+    return
+  }
+
+  // Jan 2022
+  await migrateIfRequired(
+    MIGRATION_DBS.GLOBAL_DB,
+    MIGRATIONS.QUOTAS_1,
+    async () => {
+      await syncApps.run()
+      await syncRows.run()
+    }
+  )
+}

package/src/migrations/usageQuotas/syncApps.js

@@ -0,0 +1,18 @@
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
+const { getAllApps } = require("@budibase/backend-core/db")
+const CouchDB = require("../../db")
+const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
+
+exports.run = async () => {
+  const db = getGlobalDB()
+  // get app count
+  const devApps = await getAllApps(CouchDB, { dev: true })
+  const appCount = devApps ? devApps.length : 0
+
+  // sync app count
+  const tenantId = getTenantId()
+  console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.apps = appCount
+  await db.put(usageDoc)
+}

package/src/migrations/usageQuotas/syncRows.js

@@ -0,0 +1,21 @@
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
+const { getAllApps } = require("@budibase/backend-core/db")
+const CouchDB = require("../../db")
+const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
+const { getUniqueRows } = require("../../utilities/usageQuota/rows")
+
+exports.run = async () => {
+  const db = getGlobalDB()
+  // get all rows in all apps
+  const allApps = await getAllApps(CouchDB, { all: true })
+  const appIds = allApps ? allApps.map(app => app.appId) : []
+  const rows = await getUniqueRows(appIds)
+  const rowCount = rows ? rows.length : 0
+
+  // sync row count
+  const tenantId = getTenantId()
+  console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.rows = rowCount
+  await db.put(usageDoc)
+}

package/src/tests/utilities/TestConfiguration.js

@@ -22,6 +22,7 @@ const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 const { createASession } = require("@budibase/backend-core/sessions")
 const { user: userCache } = require("@budibase/backend-core/cache")
 const CouchDB = require("../../db")
+const newid = require("../../db/newid")
 core.init(CouchDB)
 
 const GLOBAL_USER_ID = "us_uuid1"
@@ -98,7 +99,8 @@ class TestConfiguration {
     }
   }
 
-  async init(appName = "test_application") {
+  // use a new id as the name to avoid name collisions
+  async init(appName = newid()) {
     await this.globalUser()
     return this.createApp(appName)
   }

package/src/threads/query.js

@@ -78,6 +78,11 @@ class QueryRunner {
       return this.execute()
     }
 
+    // check for undefined response
+    if (!rows) {
+      rows = []
+    }
+
     // needs to an array for next step
     if (!Array.isArray(rows)) {
       rows = [rows]

package/src/utilities/tests/usageQuota/usageQuota.spec.js

@@ -0,0 +1,72 @@
+const getTenantId = jest.fn()
+jest.mock("@budibase/backend-core/tenancy", () => ({
+  getTenantId
+}))
+const usageQuota = require("../../usageQuota")
+const env = require("../../../environment")
+
+class TestConfiguration {
+  constructor() {
+    this.enableQuotas()
+  }
+
+  enableQuotas = () => {
+    env.USE_QUOTAS = 1
+  }
+
+  disableQuotas = () => {
+    env.USE_QUOTAS = null
+  }
+
+  setTenantId = (tenantId) => {
+    getTenantId.mockReturnValue(tenantId)
+  } 
+
+  setExcludedTenants = (tenants) => {
+    env.EXCLUDE_QUOTAS_TENANTS = tenants
+  }
+
+  reset = () => {
+    this.disableQuotas()
+    this.setExcludedTenants(null)
+  }
+}
+
+describe("usageQuota", () => {
+  let config
+
+  beforeEach(() => {
+    config = new TestConfiguration()
+  })
+
+  afterEach(() => {
+    config.reset()
+  })
+
+  describe("useQuotas", () => {
+    it("works when no settings have been provided", () => {
+      config.reset()
+      expect(usageQuota.useQuotas()).toBe(false)
+    })
+    it("honours USE_QUOTAS setting", () => {
+      config.disableQuotas()
+      expect(usageQuota.useQuotas()).toBe(false)
+
+      config.enableQuotas()
+      expect(usageQuota.useQuotas()).toBe(true)
+    })
+    it("honours EXCLUDE_QUOTAS_TENANTS setting", () => {
+      config.setTenantId("test")
+
+      // tenantId is in the list
+      config.setExcludedTenants("test, test2, test2")
+      expect(usageQuota.useQuotas()).toBe(false)
+      config.setExcludedTenants("test,test2,test2")
+      expect(usageQuota.useQuotas()).toBe(false)
+
+      // tenantId is not in the list
+      config.setTenantId("other")
+      expect(usageQuota.useQuotas()).toBe(true)
+    })
+  })
+})

package/src/utilities/usageQuota/index.js

@@ -0,0 +1,92 @@
+const env = require("../../environment")
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
+const {
+  StaticDatabases,
+  generateNewUsageQuotaDoc,
+} = require("@budibase/backend-core/db")
+
+exports.useQuotas = () => {
+  // check if quotas are enabled
+  if (env.USE_QUOTAS) {
+    // check if there are any tenants without limits
+    if (env.EXCLUDE_QUOTAS_TENANTS) {
+      const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.replace(
+        /\s/g,
+        ""
+      ).split(",")
+      const tenantId = getTenantId()
+      if (excludedTenants.includes(tenantId)) {
+        return false
+      }
+    }
+    return true
+  }
+  return false
+}
+
+exports.Properties = {
+  ROW: "rows",
+  UPLOAD: "storage", // doesn't work yet
+  VIEW: "views", // doesn't work yet
+  USER: "users", // doesn't work yet
+  AUTOMATION: "automationRuns", // doesn't work yet
+  APPS: "apps",
+  EMAILS: "emails", // doesn't work yet
+}
+
+exports.getUsageQuotaDoc = async db => {
+  let quota
+  try {
+    quota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota)
+  } catch (err) {
+    // doc doesn't exist. Create it
+    quota = generateNewUsageQuotaDoc()
+    const response = await db.put(quota)
+    quota._rev = response.rev
+  }
+
+  return quota
+}
+
+/**
+ * Given a specified tenantId this will add to the usage object for the specified property.
+ * @param {string} property The property which is to be added to (within the nested usageQuota object).
+ * @param {number} usage The amount (this can be negative) to adjust the number by.
+ * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
+ * also been reset after this call.
+ */
+exports.update = async (property, usage, opts = { dryRun: false }) => {
+  if (!exports.useQuotas()) {
+    return
+  }
+
+  try {
+    const db = getGlobalDB()
+    const quota = await exports.getUsageQuotaDoc(db)
+
+    // increment the quota
+    quota.usageQuota[property] += usage
+
+    if (
+      quota.usageQuota[property] > quota.usageLimits[property] &&
+      usage > 0 // allow for decrementing usage when the quota is already exceeded
+    ) {
+      throw new Error(
+        `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
+      )
+    }
+
+    if (quota.usageQuota[property] < 0) {
+      // never go negative if the quota has previously been exceeded
+      quota.usageQuota[property] = 0
+    }
+
+    // update the usage quotas
+    if (!opts.dryRun) {
+      await db.put(quota)
+    }
+  } catch (err) {
+    console.error(`Error updating usage quotas for ${property}`, err)
+    throw err
+  }
+}

package/src/utilities/usageQuota/rows.js

@@ -0,0 +1,74 @@
+const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils")
+const CouchDB = require("../../db")
+const { isDevAppID, getDevelopmentAppID } = require("@budibase/backend-core/db")
+
+const ROW_EXCLUSIONS = [USER_METDATA_PREFIX]
+
+const getAppPairs = appIds => {
+  // collect the app ids into dev / prod pairs
+  // keyed by the dev app id
+  const pairs = {}
+  for (let appId of appIds) {
+    const devId = getDevelopmentAppID(appId)
+    if (!pairs[devId]) {
+      pairs[devId] = {}
+    }
+    if (isDevAppID(appId)) {
+      pairs[devId].devId = appId
+    } else {
+      pairs[devId].prodId = appId
+    }
+  }
+  return pairs
+}
+
+const getAppRows = async appId => {
+  const appDb = new CouchDB(appId)
+  const response = await appDb.allDocs(
+    getRowParams(null, null, {
+      include_docs: false,
+    })
+  )
+  return response.rows
+    .map(r => r.id)
+    .filter(id => {
+      for (let exclusion of ROW_EXCLUSIONS) {
+        if (id.startsWith(exclusion)) {
+          return false
+        }
+      }
+      return true
+    })
+}
+
+/**
+ * Return a set of all rows in the given app ids.
+ * The returned rows will be unique on a per dev/prod app basis.
+ * Rows duplicates may exist across apps due to data import so they are not filtered out.
+ */
+exports.getUniqueRows = async appIds => {
+  let uniqueRows = []
+  const pairs = getAppPairs(appIds)
+
+  for (let pair of Object.values(pairs)) {
+    let appRows = []
+    for (let appId of [pair.devId, pair.prodId]) {
+      if (!appId) {
+        continue
+      }
+      try {
+        appRows.push(await getAppRows(appId))
+      } catch (e) {
+        console.error(e)
+        // don't error out if we can't count the app rows, just continue
+      }
+    }
+
+    // ensure uniqueness on a per app pair basis
+    // this can't be done on all rows because app import results in
+    // duplicate row ids across apps
+    uniqueRows = uniqueRows.concat(...new Set(appRows))
+  }
+
+  return uniqueRows
+}

package/src/utilities/usageQuota/usageQuoteReset.js

@@ -0,0 +1,18 @@
+// UNUSED CODE
+// Preserved for future use
+
+/* eslint-disable no-unused-vars */
+
+function getNewQuotaReset() {
+  return Date.now() + 2592000000
+}
+
+function resetQuotasIfRequired(quota) {
+  // Check if the quota needs reset
+  if (Date.now() >= quota.quotaReset) {
+    quota.quotaReset = getNewQuotaReset()
+    for (let prop of Object.keys(quota.usageQuota)) {
+      quota.usageQuota[prop] = 0
+    }
+  }
+}

package/src/utilities/workerRequests.js

@@ -58,29 +58,6 @@ exports.sendSmtpEmail = async (to, from, subject, contents, automation) => {
   return response.json()
 }
 
-exports.getDeployedApps = async () => {
-  try {
-    const response = await fetch(
-      checkSlashesInUrl(env.WORKER_URL + `/api/apps`),
-      request(null, {
-        method: "GET",
-      })
-    )
-    const json = await response.json()
-    const apps = {}
-    for (let [key, value] of Object.entries(json)) {
-      if (value.url) {
-        value.url = value.url.toLowerCase()
-        apps[key.toLowerCase()] = value
-      }
-    }
-    return apps
-  } catch (err) {
-    // error, cannot determine deployed apps, don't stop app creation - sort this later
-    return {}
-  }
-}
-
 exports.getGlobalSelf = async (ctx, appId = null) => {
   const endpoint = `/api/global/users/self`
   const response = await fetch(

package/dist/api/controllers/hosting.js

@@ -1,30 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-const CouchDB = require("../../db");
-const { getDeployedApps } = require("../../utilities/workerRequests");
-const { getScopedConfig } = require("@budibase/backend-core/db");
-const { Configs } = require("@budibase/backend-core/constants");
-const { checkSlashesInUrl } = require("../../utilities");
-exports.fetchUrls = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
-    const appId = ctx.appId;
-    const db = new CouchDB(appId);
-    const settings = yield getScopedConfig(db, { type: Configs.SETTINGS });
-    let appUrl = "http://localhost:10000/app";
-    if (settings && settings["platformUrl"]) {
-        appUrl = checkSlashesInUrl(`${settings["platformUrl"]}/app`);
-    }
-    ctx.body = {
-        app: appUrl,
-    };
-});
-exports.getDeployedApps = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
-    ctx.body = yield getDeployedApps();
-});

package/dist/api/routes/hosting.js

@@ -1,11 +0,0 @@
-"use strict";
-const Router = require("@koa/router");
-const controller = require("../controllers/hosting");
-const authorized = require("../../middleware/authorized");
-const { BUILDER } = require("@budibase/backend-core/permissions");
-const router = Router();
-router
-    .get("/api/hosting/urls", authorized(BUILDER), controller.fetchUrls)
-    // this isn't risky, doesn't return anything about apps other than names and URLs
-    .get("/api/hosting/apps", controller.getDeployedApps);
-module.exports = router;