@budibase/server 1.0.49 → 1.0.50-alpha.3

package/src/middleware/currentapp.js

@@ -11,9 +11,9 @@ const { generateUserMetadataID, isDevAppID } = require("../db/utils")
 const { dbExists } = require("@budibase/backend-core/db")
 const { isUserInAppTenant } = require("@budibase/backend-core/tenancy")
 const { getCachedSelf } = require("../utilities/global")
-const CouchDB = require("../db")
 const env = require("../environment")
 const { isWebhookEndpoint } = require("./utils")
+const { doInAppContext } = require("@budibase/backend-core/context")
 
 module.exports = async (ctx, next) => {
   // try to get the appID from the request
@@ -31,7 +31,7 @@ module.exports = async (ctx, next) => {
   // check the app exists referenced in cookie
   if (appCookie) {
     const appId = appCookie.appId
-    const exists = await dbExists(CouchDB, appId)
+    const exists = await dbExists(appId)
     if (!exists) {
       clearCookie(ctx, Cookies.CurrentApp)
       return next()
@@ -41,13 +41,15 @@ module.exports = async (ctx, next) => {
   }
 
   // deny access to application preview
-  if (
-    isDevAppID(requestAppId) &&
-    !isWebhookEndpoint(ctx) &&
-    (!ctx.user || !ctx.user.builder || !ctx.user.builder.global)
-  ) {
-    clearCookie(ctx, Cookies.CurrentApp)
-    return ctx.redirect("/")
+  if (!env.isTest()) {
+    if (
+      isDevAppID(requestAppId) &&
+      !isWebhookEndpoint(ctx) &&
+      (!ctx.user || !ctx.user.builder || !ctx.user.builder.global)
+    ) {
+      clearCookie(ctx, Cookies.CurrentApp)
+      return ctx.redirect("/")
+    }
   }
 
   let appId,
@@ -68,44 +70,46 @@ module.exports = async (ctx, next) => {
     return next()
   }
 
-  let noCookieSet = false
-  // if the user not in the right tenant then make sure they have no permissions
-  // need to judge this only based on the request app ID,
-  if (
-    env.MULTI_TENANCY &&
-    ctx.user &&
-    requestAppId &&
-    !isUserInAppTenant(requestAppId)
-  ) {
-    // don't error, simply remove the users rights (they are a public user)
-    delete ctx.user.builder
-    delete ctx.user.admin
-    delete ctx.user.roles
-    roleId = BUILTIN_ROLE_IDS.PUBLIC
-    noCookieSet = true
-  }
+  return doInAppContext(appId, async () => {
+    let noCookieSet = false
+    // if the user not in the right tenant then make sure they have no permissions
+    // need to judge this only based on the request app ID,
+    if (
+      env.MULTI_TENANCY &&
+      ctx.user &&
+      requestAppId &&
+      !isUserInAppTenant(requestAppId)
+    ) {
+      // don't error, simply remove the users rights (they are a public user)
+      delete ctx.user.builder
+      delete ctx.user.admin
+      delete ctx.user.roles
+      roleId = BUILTIN_ROLE_IDS.PUBLIC
+      noCookieSet = true
+    }
 
-  ctx.appId = appId
-  if (roleId) {
-    ctx.roleId = roleId
-    const userId = ctx.user ? generateUserMetadataID(ctx.user._id) : null
-    ctx.user = {
-      ...ctx.user,
-      // override userID with metadata one
-      _id: userId,
-      userId,
-      roleId,
-      role: await getRole(appId, roleId),
+    ctx.appId = appId
+    if (roleId) {
+      ctx.roleId = roleId
+      const userId = ctx.user ? generateUserMetadataID(ctx.user._id) : null
+      ctx.user = {
+        ...ctx.user,
+        // override userID with metadata one
+        _id: userId,
+        userId,
+        roleId,
+        role: await getRole(roleId),
+      }
+    }
+    if (
+      (requestAppId !== appId ||
+        appCookie == null ||
+        appCookie.appId !== requestAppId) &&
+      !noCookieSet
+    ) {
+      setCookie(ctx, { appId }, Cookies.CurrentApp)
     }
-  }
-  if (
-    (requestAppId !== appId ||
-      appCookie == null ||
-      appCookie.appId !== requestAppId) &&
-    !noCookieSet
-  ) {
-    setCookie(ctx, { appId }, Cookies.CurrentApp)
-  }
 
-  return next()
+    return next()
+  })
 }

package/src/middleware/tests/authorized.spec.js

@@ -11,26 +11,33 @@ const authorizedMiddleware = require("../authorized")
 const env = require("../../environment")
 const { PermissionTypes, PermissionLevels } = require("@budibase/backend-core/permissions")
 require("@budibase/backend-core").init(require("../../db"))
+const { doInAppContext } = require("@budibase/backend-core/context")
+
+const APP_ID = ""
 
 class TestConfiguration {
   constructor(role) {
     this.middleware = authorizedMiddleware(role)
     this.next = jest.fn()
     this.throw = jest.fn()
+    this.headers = {}
     this.ctx = {
       headers: {},
       request: {
         url: ""
       },
-      appId: "",
+      appId: APP_ID,
       auth: {},
       next: this.next,
-      throw: this.throw
+      throw: this.throw,
+      get: (name) => this.headers[name],
     }
   }
 
   executeMiddleware() {
-    return this.middleware(this.ctx, this.next)
+    return doInAppContext(APP_ID, () => {
+      return this.middleware(this.ctx, this.next)
+    })
   }
 
   setUser(user) {
@@ -46,7 +53,7 @@ class TestConfiguration {
   }
 
   setAuthenticated(isAuthed) {
-    this.ctx.auth = { authenticated: isAuthed }
+    this.ctx.isAuthenticated = isAuthed
   }
 
   setRequestUrl(url) {
@@ -107,7 +114,7 @@ describe("Authorization middleware", () => {
       expect(config.next).toHaveBeenCalled()
     })
     
-    it("throws if the user has only builder permissions", async () => {
+    it("throws if the user does not have builder permissions", async () => {
       config.setEnvironment(false)
       config.setMiddlewareRequiredPermission(PermissionTypes.BUILDER)
       config.setUser({
@@ -133,7 +140,7 @@ describe("Authorization middleware", () => {
       expect(config.next).toHaveBeenCalled()
     })
 
-    it("throws if the user session is not authenticated after permission checks", async () => {
+    it("throws if the user session is not authenticated", async () => {
       config.setUser({
         role: {
           _id: ""

package/src/middleware/tests/currentapp.spec.js

@@ -1,6 +1,11 @@
 mockAuthWithNoCookie()
 mockWorker()
 
+jest.mock("@budibase/backend-core/db", () => ({
+  ...jest.requireActual("@budibase/backend-core/db"),
+  dbExists: () => true,
+}))
+
 function mockWorker() {
   jest.mock("../../utilities/workerRequests", () => ({
     getGlobalSelf: () => {
@@ -50,6 +55,7 @@ function mockAuthWithCookie() {
       return "app_test"
     },
     setCookie: jest.fn(),
+    clearCookie: jest.fn(),
     getCookie: () => ({appId: "app_different", roleId: "PUBLIC"}),
   }))
   jest.mock("@budibase/backend-core/constants", () => ({

package/src/middleware/tests/usageQuota.spec.js

@@ -1,11 +1,5 @@
 jest.mock("../../db")
 jest.mock("../../utilities/usageQuota")
-jest.mock("../../environment", () => ({
-  isTest: () => true,
-  isProd: () => false,
-  isDev: () => true,
-  _set: () => {},
-}))
 jest.mock("@budibase/backend-core/tenancy", () => ({
   getTenantId: () => "testing123"
 }))
@@ -29,9 +23,10 @@ class TestConfiguration {
       },
       req: {
         method: "POST",
-        url: "/rows"
+        url: "/applications"
       }
     }
+    usageQuota.useQuotas = () => true
   }
 
   executeMiddleware() {
@@ -113,7 +108,6 @@ describe("usageQuota middleware", () => {
 
   it("calculates and persists the correct usage quota for the relevant action", async () => {
     config.setUrl("/rows")
-    config.setProd(true)
 
     await config.executeMiddleware()
 
@@ -121,20 +115,20 @@ describe("usageQuota middleware", () => {
     expect(config.next).toHaveBeenCalled()
   })
 
-  it("calculates the correct file size from a file upload call and adds it to quota", async () => {
-    config.setUrl("/upload")
-    config.setProd(true)
-    config.setFiles([
-      {
-        size: 100
-      },
-      {
-        size: 10000
-      },
-    ])
-    await config.executeMiddleware()
-
-    expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
-    expect(config.next).toHaveBeenCalled()
-  })
+  // it("calculates the correct file size from a file upload call and adds it to quota", async () => {
+  //   config.setUrl("/upload")
+  //   config.setProd(true)
+  //   config.setFiles([
+  //     {
+  //       size: 100
+  //     },
+  //     {
+  //       size: 10000
+  //     },
+  //   ])
+  //   await config.executeMiddleware()
+
+  //   expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
+  //   expect(config.next).toHaveBeenCalled()
+  // })
 })

package/src/middleware/usageQuota.js

@@ -1,14 +1,10 @@
-const CouchDB = require("../db")
 const usageQuota = require("../utilities/usageQuota")
-const env = require("../environment")
-const { getTenantId } = require("@budibase/backend-core/tenancy")
+const { getUniqueRows } = require("../utilities/usageQuota/rows")
 const {
   isExternalTable,
   isRowId: isExternalRowId,
 } = require("../integrations/utils")
-
-// tenants without limits
-const EXCLUDED_TENANTS = ["bb", "default", "bbtest", "bbstaging"]
+const { getAppDB } = require("@budibase/backend-core/context")
 
 // currently only counting new writes and deletes
 const METHOD_MAP = {
@@ -18,13 +14,13 @@ const METHOD_MAP = {
 
 const DOMAIN_MAP = {
   rows: usageQuota.Properties.ROW,
-  upload: usageQuota.Properties.UPLOAD,
-  views: usageQuota.Properties.VIEW,
-  users: usageQuota.Properties.USER,
+  // upload: usageQuota.Properties.UPLOAD,            // doesn't work yet
+  // views: usageQuota.Properties.VIEW,               // doesn't work yet
+  // users: usageQuota.Properties.USER,               // doesn't work yet
   applications: usageQuota.Properties.APPS,
   // this will not be updated by endpoint calls
   // instead it will be updated by triggerInfo
-  automationRuns: usageQuota.Properties.AUTOMATION,
+  // automationRuns: usageQuota.Properties.AUTOMATION, // doesn't work yet
 }
 
 function getProperty(url) {
@@ -36,10 +32,7 @@ function getProperty(url) {
 }
 
 module.exports = async (ctx, next) => {
-  const tenantId = getTenantId()
-
-  // if in development or a self hosted cloud usage quotas should not be executed
-  if (env.isDev() || env.SELF_HOSTED || EXCLUDED_TENANTS.includes(tenantId)) {
+  if (!usageQuota.useQuotas()) {
     return next()
   }
 
@@ -53,7 +46,7 @@ module.exports = async (ctx, next) => {
     const usageId = ctx.request.body._id
     try {
       if (ctx.appId) {
-        const db = new CouchDB(ctx.appId)
+        const db = getAppDB()
         await db.get(usageId)
       }
       return next()
@@ -80,9 +73,92 @@ module.exports = async (ctx, next) => {
     usage = files.map(file => file.size).reduce((total, size) => total + size)
   }
   try {
-    await usageQuota.update(property, usage)
-    return next()
+    await performRequest(ctx, next, property, usage)
   } catch (err) {
     ctx.throw(400, err)
   }
 }
+
+const performRequest = async (ctx, next, property, usage) => {
+  const usageContext = {
+    skipNext: false,
+    skipUsage: false,
+    [usageQuota.Properties.APPS]: {},
+  }
+
+  if (usage === -1) {
+    if (PRE_DELETE[property]) {
+      await PRE_DELETE[property](ctx, usageContext)
+    }
+  } else {
+    if (PRE_CREATE[property]) {
+      await PRE_CREATE[property](ctx, usageContext)
+    }
+  }
+
+  // run the request
+  if (!usageContext.skipNext) {
+    await usageQuota.update(property, usage, { dryRun: true })
+    await next()
+  }
+
+  if (usage === -1) {
+    if (POST_DELETE[property]) {
+      await POST_DELETE[property](ctx, usageContext)
+    }
+  } else {
+    if (POST_CREATE[property]) {
+      await POST_CREATE[property](ctx, usageContext)
+    }
+  }
+
+  // update the usage
+  if (!usageContext.skipUsage) {
+    await usageQuota.update(property, usage)
+  }
+}
+
+const appPreDelete = async (ctx, usageContext) => {
+  if (ctx.query.unpublish) {
+    // don't run usage decrement for unpublish
+    usageContext.skipUsage = true
+    return
+  }
+
+  // store the row count to delete
+  const rows = await getUniqueRows([ctx.appId])
+  if (rows.length) {
+    usageContext[usageQuota.Properties.APPS] = { rowCount: rows.length }
+  }
+}
+
+const appPostDelete = async (ctx, usageContext) => {
+  // delete the app rows from usage
+  const rowCount = usageContext[usageQuota.Properties.APPS].rowCount
+  if (rowCount) {
+    await usageQuota.update(usageQuota.Properties.ROW, -rowCount)
+  }
+}
+
+const appPostCreate = async ctx => {
+  // app import & template creation
+  if (ctx.request.body.useTemplate === "true") {
+    const rows = await getUniqueRows([ctx.response.body.appId])
+    const rowCount = rows ? rows.length : 0
+    await usageQuota.update(usageQuota.Properties.ROW, rowCount)
+  }
+}
+
+const PRE_DELETE = {
+  [usageQuota.Properties.APPS]: appPreDelete,
+}
+
+const POST_DELETE = {
+  [usageQuota.Properties.APPS]: appPostDelete,
+}
+
+const PRE_CREATE = {}
+
+const POST_CREATE = {
+  [usageQuota.Properties.APPS]: appPostCreate,
+}

package/src/migrations/functions/appUrls.ts

@@ -0,0 +1,34 @@
+const { DocumentTypes } = require("@budibase/backend-core/db")
+import { getAppUrl } from "../../api/controllers/application"
+
+/**
+ * Date:
+ * January 2022
+ *
+ * Description:
+ * Add the url to the app metadata if it doesn't exist
+ */
+export const run = async (appDb: any) => {
+  let metadata
+  try {
+    metadata = await appDb.get(DocumentTypes.APP_METADATA)
+  } catch (e) {
+    // sometimes the metadata document doesn't exist
+    // exit early instead of failing the migration
+    console.error("Error retrieving app metadata. Skipping", e)
+    return
+  }
+
+  if (!metadata.url) {
+    const context = {
+      request: {
+        body: {
+          name: metadata.name,
+        },
+      },
+    }
+    metadata.url = getAppUrl(context)
+    console.log(`Adding url to app: ${metadata.url}`)
+    await appDb.put(metadata)
+  }
+}

package/src/migrations/functions/quotas1.ts

@@ -0,0 +1,20 @@
+import { runQuotaMigration } from "./usageQuotas"
+import * as syncApps from "./usageQuotas/syncApps"
+import * as syncRows from "./usageQuotas/syncRows"
+
+/**
+ * Date:
+ * January 2022
+ *
+ * Description:
+ * Synchronise the app and row quotas to the state of the db after it was
+ * discovered that the quota resets were still in place and the row quotas
+ * weren't being decremented correctly.
+ */
+
+export const run = async () => {
+  await runQuotaMigration(async () => {
+    await syncApps.run()
+    await syncRows.run()
+  })
+}

package/src/migrations/functions/tests/appUrls.spec.js

@@ -0,0 +1,29 @@
+const { DocumentTypes } = require("@budibase/backend-core/db")
+const env = require("../../../environment")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+
+const migration = require("../appUrls")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+  const CouchDB = config.getCouch()
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    const app = await config.createApp("testApp")
+    const appDb = new CouchDB(app.appId)
+    let metadata = await appDb.get(DocumentTypes.APP_METADATA)
+    delete metadata.url
+    await appDb.put(metadata)
+    
+    await migration.run(appDb)
+
+    metadata = await appDb.get(DocumentTypes.APP_METADATA)
+    expect(metadata.url).toEqual("/testapp")
+  })
+})

package/src/migrations/functions/tests/quotas1.spec.js

@@ -0,0 +1,27 @@
+const env = require("../../../environment")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+
+const syncApps = jest.fn()
+const syncRows = jest.fn()
+
+jest.mock("../usageQuotas/syncApps", () => ({ run: syncApps }) )
+jest.mock("../usageQuotas/syncRows", () => ({ run: syncRows }) )
+
+const migration = require("../quotas1")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)  
+
+  it("runs ", async () => {
+    await migration.run()
+    expect(syncApps).toHaveBeenCalledTimes(1)
+    expect(syncRows).toHaveBeenCalledTimes(1)
+  })
+})

package/src/migrations/functions/tests/userEmailViewCasing.spec.js

@@ -0,0 +1,25 @@
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+
+// mock email view creation
+const coreDb = require("@budibase/backend-core/db")
+const createUserEmailView = jest.fn()
+coreDb.createUserEmailView = createUserEmailView
+
+const migration = require("../userEmailViewCasing")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+  const globalDb = getGlobalDB()
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    await migration.run(globalDb)
+    expect(createUserEmailView).toHaveBeenCalledTimes(1)
+  })
+})

package/src/migrations/functions/usageQuotas/index.ts

@@ -0,0 +1,8 @@
+const { useQuotas } = require("../../../utilities/usageQuota")
+
+export const runQuotaMigration = async (migration: Function) => {
+  if (!useQuotas()) {
+    return
+  }
+  await migration()
+}

package/src/migrations/functions/usageQuotas/syncApps.ts

@@ -0,0 +1,18 @@
+import { getGlobalDB, getTenantId } from "@budibase/backend-core/tenancy"
+import { getAllApps } from "@budibase/backend-core/db"
+import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
+
+export const run = async () => {
+  const db = getGlobalDB()
+  // get app count
+  // @ts-ignore
+  const devApps = await getAllApps({ dev: true })
+  const appCount = devApps ? devApps.length : 0
+
+  // sync app count
+  const tenantId = getTenantId()
+  console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.apps = appCount
+  await db.put(usageDoc)
+}

package/src/migrations/functions/usageQuotas/syncRows.ts

@@ -0,0 +1,22 @@
+import { getGlobalDB, getTenantId } from "@budibase/backend-core/tenancy"
+import { getAllApps } from "@budibase/backend-core/db"
+import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
+import { getUniqueRows } from "../../../utilities/usageQuota/rows"
+
+export const run = async () => {
+  const db = getGlobalDB()
+  // get all rows in all apps
+  // @ts-ignore
+  const allApps = await getAllApps({ all: true })
+  // @ts-ignore
+  const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : []
+  const rows = await getUniqueRows(appIds)
+  const rowCount = rows ? rows.length : 0
+
+  // sync row count
+  const tenantId = getTenantId()
+  console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.rows = rowCount
+  await db.put(usageDoc)
+}

package/src/migrations/functions/usageQuotas/tests/syncApps.spec.js

@@ -0,0 +1,37 @@
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const TestConfig = require("../../../../tests/utilities/TestConfiguration")
+const { getUsageQuotaDoc, update, Properties } = require("../../../../utilities/usageQuota")
+const syncApps = require("../syncApps")
+const env = require("../../../../environment")
+
+describe("syncApps", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    // create the usage quota doc and mock usages
+    const db = getGlobalDB()
+    await getUsageQuotaDoc(db)
+    await update(Properties.APPS, 3)
+
+    let usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.apps).toEqual(3)
+
+    // create an extra app to test the migration
+    await config.createApp("quota-test")
+
+    // migrate
+    await syncApps.run()
+
+    // assert the migration worked 
+    usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.apps).toEqual(2)
+  })
+})
+