@budibase/server 1.0.46 → 1.0.49-alpha.1

package/src/middleware/tests/usageQuota.spec.js

@@ -1,11 +1,5 @@
 jest.mock("../../db")
 jest.mock("../../utilities/usageQuota")
-jest.mock("../../environment", () => ({
-  isTest: () => true,
-  isProd: () => false,
-  isDev: () => true,
-  _set: () => {},
-}))
 jest.mock("@budibase/backend-core/tenancy", () => ({
   getTenantId: () => "testing123"
 }))
@@ -29,9 +23,10 @@ class TestConfiguration {
       },
       req: {
         method: "POST",
-        url: "/rows"
+        url: "/applications"
       }
     }
+    usageQuota.useQuotas = () => true
   }
 
   executeMiddleware() {
@@ -113,7 +108,6 @@ describe("usageQuota middleware", () => {
 
   it("calculates and persists the correct usage quota for the relevant action", async () => {
     config.setUrl("/rows")
-    config.setProd(true)
 
     await config.executeMiddleware()
 
@@ -121,20 +115,20 @@ describe("usageQuota middleware", () => {
     expect(config.next).toHaveBeenCalled()
   })
 
-  it("calculates the correct file size from a file upload call and adds it to quota", async () => {
-    config.setUrl("/upload")
-    config.setProd(true)
-    config.setFiles([
-      {
-        size: 100
-      },
-      {
-        size: 10000
-      },
-    ])
-    await config.executeMiddleware()
-
-    expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
-    expect(config.next).toHaveBeenCalled()
-  })
+  // it("calculates the correct file size from a file upload call and adds it to quota", async () => {
+  //   config.setUrl("/upload")
+  //   config.setProd(true)
+  //   config.setFiles([
+  //     {
+  //       size: 100
+  //     },
+  //     {
+  //       size: 10000
+  //     },
+  //   ])
+  //   await config.executeMiddleware()
+
+  //   expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
+  //   expect(config.next).toHaveBeenCalled()
+  // })
 })

package/src/middleware/usageQuota.js

@@ -1,14 +1,11 @@
 const CouchDB = require("../db")
 const usageQuota = require("../utilities/usageQuota")
-const env = require("../environment")
-const { getTenantId } = require("@budibase/backend-core/tenancy")
+const { getUniqueRows } = require("../utilities/usageQuota/rows")
 const {
   isExternalTable,
   isRowId: isExternalRowId,
 } = require("../integrations/utils")
-
-// tenants without limits
-const EXCLUDED_TENANTS = ["bb", "default", "bbtest", "bbstaging"]
+const migration = require("../migrations/usageQuotas")
 
 // currently only counting new writes and deletes
 const METHOD_MAP = {
@@ -18,13 +15,13 @@ const METHOD_MAP = {
 
 const DOMAIN_MAP = {
   rows: usageQuota.Properties.ROW,
-  upload: usageQuota.Properties.UPLOAD,
-  views: usageQuota.Properties.VIEW,
-  users: usageQuota.Properties.USER,
+  // upload: usageQuota.Properties.UPLOAD,            // doesn't work yet
+  // views: usageQuota.Properties.VIEW,               // doesn't work yet
+  // users: usageQuota.Properties.USER,               // doesn't work yet
   applications: usageQuota.Properties.APPS,
   // this will not be updated by endpoint calls
   // instead it will be updated by triggerInfo
-  automationRuns: usageQuota.Properties.AUTOMATION,
+  // automationRuns: usageQuota.Properties.AUTOMATION, // doesn't work yet
 }
 
 function getProperty(url) {
@@ -36,10 +33,7 @@ function getProperty(url) {
 }
 
 module.exports = async (ctx, next) => {
-  const tenantId = getTenantId()
-
-  // if in development or a self hosted cloud usage quotas should not be executed
-  if (env.isDev() || env.SELF_HOSTED || EXCLUDED_TENANTS.includes(tenantId)) {
+  if (!usageQuota.useQuotas()) {
     return next()
   }
 
@@ -80,9 +74,93 @@ module.exports = async (ctx, next) => {
     usage = files.map(file => file.size).reduce((total, size) => total + size)
   }
   try {
-    await usageQuota.update(property, usage)
-    return next()
+    await migration.run()
+    await performRequest(ctx, next, property, usage)
   } catch (err) {
     ctx.throw(400, err)
   }
 }
+
+const performRequest = async (ctx, next, property, usage) => {
+  const usageContext = {
+    skipNext: false,
+    skipUsage: false,
+    [usageQuota.Properties.APPS]: {},
+  }
+
+  if (usage === -1) {
+    if (PRE_DELETE[property]) {
+      await PRE_DELETE[property](ctx, usageContext)
+    }
+  } else {
+    if (PRE_CREATE[property]) {
+      await PRE_CREATE[property](ctx, usageContext)
+    }
+  }
+
+  // run the request
+  if (!usageContext.skipNext) {
+    await usageQuota.update(property, usage, { dryRun: true })
+    await next()
+  }
+
+  if (usage === -1) {
+    if (POST_DELETE[property]) {
+      await POST_DELETE[property](ctx, usageContext)
+    }
+  } else {
+    if (POST_CREATE[property]) {
+      await POST_CREATE[property](ctx, usageContext)
+    }
+  }
+
+  // update the usage
+  if (!usageContext.skipUsage) {
+    await usageQuota.update(property, usage)
+  }
+}
+
+const appPreDelete = async (ctx, usageContext) => {
+  if (ctx.query.unpublish) {
+    // don't run usage decrement for unpublish
+    usageContext.skipUsage = true
+    return
+  }
+
+  // store the row count to delete
+  const rows = await getUniqueRows([ctx.appId])
+  if (rows.length) {
+    usageContext[usageQuota.Properties.APPS] = { rowCount: rows.length }
+  }
+}
+
+const appPostDelete = async (ctx, usageContext) => {
+  // delete the app rows from usage
+  const rowCount = usageContext[usageQuota.Properties.APPS].rowCount
+  if (rowCount) {
+    await usageQuota.update(usageQuota.Properties.ROW, -rowCount)
+  }
+}
+
+const appPostCreate = async ctx => {
+  // app import & template creation
+  if (ctx.request.body.useTemplate === "true") {
+    const rows = await getUniqueRows([ctx.response.body.appId])
+    const rowCount = rows ? rows.length : 0
+    await usageQuota.update(usageQuota.Properties.ROW, rowCount)
+  }
+}
+
+const PRE_DELETE = {
+  [usageQuota.Properties.APPS]: appPreDelete,
+}
+
+const POST_DELETE = {
+  [usageQuota.Properties.APPS]: appPostDelete,
+}
+
+const PRE_CREATE = {}
+
+const POST_CREATE = {
+  [usageQuota.Properties.APPS]: appPostCreate,
+}

package/src/migrations/tests/usageQuotas/index.spec.js

@@ -0,0 +1,27 @@
+const env = require("../../../environment")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+
+const syncApps = jest.fn()
+const syncRows = jest.fn()
+
+jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) )
+jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) )
+
+const migrations = require("../../usageQuotas")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)  
+
+  it("runs the required migrations", async () => {
+    await migrations.run()
+    expect(syncApps).toHaveBeenCalledTimes(1)
+    expect(syncRows).toHaveBeenCalledTimes(1)
+  })
+})

package/src/migrations/tests/usageQuotas/syncApps.spec.js

@@ -0,0 +1,37 @@
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
+const syncApps = require("../../usageQuotas/syncApps")
+const env = require("../../../environment")
+
+describe("syncApps", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    // create the usage quota doc and mock usages
+    const db = getGlobalDB()
+    await getUsageQuotaDoc(db)
+    await update(Properties.APPS, 3)
+
+    let usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.apps).toEqual(3)
+
+    // create an extra app to test the migration
+    await config.createApp("quota-test")
+
+    // migrate
+    await syncApps.run()
+
+    // assert the migration worked 
+    usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.apps).toEqual(2)
+  })
+})
+

package/src/migrations/tests/usageQuotas/syncRows.spec.js

@@ -0,0 +1,43 @@
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
+const syncRows = require("../../usageQuotas/syncRows")
+const env = require("../../../environment")
+
+describe("syncRows", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    // create the usage quota doc and mock usages
+    const db = getGlobalDB()
+    await getUsageQuotaDoc(db)
+    await update(Properties.ROW, 300)
+
+    let usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.rows).toEqual(300)
+
+    // app 1
+    await config.createTable()
+    await config.createRow()
+    // app 2
+    await config.createApp("second-app")
+    await config.createTable()
+    await config.createRow()
+    await config.createRow()
+    
+    // migrate
+    await syncRows.run()
+
+    // assert the migration worked 
+    usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.rows).toEqual(3)
+  })
+})
+

package/src/migrations/usageQuotas/index.js

@@ -0,0 +1,24 @@
+const {
+  MIGRATIONS,
+  MIGRATION_DBS,
+  migrateIfRequired,
+} = require("@budibase/backend-core/migrations")
+const { useQuotas } = require("../../utilities/usageQuota")
+const syncApps = require("./syncApps")
+const syncRows = require("./syncRows")
+
+exports.run = async () => {
+  if (!useQuotas()) {
+    return
+  }
+
+  // Jan 2022
+  await migrateIfRequired(
+    MIGRATION_DBS.GLOBAL_DB,
+    MIGRATIONS.QUOTAS_1,
+    async () => {
+      await syncApps.run()
+      await syncRows.run()
+    }
+  )
+}

package/src/migrations/usageQuotas/syncApps.js

@@ -0,0 +1,18 @@
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
+const { getAllApps } = require("@budibase/backend-core/db")
+const CouchDB = require("../../db")
+const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
+
+exports.run = async () => {
+  const db = getGlobalDB()
+  // get app count
+  const devApps = await getAllApps(CouchDB, { dev: true })
+  const appCount = devApps ? devApps.length : 0
+
+  // sync app count
+  const tenantId = getTenantId()
+  console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.apps = appCount
+  await db.put(usageDoc)
+}

package/src/migrations/usageQuotas/syncRows.js

@@ -0,0 +1,21 @@
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
+const { getAllApps } = require("@budibase/backend-core/db")
+const CouchDB = require("../../db")
+const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
+const { getUniqueRows } = require("../../utilities/usageQuota/rows")
+
+exports.run = async () => {
+  const db = getGlobalDB()
+  // get all rows in all apps
+  const allApps = await getAllApps(CouchDB, { all: true })
+  const appIds = allApps ? allApps.map(app => app.appId) : []
+  const rows = await getUniqueRows(appIds)
+  const rowCount = rows ? rows.length : 0
+
+  // sync row count
+  const tenantId = getTenantId()
+  console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.rows = rowCount
+  await db.put(usageDoc)
+}

package/src/tests/utilities/TestConfiguration.js

@@ -22,10 +22,12 @@ const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 const { createASession } = require("@budibase/backend-core/sessions")
 const { user: userCache } = require("@budibase/backend-core/cache")
 const CouchDB = require("../../db")
+const newid = require("../../db/newid")
 core.init(CouchDB)
 
 const GLOBAL_USER_ID = "us_uuid1"
 const EMAIL = "babs@babs.com"
+const CSRF_TOKEN = "e3727778-7af0-4226-b5eb-f43cbe60a306"
 
 class TestConfiguration {
   constructor(openServer = true) {
@@ -85,7 +87,11 @@ class TestConfiguration {
       roles: roles || {},
       tenantId: TENANT_ID,
     }
-    await createASession(id, { sessionId: "sessionid", tenantId: TENANT_ID })
+    await createASession(id, {
+      sessionId: "sessionid",
+      tenantId: TENANT_ID,
+      csrfToken: CSRF_TOKEN,
+    })
     if (builder) {
       user.builder = { global: true }
     } else {
@@ -98,7 +104,8 @@ class TestConfiguration {
     }
   }
 
-  async init(appName = "test_application") {
+  // use a new id as the name to avoid name collisions
+  async init(appName = newid()) {
     await this.globalUser()
     return this.createApp(appName)
   }
@@ -131,6 +138,7 @@ class TestConfiguration {
         `${Cookies.Auth}=${authToken}`,
         `${Cookies.CurrentApp}=${appToken}`,
       ],
+      [Headers.CSRF_TOKEN]: CSRF_TOKEN,
     }
     if (this.appId) {
       headers[Headers.APP_ID] = this.appId
@@ -424,10 +432,6 @@ class TestConfiguration {
         roles: { [this.prodAppId]: roleId },
       })
     }
-    await createASession(userId, {
-      sessionId: "sessionid",
-      tenantId: TENANT_ID,
-    })
     // have to fake this
     const auth = {
       userId,

package/src/threads/query.js

@@ -78,6 +78,11 @@ class QueryRunner {
       return this.execute()
     }
 
+    // check for undefined response
+    if (!rows) {
+      rows = []
+    }
+
     // needs to an array for next step
     if (!Array.isArray(rows)) {
       rows = [rows]

package/src/utilities/rowProcessor/index.js

@@ -180,6 +180,8 @@ function processAutoColumn(
 }
 exports.processAutoColumn = processAutoColumn
 
+exports.processFormulas = processFormulas
+
 /**
  * This will coerce a value to the correct types based on the type transform map
  * @param {object} row The value to coerce
@@ -229,11 +231,12 @@ exports.inputProcessing = (
       }
       continue
     }
-    // specific case to delete formula values if they get saved
-    // type coercion cannot completely remove the field, so have to do it here
+    // remove any formula values, they are to be generated
     if (field.type === FieldTypes.FORMULA) {
       delete clonedRow[key]
-    } else {
+    }
+    // otherwise coerce what is there to correct types
+    else {
       clonedRow[key] = exports.coerce(value, field.type)
     }
   }
@@ -250,7 +253,7 @@ exports.inputProcessing = (
 /**
  * This function enriches the input rows with anything they are supposed to contain, for example
  * link records or attachment links.
- * @param {object} ctx the request which is looking for enriched rows.
+ * @param {string} appId the app in which the request is looking for enriched rows.
  * @param {object} table the table from which these rows came from originally, this is used to determine
  * the schema of the rows and then enrich.
  * @param {object[]|object} rows the rows which are to be enriched.
@@ -258,22 +261,21 @@ exports.inputProcessing = (
  * @returns {object[]|object} the enriched rows will be returned.
  */
 exports.outputProcessing = async (
-  ctx,
+  { appId },
   table,
   rows,
   opts = { squash: true }
 ) => {
-  const appId = ctx.appId
   let wasArray = true
   if (!(rows instanceof Array)) {
     rows = [rows]
     wasArray = false
   }
   // attach any linked row information
-  let enriched = await linkRows.attachFullLinkedDocs(ctx, table, rows)
+  let enriched = await linkRows.attachFullLinkedDocs(appId, table, rows)
 
   // process formulas
-  enriched = processFormulas(table, enriched)
+  enriched = processFormulas(table, enriched, { dynamic: true })
 
   // update the attachments URL depending on hosting
   for (let [property, column] of Object.entries(table.schema)) {
@@ -305,9 +307,15 @@ exports.outputProcessing = async (
  * @param {any} row optional - the row being removed.
  * @param {any} rows optional - if multiple rows being deleted can do this in bulk.
  * @param {any} oldRow optional - if updating a row this will determine the difference.
+ * @param {any} oldTable optional - if updating a table, can supply the old table to look for
+ * deleted attachment columns.
  * @return {Promise<void>} When all attachments have been removed this will return.
  */
-exports.cleanupAttachments = async (appId, table, { row, rows, oldRow }) => {
+exports.cleanupAttachments = async (
+  appId,
+  table,
+  { row, rows, oldRow, oldTable }
+) => {
   if (!isProdAppID(appId)) {
     const prodAppId = getDeployedAppID(appId)
     // if prod exists, then don't allow deleting
@@ -322,12 +330,16 @@ exports.cleanupAttachments = async (appId, table, { row, rows, oldRow }) => {
       files = files.concat(row[key].map(attachment => attachment.key))
     }
   }
-  for (let [key, schema] of Object.entries(table.schema)) {
+  const schemaToUse = oldTable ? oldTable.schema : table.schema
+  for (let [key, schema] of Object.entries(schemaToUse)) {
     if (schema.type !== FieldTypes.ATTACHMENT) {
       continue
     }
-    // if updating, need to manage the differences
-    if (oldRow && row) {
+    // old table had this column, new table doesn't - delete it
+    if (oldTable && !table.schema[key]) {
+      rows.forEach(row => addFiles(row, key))
+    } else if (oldRow && row) {
+      // if updating, need to manage the differences
       files = files.concat(getRemovedAttachmentKeys(oldRow, row, key))
     } else if (row) {
       addFiles(row, key)

package/src/utilities/rowProcessor/utils.js

@@ -1,23 +1,39 @@
-const { FieldTypes } = require("../../constants")
+const { FieldTypes, FormulaTypes } = require("../../constants")
 const { processStringSync } = require("@budibase/string-templates")
 
 /**
  * Looks through the rows provided and finds formulas - which it then processes.
  */
-exports.processFormulas = (table, rows) => {
+exports.processFormulas = (
+  table,
+  rows,
+  { dynamic, contextRows } = { dynamic: true }
+) => {
   const single = !Array.isArray(rows)
   if (single) {
     rows = [rows]
+    contextRows = contextRows ? [contextRows] : contextRows
   }
   for (let [column, schema] of Object.entries(table.schema)) {
-    if (schema.type !== FieldTypes.FORMULA) {
+    const isStatic = schema.formulaType === FormulaTypes.STATIC
+    if (
+      schema.type !== FieldTypes.FORMULA ||
+      (dynamic && isStatic) ||
+      (!dynamic && !isStatic)
+    ) {
       continue
     }
     // iterate through rows and process formula
-    rows = rows.map(row => ({
-      ...row,
-      [column]: processStringSync(schema.formula, row),
-    }))
+    for (let i = 0; i < rows.length; i++) {
+      if (schema.formula) {
+        let row = rows[i]
+        let context = contextRows ? contextRows[i] : row
+        rows[i] = {
+          ...row,
+          [column]: processStringSync(schema.formula, context),
+        }
+      }
+    }
   }
   return single ? rows[0] : rows
 }

package/src/utilities/tests/usageQuota/usageQuota.spec.js

@@ -0,0 +1,72 @@
+const getTenantId = jest.fn()
+jest.mock("@budibase/backend-core/tenancy", () => ({
+  getTenantId
+}))
+const usageQuota = require("../../usageQuota")
+const env = require("../../../environment")
+
+class TestConfiguration {
+  constructor() {
+    this.enableQuotas()
+  }
+
+  enableQuotas = () => {
+    env.USE_QUOTAS = 1
+  }
+
+  disableQuotas = () => {
+    env.USE_QUOTAS = null
+  }
+
+  setTenantId = (tenantId) => {
+    getTenantId.mockReturnValue(tenantId)
+  } 
+
+  setExcludedTenants = (tenants) => {
+    env.EXCLUDE_QUOTAS_TENANTS = tenants
+  }
+
+  reset = () => {
+    this.disableQuotas()
+    this.setExcludedTenants(null)
+  }
+}
+
+describe("usageQuota", () => {
+  let config
+
+  beforeEach(() => {
+    config = new TestConfiguration()
+  })
+
+  afterEach(() => {
+    config.reset()
+  })
+
+  describe("useQuotas", () => {
+    it("works when no settings have been provided", () => {
+      config.reset()
+      expect(usageQuota.useQuotas()).toBe(false)
+    })
+    it("honours USE_QUOTAS setting", () => {
+      config.disableQuotas()
+      expect(usageQuota.useQuotas()).toBe(false)
+
+      config.enableQuotas()
+      expect(usageQuota.useQuotas()).toBe(true)
+    })
+    it("honours EXCLUDE_QUOTAS_TENANTS setting", () => {
+      config.setTenantId("test")
+
+      // tenantId is in the list
+      config.setExcludedTenants("test, test2, test2")
+      expect(usageQuota.useQuotas()).toBe(false)
+      config.setExcludedTenants("test,test2,test2")
+      expect(usageQuota.useQuotas()).toBe(false)
+
+      // tenantId is not in the list
+      config.setTenantId("other")
+      expect(usageQuota.useQuotas()).toBe(true)
+    })
+  })
+})