@budibase/server 2.0.30-alpha.1 → 2.0.30

package/src/migrations/functions/tests/{syncQuotas.spec.js → quotas1.spec.js}

@@ -2,13 +2,11 @@ const TestConfig = require("../../../tests/utilities/TestConfiguration")
 
 const syncApps = jest.fn()
 const syncRows = jest.fn()
-const syncPlugins = jest.fn()
 
 jest.mock("../usageQuotas/syncApps", () => ({ run: syncApps }) )
 jest.mock("../usageQuotas/syncRows", () => ({ run: syncRows }) )
-jest.mock("../usageQuotas/syncPlugins", () => ({ run: syncPlugins }) )
 
-const migration = require("../syncQuotas")
+const migration = require("../quotas1")
 
 describe("run", () => {
   let config = new TestConfig(false)
@@ -19,10 +17,9 @@ describe("run", () => {
 
   afterAll(config.end)  
 
-  it("run", async () => {
+  it("runs ", async () => {
     await migration.run()
     expect(syncApps).toHaveBeenCalledTimes(1)
     expect(syncRows).toHaveBeenCalledTimes(1)
-    expect(syncPlugins).toHaveBeenCalledTimes(1)
   })
 })

package/src/migrations/functions/usageQuotas/syncApps.ts

@@ -5,6 +5,7 @@ import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
 
 export const run = async () => {
   // get app count
+  // @ts-ignore
   const devApps = await getAllApps({ dev: true })
   const appCount = devApps ? devApps.length : 0
 

package/src/migrations/functions/usageQuotas/syncRows.ts

@@ -2,28 +2,19 @@ import { getTenantId } from "@budibase/backend-core/tenancy"
 import { getAllApps } from "@budibase/backend-core/db"
 import { getUniqueRows } from "../../../utilities/usageQuota/rows"
 import { quotas } from "@budibase/pro"
-import { StaticQuotaName, QuotaUsageType } from "@budibase/types"
+import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
 
 export const run = async () => {
   // get all rows in all apps
+  // @ts-ignore
   const allApps = await getAllApps({ all: true })
+  // @ts-ignore
   const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : []
-  const { appRows } = await getUniqueRows(appIds)
-
-  // get the counts per app
-  const counts: { [key: string]: number } = {}
-  let rowCount = 0
-  Object.entries(appRows).forEach(([appId, rows]) => {
-    counts[appId] = rows.length
-    rowCount += rows.length
-  })
+  const rows = await getUniqueRows(appIds)
+  const rowCount = rows ? rows.length : 0
 
   // sync row count
   const tenantId = getTenantId()
   console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
-  await quotas.setUsagePerApp(
-    counts,
-    StaticQuotaName.ROWS,
-    QuotaUsageType.STATIC
-  )
+  await quotas.setUsage(rowCount, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
 }

package/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts

@@ -2,7 +2,6 @@ import TestConfig from "../../../../tests/utilities/TestConfiguration"
 import * as syncRows from "../syncRows"
 import { quotas } from "@budibase/pro"
 import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
-const { getProdAppID } = require("@budibase/backend-core/db")
 
 describe("syncRows", () => {
   let config = new TestConfig(false)
@@ -23,11 +22,10 @@ describe("syncRows", () => {
       expect(usageDoc.usageQuota.rows).toEqual(300)
 
       // app 1
-      const app1 = config.app
       await config.createTable()
       await config.createRow()
       // app 2
-      const app2 = await config.createApp("second-app")
+      await config.createApp("second-app")
       await config.createTable()
       await config.createRow()
       await config.createRow()
@@ -38,12 +36,6 @@ describe("syncRows", () => {
       // assert the migration worked
       usageDoc = await quotas.getQuotaUsage()
       expect(usageDoc.usageQuota.rows).toEqual(3)
-      expect(usageDoc.apps?.[getProdAppID(app1.appId)].usageQuota.rows).toEqual(
-        1
-      )
-      expect(usageDoc.apps?.[getProdAppID(app2.appId)].usageQuota.rows).toEqual(
-        2
-      )
     })
   })
 })

package/src/migrations/index.ts

@@ -1,18 +1,14 @@
-import { locks, migrations } from "@budibase/backend-core"
-import {
-  Migration,
-  MigrationOptions,
-  MigrationName,
-  LockType,
-  LockName,
-} from "@budibase/types"
+import { migrations, redis } from "@budibase/backend-core"
+import { Migration, MigrationOptions, MigrationName } from "@budibase/types"
 import env from "../environment"
 
 // migration functions
 import * as userEmailViewCasing from "./functions/userEmailViewCasing"
-import * as syncQuotas from "./functions/syncQuotas"
+import * as quota1 from "./functions/quotas1"
 import * as appUrls from "./functions/appUrls"
 import * as backfill from "./functions/backfill"
+import * as pluginCount from "./functions/pluginCount"
+
 /**
  * Populate the migration function and additional configuration from
  * the static migration definitions.
@@ -30,10 +26,10 @@ export const buildMigrations = () => {
         })
         break
       }
-      case MigrationName.SYNC_QUOTAS: {
+      case MigrationName.QUOTAS_1: {
         serverMigrations.push({
           ...definition,
-          fn: syncQuotas.run,
+          fn: quota1.run,
         })
         break
       }
@@ -73,6 +69,16 @@ export const buildMigrations = () => {
         })
         break
       }
+      case MigrationName.PLUGIN_COUNT: {
+        if (env.SELF_HOSTED) {
+          serverMigrations.push({
+            ...definition,
+            fn: pluginCount.run,
+            silent: !!env.SELF_HOSTED,
+            preventRetry: false,
+          })
+        }
+      }
     }
   }
 
@@ -92,14 +98,33 @@ export const migrate = async (options?: MigrationOptions) => {
 }
 
 const migrateWithLock = async (options?: MigrationOptions) => {
-  await locks.doWithLock(
-    {
-      type: LockType.TRY_ONCE,
-      name: LockName.MIGRATIONS,
-      ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes
-    },
-    async () => {
-      await migrations.runMigrations(MIGRATIONS, options)
+  // get a new lock client
+  const redlock = await redis.clients.getMigrationsRedlock()
+  // lock for 15 minutes
+  const ttl = 1000 * 60 * 15
+
+  let migrationLock
+
+  // acquire lock
+  try {
+    migrationLock = await redlock.lock("migrations", ttl)
+  } catch (e: any) {
+    if (e.name === "LockError") {
+      return
+    } else {
+      throw e
     }
-  )
+  }
+
+  // run migrations
+  try {
+    await migrations.runMigrations(MIGRATIONS, options)
+  } finally {
+    // release lock
+    try {
+      await migrationLock.unlock()
+    } catch (e) {
+      console.error("unable to release migration lock")
+    }
+  }
 }

package/src/migrations/tests/index.spec.ts

@@ -4,6 +4,7 @@ import {
   tenancy,
   DocumentType,
   context,
+  db,
 } from "@budibase/backend-core"
 import TestConfig from "../../tests/utilities/TestConfiguration"
 import structures from "../../tests/utilities/structures"

package/src/utilities/usageQuota/rows.js

@@ -2,7 +2,6 @@ const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils")
 const {
   isDevAppID,
   getDevelopmentAppID,
-  getProdAppID,
   doWithDB,
 } = require("@budibase/backend-core/db")
 
@@ -53,8 +52,7 @@ const getAppRows = async appId => {
  * Rows duplicates may exist across apps due to data import so they are not filtered out.
  */
 exports.getUniqueRows = async appIds => {
-  let uniqueRows = [],
-    rowsByApp = {}
+  let uniqueRows = []
   const pairs = getAppPairs(appIds)
 
   for (let pair of Object.values(pairs)) {
@@ -75,10 +73,8 @@ exports.getUniqueRows = async appIds => {
     // this can't be done on all rows because app import results in
     // duplicate row ids across apps
     // the array pre-concat is important to avoid stack overflow
-    const prodId = getProdAppID(pair.devId || pair.prodId)
-    rowsByApp[prodId] = [...new Set(appRows)]
-    uniqueRows = uniqueRows.concat(rowsByApp[prodId])
+    uniqueRows = uniqueRows.concat([...new Set(appRows)])
   }
 
-  return { rows: uniqueRows, appRows: rowsByApp }
+  return uniqueRows
 }

package/src/migrations/functions/usageQuotas/syncPlugins.ts

@@ -1,10 +0,0 @@
-import { logging } from "@budibase/backend-core"
-import { plugins } from "@budibase/pro"
-
-export const run = async () => {
-  try {
-    await plugins.checkPluginQuotas()
-  } catch (err) {
-    logging.logAlert("Failed to update plugin quotas", err)
-  }
-}