@budibase/server 2.0.30-alpha.7 → 2.0.31

package/src/integrations/tests/rest.spec.ts

@@ -1,16 +1,17 @@
-const mockFetch = jest.fn(() => ({
-  headers: {
-    raw: () => {
-      return { "content-type": ["application/json"] }
+jest.mock("node-fetch", () =>
+  jest.fn(() => ({
+    headers: {
+      raw: () => {
+        return { "content-type": ["application/json"] }
+      },
+      get: () => ["application/json"],
     },
-    get: () => ["application/json"],
-  },
-  json: jest.fn(() => ({
-    my_next_cursor: 123,
-  })),
-  text: jest.fn(),
-}))
-jest.mock("node-fetch", () => mockFetch)
+    json: jest.fn(() => ({
+      my_next_cursor: 123,
+    })),
+    text: jest.fn(),
+  }))
+)
 import fetch from "node-fetch"
 import { default as RestIntegration } from "../rest"
 const FormData = require("form-data")
@@ -255,7 +256,7 @@ describe("REST Integration", () => {
         authConfigId: "c59c14bd1898a43baa08da68959b24686",
       }
       await config.integration.read(query)
-      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/`, {
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/?`, {
         method: "GET",
         headers: {
           Authorization: "Basic dXNlcjpwYXNzd29yZA==",
@@ -268,7 +269,7 @@ describe("REST Integration", () => {
         authConfigId: "0d91d732f34e4befabeff50b392a8ff3",
       }
       await config.integration.read(query)
-      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/`, {
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/?`, {
         method: "GET",
         headers: {
           Authorization: "Bearer mytoken",
@@ -326,7 +327,7 @@ describe("REST Integration", () => {
         },
       }
       await config.integration.create(query)
-      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api`, {
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?`, {
         body: JSON.stringify({
           [pageParam]: pageValue,
           [sizeParam]: sizeValue,
@@ -358,7 +359,7 @@ describe("REST Integration", () => {
         },
       }
       await config.integration.create(query)
-      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api`, {
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?`, {
         body: expect.any(FormData),
         headers: {},
         method: "POST",
@@ -389,7 +390,7 @@ describe("REST Integration", () => {
         },
       }
       await config.integration.create(query)
-      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api`, {
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?`, {
         body: expect.any(URLSearchParams),
         headers: {},
         method: "POST",
@@ -455,7 +456,7 @@ describe("REST Integration", () => {
         },
       }
       const res = await config.integration.create(query)
-      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api`, {
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?`, {
         body: JSON.stringify({
           [pageParam]: pageValue,
           [sizeParam]: sizeValue,
@@ -489,7 +490,7 @@ describe("REST Integration", () => {
         },
       }
       const res = await config.integration.create(query)
-      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api`, {
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?`, {
         body: expect.any(FormData),
         headers: {},
         method: "POST",
@@ -522,7 +523,7 @@ describe("REST Integration", () => {
         },
       }
       const res = await config.integration.create(query)
-      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api`, {
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?`, {
         body: expect.any(URLSearchParams),
         headers: {},
         method: "POST",
@@ -562,7 +563,7 @@ describe("REST Integration", () => {
         legacyHttpParser: true,
       })
       await config.integration.read({})
-      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/`, {
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/?`, {
         method: "GET",
         headers: {},
         extraHttpOptions: {
@@ -571,21 +572,4 @@ describe("REST Integration", () => {
       })
     })
   })
-
-  it("Attaches custom agent when Reject Unauthorized option is false", async () => {
-    config = new TestConfiguration({
-      url: BASE_URL,
-      rejectUnauthorized: false,
-    })
-    await config.integration.read({})
-
-    const calls: any = mockFetch.mock.calls[0]
-    const url = calls[0]
-    expect(url).toBe(`${BASE_URL}/`)
-
-    const calledConfig = calls[1]
-    expect(calledConfig.method).toBe("GET")
-    expect(calledConfig.headers).toEqual({})
-    expect(calledConfig.agent.options.rejectUnauthorized).toBe(false)
-  })
 })

package/src/migrations/functions/backfill/global.ts

@@ -8,7 +8,7 @@ import {
   accounts,
   db as dbUtils,
 } from "@budibase/backend-core"
-import { QuotaUsage } from "@budibase/types"
+import { QuotaUsage } from "@budibase/pro"
 import {
   CloudAccount,
   App,

package/src/migrations/functions/pluginCount.ts

@@ -0,0 +1,12 @@
+import { tenancy, logging } from "@budibase/backend-core"
+import { plugins } from "@budibase/pro"
+
+export const run = async () => {
+  try {
+    await tenancy.doInTenant(tenancy.DEFAULT_TENANT_ID, async () => {
+      await plugins.checkPluginQuotas()
+    })
+  } catch (err) {
+    logging.logAlert("Failed to update plugin quotas", err)
+  }
+}

package/src/migrations/functions/{syncQuotas.ts → quotas1.ts}

@@ -1,15 +1,20 @@
 import { runQuotaMigration } from "./usageQuotas"
 import * as syncApps from "./usageQuotas/syncApps"
 import * as syncRows from "./usageQuotas/syncRows"
-import * as syncPlugins from "./usageQuotas/syncPlugins"
 
 /**
- * Synchronise quotas to the state of the db.
+ * Date:
+ * January 2022
+ *
+ * Description:
+ * Synchronise the app and row quotas to the state of the db after it was
+ * discovered that the quota resets were still in place and the row quotas
+ * weren't being decremented correctly.
  */
+
 export const run = async () => {
   await runQuotaMigration(async () => {
     await syncApps.run()
     await syncRows.run()
-    await syncPlugins.run()
   })
 }

package/src/migrations/functions/tests/{syncQuotas.spec.js → quotas1.spec.js}

@@ -2,13 +2,11 @@ const TestConfig = require("../../../tests/utilities/TestConfiguration")
 
 const syncApps = jest.fn()
 const syncRows = jest.fn()
-const syncPlugins = jest.fn()
 
 jest.mock("../usageQuotas/syncApps", () => ({ run: syncApps }) )
 jest.mock("../usageQuotas/syncRows", () => ({ run: syncRows }) )
-jest.mock("../usageQuotas/syncPlugins", () => ({ run: syncPlugins }) )
 
-const migration = require("../syncQuotas")
+const migration = require("../quotas1")
 
 describe("run", () => {
   let config = new TestConfig(false)
@@ -19,10 +17,9 @@ describe("run", () => {
 
   afterAll(config.end)  
 
-  it("run", async () => {
+  it("runs ", async () => {
     await migration.run()
     expect(syncApps).toHaveBeenCalledTimes(1)
     expect(syncRows).toHaveBeenCalledTimes(1)
-    expect(syncPlugins).toHaveBeenCalledTimes(1)
   })
 })

package/src/migrations/functions/usageQuotas/syncApps.ts

@@ -5,6 +5,7 @@ import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
 
 export const run = async () => {
   // get app count
+  // @ts-ignore
   const devApps = await getAllApps({ dev: true })
   const appCount = devApps ? devApps.length : 0
 

package/src/migrations/functions/usageQuotas/syncRows.ts

@@ -2,28 +2,19 @@ import { getTenantId } from "@budibase/backend-core/tenancy"
 import { getAllApps } from "@budibase/backend-core/db"
 import { getUniqueRows } from "../../../utilities/usageQuota/rows"
 import { quotas } from "@budibase/pro"
-import { StaticQuotaName, QuotaUsageType } from "@budibase/types"
+import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
 
 export const run = async () => {
   // get all rows in all apps
+  // @ts-ignore
   const allApps = await getAllApps({ all: true })
+  // @ts-ignore
   const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : []
-  const { appRows } = await getUniqueRows(appIds)
-
-  // get the counts per app
-  const counts: { [key: string]: number } = {}
-  let rowCount = 0
-  Object.entries(appRows).forEach(([appId, rows]) => {
-    counts[appId] = rows.length
-    rowCount += rows.length
-  })
+  const rows = await getUniqueRows(appIds)
+  const rowCount = rows ? rows.length : 0
 
   // sync row count
   const tenantId = getTenantId()
   console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
-  await quotas.setUsagePerApp(
-    counts,
-    StaticQuotaName.ROWS,
-    QuotaUsageType.STATIC
-  )
+  await quotas.setUsage(rowCount, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
 }

package/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts

@@ -2,7 +2,6 @@ import TestConfig from "../../../../tests/utilities/TestConfiguration"
 import * as syncRows from "../syncRows"
 import { quotas } from "@budibase/pro"
 import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
-const { getProdAppID } = require("@budibase/backend-core/db")
 
 describe("syncRows", () => {
   let config = new TestConfig(false)
@@ -23,11 +22,10 @@ describe("syncRows", () => {
       expect(usageDoc.usageQuota.rows).toEqual(300)
 
       // app 1
-      const app1 = config.app
       await config.createTable()
       await config.createRow()
       // app 2
-      const app2 = await config.createApp("second-app")
+      await config.createApp("second-app")
       await config.createTable()
       await config.createRow()
       await config.createRow()
@@ -38,12 +36,6 @@ describe("syncRows", () => {
       // assert the migration worked
       usageDoc = await quotas.getQuotaUsage()
       expect(usageDoc.usageQuota.rows).toEqual(3)
-      expect(usageDoc.apps?.[getProdAppID(app1.appId)].usageQuota.rows).toEqual(
-        1
-      )
-      expect(usageDoc.apps?.[getProdAppID(app2.appId)].usageQuota.rows).toEqual(
-        2
-      )
     })
   })
 })

package/src/migrations/index.ts

@@ -1,18 +1,14 @@
-import { locks, migrations } from "@budibase/backend-core"
-import {
-  Migration,
-  MigrationOptions,
-  MigrationName,
-  LockType,
-  LockName,
-} from "@budibase/types"
+import { migrations, redis } from "@budibase/backend-core"
+import { Migration, MigrationOptions, MigrationName } from "@budibase/types"
 import env from "../environment"
 
 // migration functions
 import * as userEmailViewCasing from "./functions/userEmailViewCasing"
-import * as syncQuotas from "./functions/syncQuotas"
+import * as quota1 from "./functions/quotas1"
 import * as appUrls from "./functions/appUrls"
 import * as backfill from "./functions/backfill"
+import * as pluginCount from "./functions/pluginCount"
+
 /**
  * Populate the migration function and additional configuration from
  * the static migration definitions.
@@ -30,10 +26,10 @@ export const buildMigrations = () => {
         })
         break
       }
-      case MigrationName.SYNC_QUOTAS: {
+      case MigrationName.QUOTAS_1: {
         serverMigrations.push({
           ...definition,
-          fn: syncQuotas.run,
+          fn: quota1.run,
         })
         break
       }
@@ -73,6 +69,16 @@ export const buildMigrations = () => {
         })
         break
       }
+      case MigrationName.PLUGIN_COUNT: {
+        if (env.SELF_HOSTED) {
+          serverMigrations.push({
+            ...definition,
+            fn: pluginCount.run,
+            silent: !!env.SELF_HOSTED,
+            preventRetry: false,
+          })
+        }
+      }
     }
   }
 
@@ -92,14 +98,33 @@ export const migrate = async (options?: MigrationOptions) => {
 }
 
 const migrateWithLock = async (options?: MigrationOptions) => {
-  await locks.doWithLock(
-    {
-      type: LockType.TRY_ONCE,
-      name: LockName.MIGRATIONS,
-      ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes
-    },
-    async () => {
-      await migrations.runMigrations(MIGRATIONS, options)
+  // get a new lock client
+  const redlock = await redis.clients.getMigrationsRedlock()
+  // lock for 15 minutes
+  const ttl = 1000 * 60 * 15
+
+  let migrationLock
+
+  // acquire lock
+  try {
+    migrationLock = await redlock.lock("migrations", ttl)
+  } catch (e: any) {
+    if (e.name === "LockError") {
+      return
+    } else {
+      throw e
     }
-  )
+  }
+
+  // run migrations
+  try {
+    await migrations.runMigrations(MIGRATIONS, options)
+  } finally {
+    // release lock
+    try {
+      await migrationLock.unlock()
+    } catch (e) {
+      console.error("unable to release migration lock")
+    }
+  }
 }

package/src/migrations/tests/index.spec.ts

@@ -4,6 +4,7 @@ import {
   tenancy,
   DocumentType,
   context,
+  db,
 } from "@budibase/backend-core"
 import TestConfig from "../../tests/utilities/TestConfiguration"
 import structures from "../../tests/utilities/structures"

package/src/utilities/csvParser.js

@@ -4,7 +4,6 @@ const { FieldTypes } = require("../constants")
 const VALIDATORS = {
   [FieldTypes.STRING]: () => true,
   [FieldTypes.OPTIONS]: () => true,
-  [FieldTypes.BARCODEQR]: () => true,
   [FieldTypes.NUMBER]: attribute => {
     // allow not to be present
     if (!attribute) {

package/src/utilities/rowProcessor/index.js

@@ -48,11 +48,6 @@ const TYPE_TRANSFORM_MAP = {
     [null]: "",
     [undefined]: undefined,
   },
-  [FieldTypes.BARCODEQR]: {
-    "": "",
-    [null]: "",
-    [undefined]: undefined,
-  },
   [FieldTypes.FORMULA]: {
     "": "",
     [null]: "",

package/src/utilities/usageQuota/rows.js

@@ -2,7 +2,6 @@ const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils")
 const {
   isDevAppID,
   getDevelopmentAppID,
-  getProdAppID,
   doWithDB,
 } = require("@budibase/backend-core/db")
 
@@ -53,8 +52,7 @@ const getAppRows = async appId => {
  * Rows duplicates may exist across apps due to data import so they are not filtered out.
  */
 exports.getUniqueRows = async appIds => {
-  let uniqueRows = [],
-    rowsByApp = {}
+  let uniqueRows = []
   const pairs = getAppPairs(appIds)
 
   for (let pair of Object.values(pairs)) {
@@ -75,10 +73,8 @@ exports.getUniqueRows = async appIds => {
     // this can't be done on all rows because app import results in
     // duplicate row ids across apps
     // the array pre-concat is important to avoid stack overflow
-    const prodId = getProdAppID(pair.devId || pair.prodId)
-    rowsByApp[prodId] = [...new Set(appRows)]
-    uniqueRows = uniqueRows.concat(rowsByApp[prodId])
+    uniqueRows = uniqueRows.concat([...new Set(appRows)])
   }
 
-  return { rows: uniqueRows, appRows: rowsByApp }
+  return uniqueRows
 }

package/src/migrations/functions/usageQuotas/syncPlugins.ts

@@ -1,10 +0,0 @@
-import { logging } from "@budibase/backend-core"
-import { plugins } from "@budibase/pro"
-
-export const run = async () => {
-  try {
-    await plugins.checkPluginQuotas()
-  } catch (err) {
-    logging.logAlert("Failed to update plugin quotas", err)
-  }
-}