@budibase/server 2.0.33 → 2.0.34-alpha.1

package/src/integrations/tests/rest.spec.ts

@@ -1,17 +1,16 @@
-jest.mock("node-fetch", () =>
-  jest.fn(() => ({
-    headers: {
-      raw: () => {
-        return { "content-type": ["application/json"] }
-      },
-      get: () => ["application/json"],
+const mockFetch = jest.fn(() => ({
+  headers: {
+    raw: () => {
+      return { "content-type": ["application/json"] }
     },
-    json: jest.fn(() => ({
-      my_next_cursor: 123,
-    })),
-    text: jest.fn(),
-  }))
-)
+    get: () => ["application/json"],
+  },
+  json: jest.fn(() => ({
+    my_next_cursor: 123,
+  })),
+  text: jest.fn(),
+}))
+jest.mock("node-fetch", () => mockFetch)
 import fetch from "node-fetch"
 import { default as RestIntegration } from "../rest"
 const FormData = require("form-data")
@@ -256,7 +255,7 @@ describe("REST Integration", () => {
         authConfigId: "c59c14bd1898a43baa08da68959b24686",
       }
       await config.integration.read(query)
-      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/?`, {
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/`, {
         method: "GET",
         headers: {
           Authorization: "Basic dXNlcjpwYXNzd29yZA==",
@@ -269,7 +268,7 @@ describe("REST Integration", () => {
         authConfigId: "0d91d732f34e4befabeff50b392a8ff3",
       }
       await config.integration.read(query)
-      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/?`, {
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/`, {
         method: "GET",
         headers: {
           Authorization: "Bearer mytoken",
@@ -327,7 +326,7 @@ describe("REST Integration", () => {
         },
       }
       await config.integration.create(query)
-      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?`, {
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api`, {
         body: JSON.stringify({
           [pageParam]: pageValue,
           [sizeParam]: sizeValue,
@@ -359,7 +358,7 @@ describe("REST Integration", () => {
         },
       }
       await config.integration.create(query)
-      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?`, {
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api`, {
         body: expect.any(FormData),
         headers: {},
         method: "POST",
@@ -390,7 +389,7 @@ describe("REST Integration", () => {
         },
       }
       await config.integration.create(query)
-      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?`, {
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api`, {
         body: expect.any(URLSearchParams),
         headers: {},
         method: "POST",
@@ -456,7 +455,7 @@ describe("REST Integration", () => {
         },
       }
       const res = await config.integration.create(query)
-      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?`, {
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api`, {
         body: JSON.stringify({
           [pageParam]: pageValue,
           [sizeParam]: sizeValue,
@@ -490,7 +489,7 @@ describe("REST Integration", () => {
         },
       }
       const res = await config.integration.create(query)
-      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?`, {
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api`, {
         body: expect.any(FormData),
         headers: {},
         method: "POST",
@@ -523,7 +522,7 @@ describe("REST Integration", () => {
         },
       }
       const res = await config.integration.create(query)
-      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?`, {
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api`, {
         body: expect.any(URLSearchParams),
         headers: {},
         method: "POST",
@@ -563,7 +562,7 @@ describe("REST Integration", () => {
         legacyHttpParser: true,
       })
       await config.integration.read({})
-      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/?`, {
+      expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/`, {
         method: "GET",
         headers: {},
         extraHttpOptions: {
@@ -572,4 +571,21 @@ describe("REST Integration", () => {
       })
     })
   })
+
+  it("Attaches custom agent when Reject Unauthorized option is false", async () => {
+    config = new TestConfiguration({
+      url: BASE_URL,
+      rejectUnauthorized: false,
+    })
+    await config.integration.read({})
+
+    const calls: any = mockFetch.mock.calls[0]
+    const url = calls[0]
+    expect(url).toBe(`${BASE_URL}/`)
+
+    const calledConfig = calls[1]
+    expect(calledConfig.method).toBe("GET")
+    expect(calledConfig.headers).toEqual({})
+    expect(calledConfig.agent.options.rejectUnauthorized).toBe(false)
+  })
 })

package/src/migrations/functions/backfill/app/tables.ts

@@ -1,18 +1,12 @@
 import { events } from "@budibase/backend-core"
-import { getTableParams } from "../../../../db/utils"
-import { Table } from "@budibase/types"
+import sdk from "../../../../sdk"
+import PouchDB from "pouchdb"
 
-const getTables = async (appDb: any): Promise<Table[]> => {
-  const response = await appDb.allDocs(
-    getTableParams(null, {
-      include_docs: true,
-    })
-  )
-  return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
-  const tables = await getTables(appDb)
+export const backfill = async (
+  appDb: PouchDB.Database,
+  timestamp: string | number
+) => {
+  const tables = await sdk.tables.getAllInternalTables(appDb)
 
   for (const table of tables) {
     await events.table.created(table, timestamp)

package/src/migrations/functions/backfill/global.ts

@@ -8,7 +8,7 @@ import {
   accounts,
   db as dbUtils,
 } from "@budibase/backend-core"
-import { QuotaUsage } from "@budibase/pro"
+import { QuotaUsage } from "@budibase/types"
 import {
   CloudAccount,
   App,

package/src/migrations/functions/{quotas1.ts → syncQuotas.ts}

@@ -1,20 +1,15 @@
 import { runQuotaMigration } from "./usageQuotas"
 import * as syncApps from "./usageQuotas/syncApps"
 import * as syncRows from "./usageQuotas/syncRows"
+import * as syncPlugins from "./usageQuotas/syncPlugins"
 
 /**
- * Date:
- * January 2022
- *
- * Description:
- * Synchronise the app and row quotas to the state of the db after it was
- * discovered that the quota resets were still in place and the row quotas
- * weren't being decremented correctly.
+ * Synchronise quotas to the state of the db.
  */
-
 export const run = async () => {
   await runQuotaMigration(async () => {
     await syncApps.run()
     await syncRows.run()
+    await syncPlugins.run()
   })
 }

package/src/migrations/functions/tests/{quotas1.spec.js → syncQuotas.spec.js}

@@ -2,11 +2,13 @@ const TestConfig = require("../../../tests/utilities/TestConfiguration")
 
 const syncApps = jest.fn()
 const syncRows = jest.fn()
+const syncPlugins = jest.fn()
 
 jest.mock("../usageQuotas/syncApps", () => ({ run: syncApps }) )
 jest.mock("../usageQuotas/syncRows", () => ({ run: syncRows }) )
+jest.mock("../usageQuotas/syncPlugins", () => ({ run: syncPlugins }) )
 
-const migration = require("../quotas1")
+const migration = require("../syncQuotas")
 
 describe("run", () => {
   let config = new TestConfig(false)
@@ -17,9 +19,10 @@ describe("run", () => {
 
   afterAll(config.end)  
 
-  it("runs ", async () => {
+  it("run", async () => {
     await migration.run()
     expect(syncApps).toHaveBeenCalledTimes(1)
     expect(syncRows).toHaveBeenCalledTimes(1)
+    expect(syncPlugins).toHaveBeenCalledTimes(1)
   })
 })

package/src/migrations/functions/usageQuotas/syncApps.ts

@@ -5,7 +5,6 @@ import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
 
 export const run = async () => {
   // get app count
-  // @ts-ignore
   const devApps = await getAllApps({ dev: true })
   const appCount = devApps ? devApps.length : 0
 

package/src/migrations/functions/usageQuotas/syncPlugins.ts

@@ -0,0 +1,10 @@
+import { logging } from "@budibase/backend-core"
+import { plugins } from "@budibase/pro"
+
+export const run = async () => {
+  try {
+    await plugins.checkPluginQuotas()
+  } catch (err) {
+    logging.logAlert("Failed to update plugin quotas", err)
+  }
+}

package/src/migrations/functions/usageQuotas/syncRows.ts

@@ -2,19 +2,28 @@ import { getTenantId } from "@budibase/backend-core/tenancy"
 import { getAllApps } from "@budibase/backend-core/db"
 import { getUniqueRows } from "../../../utilities/usageQuota/rows"
 import { quotas } from "@budibase/pro"
-import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
+import { StaticQuotaName, QuotaUsageType } from "@budibase/types"
 
 export const run = async () => {
   // get all rows in all apps
-  // @ts-ignore
   const allApps = await getAllApps({ all: true })
-  // @ts-ignore
   const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : []
-  const rows = await getUniqueRows(appIds)
-  const rowCount = rows ? rows.length : 0
+  const { appRows } = await getUniqueRows(appIds)
+
+  // get the counts per app
+  const counts: { [key: string]: number } = {}
+  let rowCount = 0
+  Object.entries(appRows).forEach(([appId, rows]) => {
+    counts[appId] = rows.length
+    rowCount += rows.length
+  })
 
   // sync row count
   const tenantId = getTenantId()
   console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
-  await quotas.setUsage(rowCount, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
+  await quotas.setUsagePerApp(
+    counts,
+    StaticQuotaName.ROWS,
+    QuotaUsageType.STATIC
+  )
 }

package/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts

@@ -2,6 +2,7 @@ import TestConfig from "../../../../tests/utilities/TestConfiguration"
 import * as syncRows from "../syncRows"
 import { quotas } from "@budibase/pro"
 import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
+const { getProdAppID } = require("@budibase/backend-core/db")
 
 describe("syncRows", () => {
   let config = new TestConfig(false)
@@ -22,10 +23,11 @@ describe("syncRows", () => {
       expect(usageDoc.usageQuota.rows).toEqual(300)
 
       // app 1
+      const app1 = config.app
       await config.createTable()
       await config.createRow()
       // app 2
-      await config.createApp("second-app")
+      const app2 = await config.createApp("second-app")
       await config.createTable()
       await config.createRow()
       await config.createRow()
@@ -36,6 +38,12 @@ describe("syncRows", () => {
       // assert the migration worked
       usageDoc = await quotas.getQuotaUsage()
       expect(usageDoc.usageQuota.rows).toEqual(3)
+      expect(usageDoc.apps?.[getProdAppID(app1.appId)].usageQuota.rows).toEqual(
+        1
+      )
+      expect(usageDoc.apps?.[getProdAppID(app2.appId)].usageQuota.rows).toEqual(
+        2
+      )
     })
   })
 })

package/src/migrations/index.ts

@@ -1,14 +1,18 @@
-import { migrations, redis } from "@budibase/backend-core"
-import { Migration, MigrationOptions, MigrationName } from "@budibase/types"
+import { locks, migrations } from "@budibase/backend-core"
+import {
+  Migration,
+  MigrationOptions,
+  MigrationName,
+  LockType,
+  LockName,
+} from "@budibase/types"
 import env from "../environment"
 
 // migration functions
 import * as userEmailViewCasing from "./functions/userEmailViewCasing"
-import * as quota1 from "./functions/quotas1"
+import * as syncQuotas from "./functions/syncQuotas"
 import * as appUrls from "./functions/appUrls"
 import * as backfill from "./functions/backfill"
-import * as pluginCount from "./functions/pluginCount"
-
 /**
  * Populate the migration function and additional configuration from
  * the static migration definitions.
@@ -26,10 +30,10 @@ export const buildMigrations = () => {
         })
         break
       }
-      case MigrationName.QUOTAS_1: {
+      case MigrationName.SYNC_QUOTAS: {
         serverMigrations.push({
           ...definition,
-          fn: quota1.run,
+          fn: syncQuotas.run,
         })
         break
       }
@@ -69,16 +73,6 @@ export const buildMigrations = () => {
         })
         break
       }
-      case MigrationName.PLUGIN_COUNT: {
-        if (env.SELF_HOSTED) {
-          serverMigrations.push({
-            ...definition,
-            fn: pluginCount.run,
-            silent: !!env.SELF_HOSTED,
-            preventRetry: false,
-          })
-        }
-      }
     }
   }
 
@@ -98,33 +92,15 @@ export const migrate = async (options?: MigrationOptions) => {
 }
 
 const migrateWithLock = async (options?: MigrationOptions) => {
-  // get a new lock client
-  const redlock = await redis.clients.getMigrationsRedlock()
-  // lock for 15 minutes
-  const ttl = 1000 * 60 * 15
-
-  let migrationLock
-
-  // acquire lock
-  try {
-    migrationLock = await redlock.lock("migrations", ttl)
-  } catch (e: any) {
-    if (e.name === "LockError") {
-      return
-    } else {
-      throw e
+  await locks.doWithLock(
+    {
+      type: LockType.TRY_ONCE,
+      name: LockName.MIGRATIONS,
+      ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes
+      systemLock: true,
+    },
+    async () => {
+      await migrations.runMigrations(MIGRATIONS, options)
     }
-  }
-
-  // run migrations
-  try {
-    await migrations.runMigrations(MIGRATIONS, options)
-  } finally {
-    // release lock
-    try {
-      await migrationLock.unlock()
-    } catch (e) {
-      console.error("unable to release migration lock")
-    }
-  }
+  )
 }

package/src/migrations/tests/index.spec.ts

@@ -4,7 +4,6 @@ import {
   tenancy,
   DocumentType,
   context,
-  db,
 } from "@budibase/backend-core"
 import TestConfig from "../../tests/utilities/TestConfiguration"
 import structures from "../../tests/utilities/structures"

package/src/sdk/app/backups/constants.ts

@@ -0,0 +1,2 @@
+export const DB_EXPORT_FILE = "db.txt"
+export const GLOBAL_DB_EXPORT_FILE = "global.txt"

package/src/sdk/app/backups/exports.ts

@@ -0,0 +1,171 @@
+import { db as dbCore } from "@budibase/backend-core"
+import { budibaseTempDir } from "../../../utilities/budibaseDir"
+import { retrieveDirectory } from "../../../utilities/fileSystem/utilities"
+import { streamFile, createTempFolder } from "../../../utilities/fileSystem"
+import { ObjectStoreBuckets } from "../../../constants"
+import {
+  LINK_USER_METADATA_PREFIX,
+  TABLE_ROW_PREFIX,
+  USER_METDATA_PREFIX,
+} from "../../../db/utils"
+import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
+import fs from "fs"
+import { join } from "path"
+import env from "../../../environment"
+const uuid = require("uuid/v4")
+const tar = require("tar")
+const MemoryStream = require("memorystream")
+
+type ExportOpts = {
+  filter?: any
+  exportPath?: string
+  tar?: boolean
+  excludeRows?: boolean
+}
+
+function tarFilesToTmp(tmpDir: string, files: string[]) {
+  const exportFile = join(budibaseTempDir(), `${uuid()}.tar.gz`)
+  tar.create(
+    {
+      sync: true,
+      gzip: true,
+      file: exportFile,
+      recursive: true,
+      cwd: tmpDir,
+    },
+    files
+  )
+  return exportFile
+}
+
+/**
+ * Exports a DB to either file or a variable (memory).
+ * @param {string} dbName the DB which is to be exported.
+ * @param {object} opts various options for the export, e.g. whether to stream,
+ * a filter function or the name of the export.
+ * @return {*} either a readable stream or a string
+ */
+export async function exportDB(dbName: string, opts: ExportOpts = {}) {
+  return dbCore.doWithDB(dbName, async (db: any) => {
+    // Write the dump to file if required
+    if (opts?.exportPath) {
+      const path = opts?.exportPath
+      const writeStream = fs.createWriteStream(path)
+      await db.dump(writeStream, { filter: opts?.filter })
+      return path
+    } else {
+      // Stringify the dump in memory if required
+      const memStream = new MemoryStream()
+      let appString = ""
+      memStream.on("data", (chunk: any) => {
+        appString += chunk.toString()
+      })
+      await db.dump(memStream, { filter: opts?.filter })
+      return appString
+    }
+  })
+}
+
+function defineFilter(excludeRows?: boolean) {
+  const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX]
+  if (excludeRows) {
+    ids.push(TABLE_ROW_PREFIX)
+  }
+  return (doc: any) =>
+    !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr)
+}
+
+/**
+ * Local utility to back up the database state for an app, excluding global user
+ * data or user relationships.
+ * @param {string} appId The app to back up
+ * @param {object} config Config to send to export DB/attachment export
+ * @returns {*} either a string or a stream of the backup
+ */
+export async function exportApp(appId: string, config?: ExportOpts) {
+  const prodAppId = dbCore.getProdAppID(appId)
+  const appPath = `${prodAppId}/`
+  // export bucket contents
+  let tmpPath
+  if (!env.isTest()) {
+    tmpPath = await retrieveDirectory(ObjectStoreBuckets.APPS, appPath)
+  } else {
+    tmpPath = createTempFolder(uuid())
+  }
+  const downloadedPath = join(tmpPath, appPath)
+  if (fs.existsSync(downloadedPath)) {
+    const allFiles = fs.readdirSync(downloadedPath)
+    for (let file of allFiles) {
+      const path = join(downloadedPath, file)
+      // move out of app directory, simplify structure
+      fs.renameSync(path, join(downloadedPath, "..", file))
+    }
+    // remove the old app directory created by object export
+    fs.rmdirSync(downloadedPath)
+  }
+  // enforce an export of app DB to the tmp path
+  const dbPath = join(tmpPath, DB_EXPORT_FILE)
+  await exportDB(appId, {
+    ...config,
+    filter: defineFilter(config?.excludeRows),
+    exportPath: dbPath,
+  })
+  // if tar requested, return where the tarball is
+  if (config?.tar) {
+    // now the tmpPath contains both the DB export and attachments, tar this
+    const tarPath = tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath))
+    // cleanup the tmp export files as tarball returned
+    fs.rmSync(tmpPath, { recursive: true, force: true })
+    return tarPath
+  }
+  // tar not requested, turn the directory where export is
+  else {
+    return tmpPath
+  }
+}
+
+/**
+ * Export all apps + global DB (if supplied) to a single tarball, this includes
+ * the attachments for each app as well.
+ * @param {object[]} appMetadata The IDs and names of apps to export.
+ * @param {string} globalDbContents The contents of the global DB to export as well.
+ * @return {string} The path to the tarball.
+ */
+export async function exportMultipleApps(
+  appMetadata: { appId: string; name: string }[],
+  globalDbContents?: string
+) {
+  const tmpPath = join(budibaseTempDir(), uuid())
+  fs.mkdirSync(tmpPath)
+  let exportPromises: Promise<void>[] = []
+  // export each app to a directory, then move it into the complete export
+  const exportAndMove = async (appId: string, appName: string) => {
+    const path = await exportApp(appId)
+    await fs.promises.rename(path, join(tmpPath, appName))
+  }
+  for (let metadata of appMetadata) {
+    exportPromises.push(exportAndMove(metadata.appId, metadata.name))
+  }
+  // wait for all exports to finish
+  await Promise.all(exportPromises)
+  // add the global DB contents
+  if (globalDbContents) {
+    fs.writeFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), globalDbContents)
+  }
+  const appNames = appMetadata.map(metadata => metadata.name)
+  const tarPath = tarFilesToTmp(tmpPath, [...appNames, GLOBAL_DB_EXPORT_FILE])
+  // clear up the tmp path now tarball generated
+  fs.rmSync(tmpPath, { recursive: true, force: true })
+  return tarPath
+}
+
+/**
+ * Streams a backup of the database state for an app
+ * @param {string} appId The ID of the app which is to be backed up.
+ * @param {boolean} excludeRows Flag to state whether the export should include data.
+ * @returns {*} a readable stream of the backup which is written in real time
+ */
+export async function streamExportApp(appId: string, excludeRows: boolean) {
+  const tmpPath = await exportApp(appId, { excludeRows, tar: true })
+  return streamFile(tmpPath)
+}