@budibase/server 2.5.9 → 2.5.10-alpha.1

package/src/integrations/index.ts

@@ -14,11 +14,11 @@ import firebase from "./firebase"
 import redis from "./redis"
 import snowflake from "./snowflake"
 import oracle from "./oracle"
-import { getPlugins } from "../api/controllers/plugin"
 import { SourceName, Integration, PluginType } from "@budibase/types"
 import { getDatasourcePlugin } from "../utilities/fileSystem"
 import env from "../environment"
 import { cloneDeep } from "lodash"
+import sdk from "../sdk"
 
 const DEFINITIONS: { [key: string]: Integration } = {
   [SourceName.POSTGRES]: postgres.schema,
@@ -79,7 +79,7 @@ export async function getDefinition(source: SourceName): Promise<Integration> {
 export async function getDefinitions() {
   const pluginSchemas: { [key: string]: Integration } = {}
   if (env.SELF_HOSTED) {
-    const plugins = await getPlugins(PluginType.DATASOURCE)
+    const plugins = await sdk.plugins.fetch(PluginType.DATASOURCE)
     // extract the actual schema from each custom
     for (let plugin of plugins) {
       const sourceId = plugin.name
@@ -103,7 +103,7 @@ export async function getIntegration(integration: string) {
     return INTEGRATIONS[integration]
   }
   if (env.SELF_HOSTED) {
-    const plugins = await getPlugins(PluginType.DATASOURCE)
+    const plugins = await sdk.plugins.fetch(PluginType.DATASOURCE)
     for (let plugin of plugins) {
       if (plugin.name === integration) {
         // need to use commonJS require due to its dynamic runtime nature

package/src/integrations/microsoftSqlServer.ts

@@ -243,11 +243,14 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
         if (typeof name !== "string") {
           continue
         }
+        const hasDefault = def.COLUMN_DEFAULT
+        const isAuto = !!autoColumns.find(col => col === name)
+        const required = !!requiredColumns.find(col => col === name)
         schema[name] = {
-          autocolumn: !!autoColumns.find(col => col === name),
+          autocolumn: isAuto,
           name: name,
           constraints: {
-            presence: requiredColumns.find(col => col === name),
+            presence: required && !isAuto && !hasDefault,
           },
           ...convertSqlType(def.DATA_TYPE),
           externalType: def.DATA_TYPE,

package/src/integrations/mysql.ts

@@ -229,13 +229,15 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
           if (column.Key === "PRI" && primaryKeys.indexOf(column.Key) === -1) {
             primaryKeys.push(columnName)
           }
-          const constraints = {
-            presence: column.Null !== "YES",
-          }
+          const hasDefault = column.Default != null
           const isAuto: boolean =
             typeof column.Extra === "string" &&
             (column.Extra === "auto_increment" ||
               column.Extra.toLowerCase().includes("generated"))
+          const required = column.Null !== "YES"
+          const constraints = {
+            presence: required && !isAuto && !hasDefault,
+          }
           schema[columnName] = {
             name: columnName,
             autocolumn: isAuto,

package/src/integrations/postgres.ts

@@ -262,15 +262,17 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
           column.identity_start ||
           column.identity_increment
         )
-        const constraints = {
-          presence: column.is_nullable === "NO",
-        }
-        const hasDefault =
+        const hasDefault = column.column_default != null
+        const hasNextVal =
           typeof column.column_default === "string" &&
           column.column_default.startsWith("nextval")
         const isGenerated =
           column.is_generated && column.is_generated !== "NEVER"
-        const isAuto: boolean = hasDefault || identity || isGenerated
+        const isAuto: boolean = hasNextVal || identity || isGenerated
+        const required = column.is_nullable === "NO"
+        const constraints = {
+          presence: required && !hasDefault && !isGenerated,
+        }
         tables[tableName].schema[columnName] = {
           autocolumn: isAuto,
           name: columnName,

package/src/integrations/redis.ts

@@ -6,6 +6,7 @@ interface RedisConfig {
   port: number
   username: string
   password?: string
+  db?: number
 }
 
 const SCHEMA: Integration = {
@@ -32,6 +33,12 @@ const SCHEMA: Integration = {
       type: "password",
       required: false,
     },
+    db: {
+      type: "number",
+      required: false,
+      display: "DB",
+      default: 0,
+    },
   },
   query: {
     create: {
@@ -88,6 +95,7 @@ class RedisIntegration {
       port: this.config.port,
       username: this.config.username,
       password: this.config.password,
+      db: this.config.db,
     })
   }
 

package/src/integrations/rest.ts

@@ -151,6 +151,9 @@ class RestIntegration implements IntegrationBase {
           data = data[keys[0]]
         }
         raw = rawXml
+      } else if (contentType.includes("application/pdf")) {
+        data = await response.arrayBuffer() // Save PDF as ArrayBuffer
+        raw = Buffer.from(data)
       } else {
         data = await response.text()
         raw = data

package/src/migrations/functions/syncQuotas.ts

@@ -2,6 +2,7 @@ import { runQuotaMigration } from "./usageQuotas"
 import * as syncApps from "./usageQuotas/syncApps"
 import * as syncRows from "./usageQuotas/syncRows"
 import * as syncPlugins from "./usageQuotas/syncPlugins"
+import * as syncUsers from "./usageQuotas/syncUsers"
 
 /**
  * Synchronise quotas to the state of the db.
@@ -11,5 +12,6 @@ export const run = async () => {
     await syncApps.run()
     await syncRows.run()
     await syncPlugins.run()
+    await syncUsers.run()
   })
 }

package/src/migrations/functions/usageQuotas/syncApps.ts

@@ -1,4 +1,4 @@
-import { tenancy, db as dbCore } from "@budibase/backend-core"
+import { db as dbCore } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
 
@@ -8,7 +8,6 @@ export const run = async () => {
   const appCount = devApps ? devApps.length : 0
 
   // sync app count
-  const tenantId = tenancy.getTenantId()
-  console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
+  console.log(`Syncing app count: ${appCount}`)
   await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC)
 }

package/src/migrations/functions/usageQuotas/syncRows.ts

@@ -1,4 +1,4 @@
-import { tenancy, db as dbCore } from "@budibase/backend-core"
+import { db as dbCore } from "@budibase/backend-core"
 import { getUniqueRows } from "../../../utilities/usageQuota/rows"
 import { quotas } from "@budibase/pro"
 import { StaticQuotaName, QuotaUsageType, App } from "@budibase/types"
@@ -18,8 +18,7 @@ export const run = async () => {
   })
 
   // sync row count
-  const tenantId = tenancy.getTenantId()
-  console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
+  console.log(`Syncing row count: ${rowCount}`)
   await quotas.setUsagePerApp(
     counts,
     StaticQuotaName.ROWS,

package/src/migrations/functions/usageQuotas/syncUsers.ts

@@ -0,0 +1,9 @@
+import { users } from "@budibase/backend-core"
+import { quotas } from "@budibase/pro"
+import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
+
+export const run = async () => {
+  const userCount = await users.getUserCount()
+  console.log(`Syncing user count: ${userCount}`)
+  await quotas.setUsage(userCount, StaticQuotaName.USERS, QuotaUsageType.STATIC)
+}

package/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts

@@ -24,7 +24,7 @@ describe("syncRows", () => {
 
       // app 1
       const app1 = config.app
-      await context.doInAppContext(app1.appId, async () => {
+      await context.doInAppContext(app1!.appId, async () => {
         await config.createTable()
         await config.createRow()
       })
@@ -43,7 +43,7 @@ describe("syncRows", () => {
       usageDoc = await quotas.getQuotaUsage()
       expect(usageDoc.usageQuota.rows).toEqual(3)
       expect(
-        usageDoc.apps?.[dbCore.getProdAppID(app1.appId)].usageQuota.rows
+        usageDoc.apps?.[dbCore.getProdAppID(app1!.appId)].usageQuota.rows
       ).toEqual(1)
       expect(
         usageDoc.apps?.[dbCore.getProdAppID(app2.appId)].usageQuota.rows

package/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts

@@ -0,0 +1,26 @@
+import TestConfig from "../../../../tests/utilities/TestConfiguration"
+import * as syncUsers from "../syncUsers"
+import { quotas } from "@budibase/pro"
+
+describe("syncUsers", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)
+
+  it("syncs users", async () => {
+    return config.doInContext(null, async () => {
+      await config.createUser()
+
+      await syncUsers.run()
+
+      const usageDoc = await quotas.getQuotaUsage()
+      // default + additional user
+      const userCount = 2
+      expect(usageDoc.usageQuota.users).toBe(userCount)
+    })
+  })
+})

package/src/migrations/index.ts

@@ -11,6 +11,7 @@ import env from "../environment"
 // migration functions
 import * as userEmailViewCasing from "./functions/userEmailViewCasing"
 import * as syncQuotas from "./functions/syncQuotas"
+import * as syncUsers from "./functions/usageQuotas/syncUsers"
 import * as appUrls from "./functions/appUrls"
 import * as tableSettings from "./functions/tableSettings"
 import * as backfill from "./functions/backfill"

package/src/sdk/app/applications/sync.ts

@@ -1,6 +1,117 @@
 import env from "../../../environment"
-import { db as dbCore, context } from "@budibase/backend-core"
+import {
+  db as dbCore,
+  context,
+  docUpdates,
+  constants,
+  logging,
+  roles,
+} from "@budibase/backend-core"
+import { User, ContextUser, UserGroup } from "@budibase/types"
+import { sdk as proSdk } from "@budibase/pro"
 import sdk from "../../"
+import { getGlobalUsers, processUser } from "../../../utilities/global"
+import { generateUserMetadataID, InternalTables } from "../../../db/utils"
+
+type DeletedUser = { _id: string; deleted: boolean }
+
+async function syncUsersToApp(
+  appId: string,
+  users: (User | DeletedUser)[],
+  groups: UserGroup[]
+) {
+  if (!(await dbCore.dbExists(appId))) {
+    return
+  }
+  await context.doInAppContext(appId, async () => {
+    const db = context.getAppDB()
+    for (let user of users) {
+      let ctxUser = user as ContextUser
+      let deletedUser = false
+      const metadataId = generateUserMetadataID(user._id!)
+      if ((user as DeletedUser).deleted) {
+        deletedUser = true
+      }
+
+      // make sure role is correct
+      if (!deletedUser) {
+        ctxUser = await processUser(ctxUser, { appId, groups })
+      }
+      let roleId = ctxUser.roleId
+      if (roleId === roles.BUILTIN_ROLE_IDS.PUBLIC) {
+        roleId = undefined
+      }
+
+      let metadata
+      try {
+        metadata = await db.get(metadataId)
+      } catch (err: any) {
+        if (err.status !== 404) {
+          throw err
+        }
+        // no metadata and user is to be deleted, can skip
+        // no role - user isn't in app anyway
+        if (!roleId) {
+          continue
+        } else if (!deletedUser) {
+          // doesn't exist yet, creating it
+          metadata = {
+            tableId: InternalTables.USER_METADATA,
+          }
+        }
+      }
+
+      // the user doesn't exist, or doesn't have a role anymore
+      // get rid of their metadata
+      if (deletedUser || !roleId) {
+        await db.remove(metadata)
+        continue
+      }
+
+      // assign the roleId for the metadata doc
+      if (roleId) {
+        metadata.roleId = roleId
+      }
+
+      let combined = sdk.users.combineMetadataAndUser(ctxUser, metadata)
+      // if no combined returned, there are no updates to make
+      if (combined) {
+        await db.put(combined)
+      }
+    }
+  })
+}
+
+export async function syncUsersToAllApps(userIds: string[]) {
+  // list of users, if one has been deleted it will be undefined in array
+  const users = (await getGlobalUsers(userIds, {
+    noProcessing: true,
+  })) as User[]
+  const groups = await proSdk.groups.fetch()
+  const finalUsers: (User | DeletedUser)[] = []
+  for (let userId of userIds) {
+    const user = users.find(user => user._id === userId)
+    if (!user) {
+      finalUsers.push({ _id: userId, deleted: true })
+    } else {
+      finalUsers.push(user)
+    }
+  }
+  const devAppIds = await dbCore.getDevAppIDs()
+  let promises = []
+  for (let devAppId of devAppIds) {
+    const prodAppId = dbCore.getProdAppID(devAppId)
+    for (let appId of [prodAppId, devAppId]) {
+      promises.push(syncUsersToApp(appId, finalUsers, groups))
+    }
+  }
+  const resp = await Promise.allSettled(promises)
+  const failed = resp.filter(promise => promise.status === "rejected")
+  if (failed.length > 0) {
+    const reasons = failed.map(fail => (fail as PromiseRejectedResult).reason)
+    logging.logAlert("Failed to sync users to apps", reasons)
+  }
+}
 
 export async function syncApp(
   appId: string,
@@ -23,32 +134,28 @@ export async function syncApp(
   // specific case, want to make sure setup is skipped
   const prodDb = context.getProdAppDB({ skip_setup: true })
   const exists = await prodDb.exists()
-  if (!exists) {
-    // the database doesn't exist. Don't replicate
-    return {
-      message: "App sync not required, app not deployed.",
-    }
-  }
 
-  const replication = new dbCore.Replication({
-    source: prodAppId,
-    target: appId,
-  })
   let error
-  try {
-    const replOpts = replication.appReplicateOpts()
-    if (opts?.automationOnly) {
-      replOpts.filter = (doc: any) =>
-        doc._id.startsWith(dbCore.DocumentType.AUTOMATION)
+  if (exists) {
+    const replication = new dbCore.Replication({
+      source: prodAppId,
+      target: appId,
+    })
+    try {
+      const replOpts = replication.appReplicateOpts()
+      if (opts?.automationOnly) {
+        replOpts.filter = (doc: any) =>
+          doc._id.startsWith(dbCore.DocumentType.AUTOMATION)
+      }
+      await replication.replicate(replOpts)
+    } catch (err) {
+      error = err
+    } finally {
+      await replication.close()
     }
-    await replication.replicate(replOpts)
-  } catch (err) {
-    error = err
-  } finally {
-    await replication.close()
   }
 
-  // sync the users
+  // sync the users - kept for safe keeping
   await sdk.users.syncGlobalUsers()
 
   if (error) {

package/src/sdk/app/applications/tests/sync.spec.ts

@@ -0,0 +1,137 @@
+import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
+import { events, context, roles, constants } from "@budibase/backend-core"
+import { init } from "../../../../events"
+import { rawUserMetadata } from "../../../users/utils"
+import EventEmitter from "events"
+import { UserGroup, UserMetadata, UserRoles, User } from "@budibase/types"
+
+const config = new TestConfiguration()
+let app, group: UserGroup, groupUser: User
+const ROLE_ID = roles.BUILTIN_ROLE_IDS.BASIC
+
+const emitter = new EventEmitter()
+
+function updateCb(docId: string) {
+  const isGroup = docId.startsWith(constants.DocumentType.GROUP)
+  if (isGroup) {
+    emitter.emit("update-group")
+  } else {
+    emitter.emit("update-user")
+  }
+}
+
+init(updateCb)
+
+function waitForUpdate(opts: { group?: boolean }) {
+  return new Promise<void>((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      reject()
+    }, 5000)
+    const event = opts?.group ? "update-group" : "update-user"
+    emitter.on(event, () => {
+      clearTimeout(timeout)
+      resolve()
+    })
+  })
+}
+
+beforeAll(async () => {
+  app = await config.init("syncApp")
+})
+
+async function createUser(email: string, roles: UserRoles, builder?: boolean) {
+  const user = await config.createUser({
+    email,
+    roles,
+    builder: builder || false,
+    admin: false,
+  })
+  await context.doInContext(config.appId!, async () => {
+    await events.user.created(user)
+  })
+  return user
+}
+
+async function removeUserRole(user: User) {
+  const final = await config.globalUser({
+    ...user,
+    id: user._id,
+    roles: {},
+    builder: false,
+    admin: false,
+  })
+  await context.doInContext(config.appId!, async () => {
+    await events.user.updated(final)
+  })
+}
+
+async function createGroupAndUser(email: string) {
+  groupUser = await config.createUser({
+    email,
+    roles: {},
+    builder: false,
+    admin: false,
+  })
+  group = await config.createGroup()
+  await config.addUserToGroup(group._id!, groupUser._id!)
+}
+
+async function removeUserFromGroup() {
+  await config.removeUserFromGroup(group._id!, groupUser._id!)
+  return context.doInContext(config.appId!, async () => {
+    await events.user.updated(groupUser)
+  })
+}
+
+async function getUserMetadata(): Promise<UserMetadata[]> {
+  return context.doInContext(config.appId!, async () => {
+    return await rawUserMetadata()
+  })
+}
+
+function buildRoles() {
+  return { [config.prodAppId!]: ROLE_ID }
+}
+
+describe("app user/group sync", () => {
+  const groupEmail = "test2@test.com",
+    normalEmail = "test@test.com"
+  async function checkEmail(
+    email: string,
+    opts?: { group?: boolean; notFound?: boolean }
+  ) {
+    await waitForUpdate(opts || {})
+    const metadata = await getUserMetadata()
+    const found = metadata.find(data => data.email === email)
+    if (opts?.notFound) {
+      expect(found).toBeUndefined()
+    } else {
+      expect(found).toBeDefined()
+    }
+  }
+
+  it("should be able to sync a new user, add then remove", async () => {
+    const user = await createUser(normalEmail, buildRoles())
+    await checkEmail(normalEmail)
+    await removeUserRole(user)
+    await checkEmail(normalEmail, { notFound: true })
+  })
+
+  it("should be able to sync a group", async () => {
+    await createGroupAndUser(groupEmail)
+    await checkEmail(groupEmail, { group: true })
+  })
+
+  it("should be able to remove user from group", async () => {
+    if (!group) {
+      await createGroupAndUser(groupEmail)
+    }
+    await removeUserFromGroup()
+    await checkEmail(groupEmail, { notFound: true })
+  })
+
+  it("should be able to handle builder users", async () => {
+    await createUser("test3@test.com", {}, true)
+    await checkEmail("test3@test.com")
+  })
+})

package/src/sdk/app/backups/exports.ts

@@ -3,6 +3,7 @@ import { budibaseTempDir } from "../../../utilities/budibaseDir"
 import { streamFile, createTempFolder } from "../../../utilities/fileSystem"
 import { ObjectStoreBuckets } from "../../../constants"
 import {
+  AUTOMATION_LOG_PREFIX,
   LINK_USER_METADATA_PREFIX,
   TABLE_ROW_PREFIX,
   USER_METDATA_PREFIX,
@@ -20,11 +21,15 @@ const uuid = require("uuid/v4")
 const tar = require("tar")
 const MemoryStream = require("memorystream")
 
-type ExportOpts = {
+interface DBDumpOpts {
   filter?: any
   exportPath?: string
+}
+
+interface ExportOpts extends DBDumpOpts {
   tar?: boolean
   excludeRows?: boolean
+  excludeLogs?: boolean
 }
 
 function tarFilesToTmp(tmpDir: string, files: string[]) {
@@ -49,7 +54,7 @@ function tarFilesToTmp(tmpDir: string, files: string[]) {
  * a filter function or the name of the export.
  * @return {*} either a readable stream or a string
  */
-export async function exportDB(dbName: string, opts: ExportOpts = {}) {
+export async function exportDB(dbName: string, opts: DBDumpOpts = {}) {
   const exportOpts = {
     filter: opts?.filter,
     batch_size: 1000,
@@ -76,11 +81,14 @@ export async function exportDB(dbName: string, opts: ExportOpts = {}) {
   })
 }
 
-function defineFilter(excludeRows?: boolean) {
+function defineFilter(excludeRows?: boolean, excludeLogs?: boolean) {
   const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX]
   if (excludeRows) {
     ids.push(TABLE_ROW_PREFIX)
   }
+  if (excludeLogs) {
+    ids.push(AUTOMATION_LOG_PREFIX)
+  }
   return (doc: any) =>
     !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr)
 }
@@ -130,8 +138,7 @@ export async function exportApp(appId: string, config?: ExportOpts) {
   // enforce an export of app DB to the tmp path
   const dbPath = join(tmpPath, DB_EXPORT_FILE)
   await exportDB(appId, {
-    ...config,
-    filter: defineFilter(config?.excludeRows),
+    filter: defineFilter(config?.excludeRows, config?.excludeLogs),
     exportPath: dbPath,
   })
   // if tar requested, return where the tarball is
@@ -148,41 +155,6 @@ export async function exportApp(appId: string, config?: ExportOpts) {
   }
 }
 
-/**
- * Export all apps + global DB (if supplied) to a single tarball, this includes
- * the attachments for each app as well.
- * @param {object[]} appMetadata The IDs and names of apps to export.
- * @param {string} globalDbContents The contents of the global DB to export as well.
- * @return {string} The path to the tarball.
- */
-export async function exportMultipleApps(
-  appMetadata: { appId: string; name: string }[],
-  globalDbContents?: string
-) {
-  const tmpPath = join(budibaseTempDir(), uuid())
-  fs.mkdirSync(tmpPath)
-  let exportPromises: Promise<void>[] = []
-  // export each app to a directory, then move it into the complete export
-  const exportAndMove = async (appId: string, appName: string) => {
-    const path = await exportApp(appId)
-    await fs.promises.rename(path, join(tmpPath, appName))
-  }
-  for (let metadata of appMetadata) {
-    exportPromises.push(exportAndMove(metadata.appId, metadata.name))
-  }
-  // wait for all exports to finish
-  await Promise.all(exportPromises)
-  // add the global DB contents
-  if (globalDbContents) {
-    fs.writeFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), globalDbContents)
-  }
-  const appNames = appMetadata.map(metadata => metadata.name)
-  const tarPath = tarFilesToTmp(tmpPath, [...appNames, GLOBAL_DB_EXPORT_FILE])
-  // clear up the tmp path now tarball generated
-  fs.rmSync(tmpPath, { recursive: true, force: true })
-  return tarPath
-}
-
 /**
  * Streams a backup of the database state for an app
  * @param {string} appId The ID of the app which is to be backed up.
@@ -190,6 +162,10 @@ export async function exportMultipleApps(
  * @returns {*} a readable stream of the backup which is written in real time
  */
 export async function streamExportApp(appId: string, excludeRows: boolean) {
-  const tmpPath = await exportApp(appId, { excludeRows, tar: true })
+  const tmpPath = await exportApp(appId, {
+    excludeRows,
+    excludeLogs: true,
+    tar: true,
+  })
   return streamFile(tmpPath)
 }

package/src/sdk/index.ts

@@ -6,6 +6,7 @@ import { default as datasources } from "./app/datasources"
 import { default as queries } from "./app/queries"
 import { default as rows } from "./app/rows"
 import { default as users } from "./users"
+import { default as plugins } from "./plugins"
 
 const sdk = {
   backups,
@@ -16,6 +17,7 @@ const sdk = {
   users,
   datasources,
   queries,
+  plugins,
 }
 
 // default export for TS

package/src/sdk/plugins/index.ts

@@ -0,0 +1,5 @@
+import * as plugins from "./plugins"
+
+export default {
+  ...plugins,
+}