@budibase/server 2.5.6-alpha.9 → 2.5.7

package/src/integrations/redis.ts

@@ -6,7 +6,6 @@ interface RedisConfig {
   port: number
   username: string
   password?: string
-  db?: number
 }
 
 const SCHEMA: Integration = {
@@ -33,12 +32,6 @@ const SCHEMA: Integration = {
       type: "password",
       required: false,
     },
-    db: {
-      type: "number",
-      required: false,
-      display: "DB",
-      default: 0,
-    },
   },
   query: {
     create: {
@@ -95,7 +88,6 @@ class RedisIntegration {
       port: this.config.port,
       username: this.config.username,
       password: this.config.password,
-      db: this.config.db,
     })
   }
 

package/src/integrations/rest.ts

@@ -151,9 +151,6 @@ class RestIntegration implements IntegrationBase {
           data = data[keys[0]]
         }
         raw = rawXml
-      } else if (contentType.includes("application/pdf")) {
-        data = await response.arrayBuffer() // Save PDF as ArrayBuffer
-        raw = Buffer.from(data)
       } else {
         data = await response.text()
         raw = data

package/src/migrations/functions/usageQuotas/syncApps.ts

@@ -9,6 +9,6 @@ export const run = async () => {
 
   // sync app count
   const tenantId = tenancy.getTenantId()
-  console.log(`Syncing app count: ${appCount}`)
+  console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
   await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC)
 }

package/src/migrations/functions/usageQuotas/syncRows.ts

@@ -1,4 +1,4 @@
-import { db as dbCore } from "@budibase/backend-core"
+import { tenancy, db as dbCore } from "@budibase/backend-core"
 import { getUniqueRows } from "../../../utilities/usageQuota/rows"
 import { quotas } from "@budibase/pro"
 import { StaticQuotaName, QuotaUsageType, App } from "@budibase/types"
@@ -18,7 +18,8 @@ export const run = async () => {
   })
 
   // sync row count
-  console.log(`Syncing row count: ${rowCount}`)
+  const tenantId = tenancy.getTenantId()
+  console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
   await quotas.setUsagePerApp(
     counts,
     StaticQuotaName.ROWS,

package/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts

@@ -24,7 +24,7 @@ describe("syncRows", () => {
 
       // app 1
       const app1 = config.app
-      await context.doInAppContext(app1!.appId, async () => {
+      await context.doInAppContext(app1.appId, async () => {
         await config.createTable()
         await config.createRow()
       })
@@ -43,7 +43,7 @@ describe("syncRows", () => {
       usageDoc = await quotas.getQuotaUsage()
       expect(usageDoc.usageQuota.rows).toEqual(3)
       expect(
-        usageDoc.apps?.[dbCore.getProdAppID(app1!.appId)].usageQuota.rows
+        usageDoc.apps?.[dbCore.getProdAppID(app1.appId)].usageQuota.rows
       ).toEqual(1)
       expect(
         usageDoc.apps?.[dbCore.getProdAppID(app2.appId)].usageQuota.rows

package/src/sdk/app/applications/sync.ts

@@ -1,117 +1,6 @@
 import env from "../../../environment"
-import {
-  db as dbCore,
-  context,
-  docUpdates,
-  constants,
-  logging,
-  roles,
-} from "@budibase/backend-core"
-import { User, ContextUser, UserGroup } from "@budibase/types"
-import { sdk as proSdk } from "@budibase/pro"
+import { db as dbCore, context } from "@budibase/backend-core"
 import sdk from "../../"
-import { getGlobalUsers, processUser } from "../../../utilities/global"
-import { generateUserMetadataID, InternalTables } from "../../../db/utils"
-
-type DeletedUser = { _id: string; deleted: boolean }
-
-async function syncUsersToApp(
-  appId: string,
-  users: (User | DeletedUser)[],
-  groups: UserGroup[]
-) {
-  if (!(await dbCore.dbExists(appId))) {
-    return
-  }
-  await context.doInAppContext(appId, async () => {
-    const db = context.getAppDB()
-    for (let user of users) {
-      let ctxUser = user as ContextUser
-      let deletedUser = false
-      const metadataId = generateUserMetadataID(user._id!)
-      if ((user as DeletedUser).deleted) {
-        deletedUser = true
-      }
-
-      // make sure role is correct
-      if (!deletedUser) {
-        ctxUser = await processUser(ctxUser, { appId, groups })
-      }
-      let roleId = ctxUser.roleId
-      if (roleId === roles.BUILTIN_ROLE_IDS.PUBLIC) {
-        roleId = undefined
-      }
-
-      let metadata
-      try {
-        metadata = await db.get(metadataId)
-      } catch (err: any) {
-        if (err.status !== 404) {
-          throw err
-        }
-        // no metadata and user is to be deleted, can skip
-        // no role - user isn't in app anyway
-        if (!roleId) {
-          continue
-        } else if (!deletedUser) {
-          // doesn't exist yet, creating it
-          metadata = {
-            tableId: InternalTables.USER_METADATA,
-          }
-        }
-      }
-
-      // the user doesn't exist, or doesn't have a role anymore
-      // get rid of their metadata
-      if (deletedUser || !roleId) {
-        await db.remove(metadata)
-        continue
-      }
-
-      // assign the roleId for the metadata doc
-      if (roleId) {
-        metadata.roleId = roleId
-      }
-
-      let combined = sdk.users.combineMetadataAndUser(ctxUser, metadata)
-      // if no combined returned, there are no updates to make
-      if (combined) {
-        await db.put(combined)
-      }
-    }
-  })
-}
-
-export async function syncUsersToAllApps(userIds: string[]) {
-  // list of users, if one has been deleted it will be undefined in array
-  const users = (await getGlobalUsers(userIds, {
-    noProcessing: true,
-  })) as User[]
-  const groups = await proSdk.groups.fetch()
-  const finalUsers: (User | DeletedUser)[] = []
-  for (let userId of userIds) {
-    const user = users.find(user => user._id === userId)
-    if (!user) {
-      finalUsers.push({ _id: userId, deleted: true })
-    } else {
-      finalUsers.push(user)
-    }
-  }
-  const devAppIds = await dbCore.getDevAppIDs()
-  let promises = []
-  for (let devAppId of devAppIds) {
-    const prodAppId = dbCore.getProdAppID(devAppId)
-    for (let appId of [prodAppId, devAppId]) {
-      promises.push(syncUsersToApp(appId, finalUsers, groups))
-    }
-  }
-  const resp = await Promise.allSettled(promises)
-  const failed = resp.filter(promise => promise.status === "rejected")
-  if (failed.length > 0) {
-    const reasons = failed.map(fail => (fail as PromiseRejectedResult).reason)
-    logging.logAlert("Failed to sync users to apps", reasons)
-  }
-}
 
 export async function syncApp(
   appId: string,
@@ -134,28 +23,32 @@ export async function syncApp(
   // specific case, want to make sure setup is skipped
   const prodDb = context.getProdAppDB({ skip_setup: true })
   const exists = await prodDb.exists()
+  if (!exists) {
+    // the database doesn't exist. Don't replicate
+    return {
+      message: "App sync not required, app not deployed.",
+    }
+  }
 
+  const replication = new dbCore.Replication({
+    source: prodAppId,
+    target: appId,
+  })
   let error
-  if (exists) {
-    const replication = new dbCore.Replication({
-      source: prodAppId,
-      target: appId,
-    })
-    try {
-      const replOpts = replication.appReplicateOpts()
-      if (opts?.automationOnly) {
-        replOpts.filter = (doc: any) =>
-          doc._id.startsWith(dbCore.DocumentType.AUTOMATION)
-      }
-      await replication.replicate(replOpts)
-    } catch (err) {
-      error = err
-    } finally {
-      await replication.close()
+  try {
+    const replOpts = replication.appReplicateOpts()
+    if (opts?.automationOnly) {
+      replOpts.filter = (doc: any) =>
+        doc._id.startsWith(dbCore.DocumentType.AUTOMATION)
     }
+    await replication.replicate(replOpts)
+  } catch (err) {
+    error = err
+  } finally {
+    await replication.close()
   }
 
-  // sync the users - kept for safe keeping
+  // sync the users
   await sdk.users.syncGlobalUsers()
 
   if (error) {

package/src/sdk/index.ts

@@ -6,7 +6,6 @@ import { default as datasources } from "./app/datasources"
 import { default as queries } from "./app/queries"
 import { default as rows } from "./app/rows"
 import { default as users } from "./users"
-import { default as plugins } from "./plugins"
 
 const sdk = {
   backups,
@@ -17,7 +16,6 @@ const sdk = {
   users,
   datasources,
   queries,
-  plugins,
 }
 
 // default export for TS

package/src/sdk/users/tests/utils.spec.ts

@@ -121,7 +121,38 @@ describe("syncGlobalUsers", () => {
         await syncGlobalUsers()
 
         const metadata = await rawUserMetadata()
-        expect(metadata).toHaveLength(0)
+        expect(metadata).toHaveLength(1)
+      })
+    })
+  })
+
+  it("app users are removed when app is removed from user group", async () => {
+    await config.doInTenant(async () => {
+      const group = await proSdk.groups.save(structures.userGroups.userGroup())
+      const user1 = await config.createUser({ admin: false, builder: false })
+      const user2 = await config.createUser({ admin: false, builder: false })
+      await proSdk.groups.updateGroupApps(group.id, {
+        appsToAdd: [
+          { appId: config.prodAppId!, roleId: roles.BUILTIN_ROLE_IDS.BASIC },
+        ],
+      })
+      await proSdk.groups.addUsers(group.id, [user1._id, user2._id])
+
+      await config.doInContext(config.appId, async () => {
+        await syncGlobalUsers()
+        expect(await rawUserMetadata()).toHaveLength(3)
+
+        await proSdk.groups.removeUsers(group.id, [user1._id])
+        await syncGlobalUsers()
+
+        const metadata = await rawUserMetadata()
+        expect(metadata).toHaveLength(2)
+
+        expect(metadata).not.toContainEqual(
+          expect.objectContaining({
+            _id: db.generateUserMetadataID(user1._id),
+          })
+        )
       })
     })
   })

package/src/sdk/users/utils.ts

@@ -1,13 +1,12 @@
 import { getGlobalUsers } from "../../utilities/global"
 import { context, roles as rolesCore } from "@budibase/backend-core"
 import {
-  getGlobalIDFromUserMetadataID,
   generateUserMetadataID,
   getUserMetadataParams,
   InternalTables,
 } from "../../db/utils"
 import { isEqual } from "lodash"
-import { ContextUser, UserMetadata, User } from "@budibase/types"
+import { ContextUser, UserMetadata } from "@budibase/types"
 
 export function combineMetadataAndUser(
   user: ContextUser,
@@ -38,10 +37,6 @@ export function combineMetadataAndUser(
   if (found) {
     newDoc._rev = found._rev
   }
-  // clear fields that shouldn't be in metadata
-  delete newDoc.password
-  delete newDoc.forceResetPassword
-  delete newDoc.roles
   if (found == null || !isEqual(newDoc, found)) {
     return {
       ...found,
@@ -65,9 +60,10 @@ export async function rawUserMetadata() {
 export async function syncGlobalUsers() {
   // sync user metadata
   const db = context.getAppDB()
-  const resp = await Promise.all([getGlobalUsers(), rawUserMetadata()])
-  const users = resp[0] as User[]
-  const metadata = resp[1] as UserMetadata[]
+  const [users, metadata] = await Promise.all([
+    getGlobalUsers(),
+    rawUserMetadata(),
+  ])
   const toWrite = []
   for (let user of users) {
     const combined = combineMetadataAndUser(user, metadata)
@@ -75,19 +71,5 @@ export async function syncGlobalUsers() {
       toWrite.push(combined)
     }
   }
-  let foundEmails: string[] = []
-  for (let data of metadata) {
-    if (!data._id) {
-      continue
-    }
-    const alreadyExisting = data.email && foundEmails.indexOf(data.email) !== -1
-    const globalId = getGlobalIDFromUserMetadataID(data._id)
-    if (!users.find(user => user._id === globalId) || alreadyExisting) {
-      toWrite.push({ ...data, _deleted: true })
-    }
-    if (data.email) {
-      foundEmails.push(data.email)
-    }
-  }
   await db.bulkDocs(toWrite)
 }

package/src/startup.ts

@@ -10,16 +10,19 @@ import fs from "fs"
 import { watch } from "./watch"
 import * as automations from "./automations"
 import * as fileSystem from "./utilities/fileSystem"
-import { default as eventEmitter, init as eventInit } from "./events"
+import eventEmitter from "./events"
 import * as migrations from "./migrations"
 import * as bullboard from "./automations/bullboard"
 import * as pro from "@budibase/pro"
 import * as api from "./api"
 import sdk from "./sdk"
+const pino = require("koa-pino-logger")
 
 let STARTUP_RAN = false
 
 async function initRoutes(app: any) {
+  app.use(pino(logging.pinoSettings()))
+
   if (!env.isTest()) {
     const plugin = await bullboard.init()
     app.use(plugin)
@@ -45,10 +48,8 @@ async function initPro() {
 }
 
 function shutdown(server?: any) {
-  if (server) {
-    server.close()
-    server.destroy()
-  }
+  server.close()
+  server.destroy()
 }
 
 export async function startup(app?: any, server?: any) {
@@ -63,7 +64,6 @@ export async function startup(app?: any, server?: any) {
   eventEmitter.emitPort(env.PORT)
   fileSystem.init()
   await redis.init()
-  eventInit()
 
   // run migrations on startup if not done via http
   // not recommended in a clustered environment
@@ -72,39 +72,11 @@ export async function startup(app?: any, server?: any) {
       await migrations.migrate()
     } catch (e) {
       logging.logAlert("Error performing migrations. Exiting.", e)
-      shutdown(server)
+      shutdown()
     }
   }
 
-  // monitor plugin directory if required
-  if (
-    env.SELF_HOSTED &&
-    !env.MULTI_TENANCY &&
-    env.PLUGINS_DIR &&
-    fs.existsSync(env.PLUGINS_DIR)
-  ) {
-    watch()
-  }
-
-  // check for version updates
-  await installation.checkInstallVersion()
-
-  // get the references to the queue promises, don't await as
-  // they will never end, unless the processing stops
-  let queuePromises = []
-  // configure events to use the pro audit log write
-  // can't integrate directly into backend-core due to cyclic issues
-  queuePromises.push(events.processors.init(pro.sdk.auditLogs.write))
-  queuePromises.push(automations.init())
-  queuePromises.push(initPro())
-  if (app) {
-    // bring routes online as final step once everything ready
-    await initRoutes(app)
-  }
-
   // check and create admin user if required
-  // this must be run after the api has been initialised due to
-  // the app user sync
   if (
     env.SELF_HOSTED &&
     !env.MULTI_TENANCY &&
@@ -131,8 +103,34 @@ export async function startup(app?: any, server?: any) {
         )
       } catch (e) {
         logging.logAlert("Error creating initial admin user. Exiting.", e)
-        shutdown(server)
+        shutdown()
       }
     }
   }
+
+  // monitor plugin directory if required
+  if (
+    env.SELF_HOSTED &&
+    !env.MULTI_TENANCY &&
+    env.PLUGINS_DIR &&
+    fs.existsSync(env.PLUGINS_DIR)
+  ) {
+    watch()
+  }
+
+  // check for version updates
+  await installation.checkInstallVersion()
+
+  // get the references to the queue promises, don't await as
+  // they will never end, unless the processing stops
+  let queuePromises = []
+  // configure events to use the pro audit log write
+  // can't integrate directly into backend-core due to cyclic issues
+  queuePromises.push(events.processors.init(pro.sdk.auditLogs.write))
+  queuePromises.push(automations.init())
+  queuePromises.push(initPro())
+  if (app) {
+    // bring routes online as final step once everything ready
+    await initRoutes(app)
+  }
 }

package/src/tests/jestEnv.ts

@@ -6,6 +6,7 @@ process.env.MULTI_TENANCY = "1"
 // @ts-ignore
 process.env.BUDIBASE_DIR = tmpdir("budibase-unittests")
 process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
+process.env.ENABLE_4XX_HTTP_LOGGING = "0"
 process.env.MOCK_REDIS = "1"
 process.env.PLATFORM_URL = "http://localhost:10000"
 process.env.REDIS_PASSWORD = "budibase"

package/src/tests/jestSetup.ts

@@ -1,3 +1,4 @@
+import "./logging"
 import env from "../environment"
 import { env as coreEnv, timers } from "@budibase/backend-core"
 import { testContainerUtils } from "@budibase/backend-core/tests"

package/src/tests/logging.ts

@@ -0,0 +1,34 @@
+export enum LogLevel {
+  TRACE = "trace",
+  DEBUG = "debug",
+  INFO = "info",
+  WARN = "warn",
+  ERROR = "error",
+}
+
+const LOG_INDEX: { [key in LogLevel]: number } = {
+  [LogLevel.TRACE]: 1,
+  [LogLevel.DEBUG]: 2,
+  [LogLevel.INFO]: 3,
+  [LogLevel.WARN]: 4,
+  [LogLevel.ERROR]: 5,
+}
+
+const setIndex = LOG_INDEX[process.env.LOG_LEVEL as LogLevel]
+
+if (setIndex > LOG_INDEX.trace) {
+  global.console.trace = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.debug) {
+  global.console.debug = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.info) {
+  global.console.info = jest.fn()
+  global.console.log = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.warn) {
+  global.console.warn = jest.fn()
+}

package/src/tests/utilities/TestConfiguration.ts

@@ -49,7 +49,6 @@ import {
   SearchFilters,
   UserRoles,
 } from "@budibase/types"
-import { BUILTIN_ROLE_IDS } from "@budibase/backend-core/src/security/roles"
 
 type DefaultUserValues = {
   globalUserId: string
@@ -307,33 +306,6 @@ class TestConfiguration {
     }
   }
 
-  async createGroup(roleId: string = BUILTIN_ROLE_IDS.BASIC) {
-    return context.doInTenant(this.tenantId!, async () => {
-      const baseGroup = structures.userGroups.userGroup()
-      baseGroup.roles = {
-        [this.prodAppId]: roleId,
-      }
-      const { id, rev } = await pro.sdk.groups.save(baseGroup)
-      return {
-        _id: id,
-        _rev: rev,
-        ...baseGroup,
-      }
-    })
-  }
-
-  async addUserToGroup(groupId: string, userId: string) {
-    return context.doInTenant(this.tenantId!, async () => {
-      await pro.sdk.groups.addUsers(groupId, [userId])
-    })
-  }
-
-  async removeUserFromGroup(groupId: string, userId: string) {
-    return context.doInTenant(this.tenantId!, async () => {
-      await pro.sdk.groups.removeUsers(groupId, [userId])
-    })
-  }
-
   async login({ roleId, userId, builder, prodApp = false }: any = {}) {
     const appId = prodApp ? this.prodAppId : this.appId
     return context.doInAppContext(appId, async () => {

package/src/tests/utilities/structures.ts

@@ -1,22 +1,18 @@
-import { permissions, roles, utils } from "@budibase/backend-core"
+import { permissions, roles } from "@budibase/backend-core"
 import { createHomeScreen } from "../../constants/screens"
 import { EMPTY_LAYOUT } from "../../constants/layouts"
 import { cloneDeep } from "lodash/fp"
-import {
-  BUILTIN_ACTION_DEFINITIONS,
-  TRIGGER_DEFINITIONS,
-} from "../../automations"
+import { ACTION_DEFINITIONS, TRIGGER_DEFINITIONS } from "../../automations"
 import {
   Automation,
   AutomationActionStepId,
-  AutomationStep,
-  AutomationStepType,
-  AutomationTrigger,
   AutomationTriggerStepId,
   Datasource,
   SourceName,
 } from "@budibase/types"
 
+const { v4: uuidv4 } = require("uuid")
+
 export function basicTable() {
   return {
     name: "TestTable",
@@ -75,19 +71,19 @@ export function view(tableId: string) {
 }
 
 export function automationStep(
-  actionDefinition = BUILTIN_ACTION_DEFINITIONS.CREATE_ROW
-): AutomationStep {
+  actionDefinition = ACTION_DEFINITIONS.CREATE_ROW
+) {
   return {
-    id: utils.newid(),
+    id: uuidv4(),
     ...actionDefinition,
   }
 }
 
 export function automationTrigger(
   triggerDefinition = TRIGGER_DEFINITIONS.ROW_SAVED
-): AutomationTrigger {
+) {
   return {
-    id: utils.newid(),
+    id: uuidv4(),
     ...triggerDefinition,
   }
 }
@@ -110,7 +106,7 @@ export function newAutomation({ steps, trigger }: any = {}) {
   return automation
 }
 
-export function basicAutomation(appId?: string): Automation {
+export function basicAutomation(appId?: string) {
   return {
     name: "My Automation",
     screenId: "kasdkfldsafkl",
@@ -123,22 +119,18 @@ export function basicAutomation(appId?: string): Automation {
         tagline: "test",
         icon: "test",
         description: "test",
-        type: AutomationStepType.TRIGGER,
+        type: "trigger",
         id: "test",
         inputs: {},
         schema: {
-          inputs: {
-            properties: {},
-          },
-          outputs: {
-            properties: {},
-          },
+          inputs: {},
+          outputs: {},
         },
       },
       steps: [],
     },
     type: "automation",
-    appId: appId!,
+    appId,
   }
 }
 
@@ -162,7 +154,7 @@ export function loopAutomation(tableId: string, loopOpts?: any): Automation {
           inputs: {
             tableId,
           },
-          schema: BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS.schema,
+          schema: ACTION_DEFINITIONS.QUERY_ROWS.schema,
         },
         {
           id: "c",
@@ -171,7 +163,7 @@ export function loopAutomation(tableId: string, loopOpts?: any): Automation {
           internal: true,
           inputs: loopOpts,
           blockToLoop: "d",
-          schema: BUILTIN_ACTION_DEFINITIONS.LOOP.schema,
+          schema: ACTION_DEFINITIONS.LOOP.schema,
         },
         {
           id: "d",
@@ -181,7 +173,7 @@ export function loopAutomation(tableId: string, loopOpts?: any): Automation {
           inputs: {
             text: "log statement",
           },
-          schema: BUILTIN_ACTION_DEFINITIONS.SERVER_LOG.schema,
+          schema: ACTION_DEFINITIONS.SERVER_LOG.schema,
         },
       ],
       trigger: {