@budibase/server 2.5.9 → 2.5.10-alpha.1

package/src/sdk/plugins/plugins.ts

@@ -0,0 +1,41 @@
+import { FileType, Plugin, PluginSource, PluginType } from "@budibase/types"
+import {
+  db as dbCore,
+  objectStore,
+  plugins as pluginCore,
+  tenancy,
+} from "@budibase/backend-core"
+import { fileUpload } from "../../api/controllers/plugin/file"
+import env from "../../environment"
+import { clientAppSocket } from "../../websockets"
+import { sdk as pro } from "@budibase/pro"
+
+export async function fetch(type?: PluginType) {
+  const db = tenancy.getGlobalDB()
+  const response = await db.allDocs(
+    dbCore.getPluginParams(null, {
+      include_docs: true,
+    })
+  )
+  let plugins = response.rows.map((row: any) => row.doc) as Plugin[]
+  plugins = objectStore.enrichPluginURLs(plugins)
+  if (type) {
+    return plugins.filter((plugin: Plugin) => plugin.schema?.type === type)
+  } else {
+    return plugins
+  }
+}
+
+export async function processUploaded(plugin: FileType, source?: PluginSource) {
+  const { metadata, directory } = await fileUpload(plugin)
+  pluginCore.validate(metadata?.schema)
+
+  // Only allow components in cloud
+  if (!env.SELF_HOSTED && metadata?.schema?.type !== PluginType.COMPONENT) {
+    throw new Error("Only component plugins are supported outside of self-host")
+  }
+
+  const doc = await pro.plugins.storePlugin(metadata, directory, source)
+  clientAppSocket.emit("plugin-update", { name: doc.name, hash: doc.hash })
+  return doc
+}

package/src/sdk/users/tests/utils.spec.ts

@@ -121,38 +121,7 @@ describe("syncGlobalUsers", () => {
         await syncGlobalUsers()
 
         const metadata = await rawUserMetadata()
-        expect(metadata).toHaveLength(1)
-      })
-    })
-  })
-
-  it("app users are removed when app is removed from user group", async () => {
-    await config.doInTenant(async () => {
-      const group = await proSdk.groups.save(structures.userGroups.userGroup())
-      const user1 = await config.createUser({ admin: false, builder: false })
-      const user2 = await config.createUser({ admin: false, builder: false })
-      await proSdk.groups.updateGroupApps(group.id, {
-        appsToAdd: [
-          { appId: config.prodAppId!, roleId: roles.BUILTIN_ROLE_IDS.BASIC },
-        ],
-      })
-      await proSdk.groups.addUsers(group.id, [user1._id, user2._id])
-
-      await config.doInContext(config.appId, async () => {
-        await syncGlobalUsers()
-        expect(await rawUserMetadata()).toHaveLength(3)
-
-        await proSdk.groups.removeUsers(group.id, [user1._id])
-        await syncGlobalUsers()
-
-        const metadata = await rawUserMetadata()
-        expect(metadata).toHaveLength(2)
-
-        expect(metadata).not.toContainEqual(
-          expect.objectContaining({
-            _id: db.generateUserMetadataID(user1._id),
-          })
-        )
+        expect(metadata).toHaveLength(0)
       })
     })
   })

package/src/sdk/users/utils.ts

@@ -1,12 +1,13 @@
 import { getGlobalUsers } from "../../utilities/global"
 import { context, roles as rolesCore } from "@budibase/backend-core"
 import {
+  getGlobalIDFromUserMetadataID,
   generateUserMetadataID,
   getUserMetadataParams,
   InternalTables,
 } from "../../db/utils"
 import { isEqual } from "lodash"
-import { ContextUser, UserMetadata } from "@budibase/types"
+import { ContextUser, UserMetadata, User } from "@budibase/types"
 
 export function combineMetadataAndUser(
   user: ContextUser,
@@ -37,6 +38,10 @@ export function combineMetadataAndUser(
   if (found) {
     newDoc._rev = found._rev
   }
+  // clear fields that shouldn't be in metadata
+  delete newDoc.password
+  delete newDoc.forceResetPassword
+  delete newDoc.roles
   if (found == null || !isEqual(newDoc, found)) {
     return {
       ...found,
@@ -60,10 +65,9 @@ export async function rawUserMetadata() {
 export async function syncGlobalUsers() {
   // sync user metadata
   const db = context.getAppDB()
-  const [users, metadata] = await Promise.all([
-    getGlobalUsers(),
-    rawUserMetadata(),
-  ])
+  const resp = await Promise.all([getGlobalUsers(), rawUserMetadata()])
+  const users = resp[0] as User[]
+  const metadata = resp[1] as UserMetadata[]
   const toWrite = []
   for (let user of users) {
     const combined = combineMetadataAndUser(user, metadata)
@@ -71,5 +75,19 @@ export async function syncGlobalUsers() {
       toWrite.push(combined)
     }
   }
+  let foundEmails: string[] = []
+  for (let data of metadata) {
+    if (!data._id) {
+      continue
+    }
+    const alreadyExisting = data.email && foundEmails.indexOf(data.email) !== -1
+    const globalId = getGlobalIDFromUserMetadataID(data._id)
+    if (!users.find(user => user._id === globalId) || alreadyExisting) {
+      toWrite.push({ ...data, _deleted: true })
+    }
+    if (data.email) {
+      foundEmails.push(data.email)
+    }
+  }
   await db.bulkDocs(toWrite)
 }

package/src/startup.ts

@@ -10,19 +10,16 @@ import fs from "fs"
 import { watch } from "./watch"
 import * as automations from "./automations"
 import * as fileSystem from "./utilities/fileSystem"
-import eventEmitter from "./events"
+import { default as eventEmitter, init as eventInit } from "./events"
 import * as migrations from "./migrations"
 import * as bullboard from "./automations/bullboard"
 import * as pro from "@budibase/pro"
 import * as api from "./api"
 import sdk from "./sdk"
-const pino = require("koa-pino-logger")
 
 let STARTUP_RAN = false
 
 async function initRoutes(app: any) {
-  app.use(pino(logging.pinoSettings()))
-
   if (!env.isTest()) {
     const plugin = await bullboard.init()
     app.use(plugin)
@@ -48,8 +45,10 @@ async function initPro() {
 }
 
 function shutdown(server?: any) {
-  server.close()
-  server.destroy()
+  if (server) {
+    server.close()
+    server.destroy()
+  }
 }
 
 export async function startup(app?: any, server?: any) {
@@ -64,6 +63,7 @@ export async function startup(app?: any, server?: any) {
   eventEmitter.emitPort(env.PORT)
   fileSystem.init()
   await redis.init()
+  eventInit()
 
   // run migrations on startup if not done via http
   // not recommended in a clustered environment
@@ -72,11 +72,39 @@ export async function startup(app?: any, server?: any) {
       await migrations.migrate()
     } catch (e) {
       logging.logAlert("Error performing migrations. Exiting.", e)
-      shutdown()
+      shutdown(server)
     }
   }
 
+  // monitor plugin directory if required
+  if (
+    env.SELF_HOSTED &&
+    !env.MULTI_TENANCY &&
+    env.PLUGINS_DIR &&
+    fs.existsSync(env.PLUGINS_DIR)
+  ) {
+    watch()
+  }
+
+  // check for version updates
+  await installation.checkInstallVersion()
+
+  // get the references to the queue promises, don't await as
+  // they will never end, unless the processing stops
+  let queuePromises = []
+  // configure events to use the pro audit log write
+  // can't integrate directly into backend-core due to cyclic issues
+  queuePromises.push(events.processors.init(pro.sdk.auditLogs.write))
+  queuePromises.push(automations.init())
+  queuePromises.push(initPro())
+  if (app) {
+    // bring routes online as final step once everything ready
+    await initRoutes(app)
+  }
+
   // check and create admin user if required
+  // this must be run after the api has been initialised due to
+  // the app user sync
   if (
     env.SELF_HOSTED &&
     !env.MULTI_TENANCY &&
@@ -103,34 +131,8 @@ export async function startup(app?: any, server?: any) {
         )
       } catch (e) {
         logging.logAlert("Error creating initial admin user. Exiting.", e)
-        shutdown()
+        shutdown(server)
       }
     }
   }
-
-  // monitor plugin directory if required
-  if (
-    env.SELF_HOSTED &&
-    !env.MULTI_TENANCY &&
-    env.PLUGINS_DIR &&
-    fs.existsSync(env.PLUGINS_DIR)
-  ) {
-    watch()
-  }
-
-  // check for version updates
-  await installation.checkInstallVersion()
-
-  // get the references to the queue promises, don't await as
-  // they will never end, unless the processing stops
-  let queuePromises = []
-  // configure events to use the pro audit log write
-  // can't integrate directly into backend-core due to cyclic issues
-  queuePromises.push(events.processors.init(pro.sdk.auditLogs.write))
-  queuePromises.push(automations.init())
-  queuePromises.push(initPro())
-  if (app) {
-    // bring routes online as final step once everything ready
-    await initRoutes(app)
-  }
 }

package/src/tests/jestEnv.ts

@@ -6,7 +6,6 @@ process.env.MULTI_TENANCY = "1"
 // @ts-ignore
 process.env.BUDIBASE_DIR = tmpdir("budibase-unittests")
 process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
-process.env.ENABLE_4XX_HTTP_LOGGING = "0"
 process.env.MOCK_REDIS = "1"
 process.env.PLATFORM_URL = "http://localhost:10000"
 process.env.REDIS_PASSWORD = "budibase"

package/src/tests/jestSetup.ts

@@ -1,4 +1,3 @@
-import "./logging"
 import env from "../environment"
 import { env as coreEnv, timers } from "@budibase/backend-core"
 import { testContainerUtils } from "@budibase/backend-core/tests"

package/src/tests/utilities/TestConfiguration.ts

@@ -49,6 +49,7 @@ import {
   SearchFilters,
   UserRoles,
 } from "@budibase/types"
+import { BUILTIN_ROLE_IDS } from "@budibase/backend-core/src/security/roles"
 
 type DefaultUserValues = {
   globalUserId: string
@@ -306,6 +307,33 @@ class TestConfiguration {
     }
   }
 
+  async createGroup(roleId: string = BUILTIN_ROLE_IDS.BASIC) {
+    return context.doInTenant(this.tenantId!, async () => {
+      const baseGroup = structures.userGroups.userGroup()
+      baseGroup.roles = {
+        [this.prodAppId]: roleId,
+      }
+      const { id, rev } = await pro.sdk.groups.save(baseGroup)
+      return {
+        _id: id,
+        _rev: rev,
+        ...baseGroup,
+      }
+    })
+  }
+
+  async addUserToGroup(groupId: string, userId: string) {
+    return context.doInTenant(this.tenantId!, async () => {
+      await pro.sdk.groups.addUsers(groupId, [userId])
+    })
+  }
+
+  async removeUserFromGroup(groupId: string, userId: string) {
+    return context.doInTenant(this.tenantId!, async () => {
+      await pro.sdk.groups.removeUsers(groupId, [userId])
+    })
+  }
+
   async login({ roleId, userId, builder, prodApp = false }: any = {}) {
     const appId = prodApp ? this.prodAppId : this.appId
     return context.doInAppContext(appId, async () => {

package/src/tests/utilities/structures.ts

@@ -1,18 +1,22 @@
-import { permissions, roles } from "@budibase/backend-core"
+import { permissions, roles, utils } from "@budibase/backend-core"
 import { createHomeScreen } from "../../constants/screens"
 import { EMPTY_LAYOUT } from "../../constants/layouts"
 import { cloneDeep } from "lodash/fp"
-import { ACTION_DEFINITIONS, TRIGGER_DEFINITIONS } from "../../automations"
+import {
+  BUILTIN_ACTION_DEFINITIONS,
+  TRIGGER_DEFINITIONS,
+} from "../../automations"
 import {
   Automation,
   AutomationActionStepId,
+  AutomationStep,
+  AutomationStepType,
+  AutomationTrigger,
   AutomationTriggerStepId,
   Datasource,
   SourceName,
 } from "@budibase/types"
 
-const { v4: uuidv4 } = require("uuid")
-
 export function basicTable() {
   return {
     name: "TestTable",
@@ -71,19 +75,19 @@ export function view(tableId: string) {
 }
 
 export function automationStep(
-  actionDefinition = ACTION_DEFINITIONS.CREATE_ROW
-) {
+  actionDefinition = BUILTIN_ACTION_DEFINITIONS.CREATE_ROW
+): AutomationStep {
   return {
-    id: uuidv4(),
+    id: utils.newid(),
     ...actionDefinition,
   }
 }
 
 export function automationTrigger(
   triggerDefinition = TRIGGER_DEFINITIONS.ROW_SAVED
-) {
+): AutomationTrigger {
   return {
-    id: uuidv4(),
+    id: utils.newid(),
     ...triggerDefinition,
   }
 }
@@ -106,7 +110,7 @@ export function newAutomation({ steps, trigger }: any = {}) {
   return automation
 }
 
-export function basicAutomation(appId?: string) {
+export function basicAutomation(appId?: string): Automation {
   return {
     name: "My Automation",
     screenId: "kasdkfldsafkl",
@@ -119,18 +123,22 @@ export function basicAutomation(appId?: string) {
         tagline: "test",
         icon: "test",
         description: "test",
-        type: "trigger",
+        type: AutomationStepType.TRIGGER,
         id: "test",
         inputs: {},
         schema: {
-          inputs: {},
-          outputs: {},
+          inputs: {
+            properties: {},
+          },
+          outputs: {
+            properties: {},
+          },
         },
       },
       steps: [],
     },
     type: "automation",
-    appId,
+    appId: appId!,
   }
 }
 
@@ -154,7 +162,7 @@ export function loopAutomation(tableId: string, loopOpts?: any): Automation {
           inputs: {
             tableId,
           },
-          schema: ACTION_DEFINITIONS.QUERY_ROWS.schema,
+          schema: BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS.schema,
         },
         {
           id: "c",
@@ -163,7 +171,7 @@ export function loopAutomation(tableId: string, loopOpts?: any): Automation {
           internal: true,
           inputs: loopOpts,
           blockToLoop: "d",
-          schema: ACTION_DEFINITIONS.LOOP.schema,
+          schema: BUILTIN_ACTION_DEFINITIONS.LOOP.schema,
         },
         {
           id: "d",
@@ -173,7 +181,7 @@ export function loopAutomation(tableId: string, loopOpts?: any): Automation {
           inputs: {
             text: "log statement",
           },
-          schema: ACTION_DEFINITIONS.SERVER_LOG.schema,
+          schema: BUILTIN_ACTION_DEFINITIONS.SERVER_LOG.schema,
         },
       ],
       trigger: {

package/src/threads/automation.ts

@@ -27,15 +27,15 @@ import { processObject } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
 import * as sdkUtils from "../sdk/utils"
 import env from "../environment"
-const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
-const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
+const FILTER_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.FILTER.stepId
+const LOOP_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.LOOP.stepId
 const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED }
 
 function getLoopIterations(loopStep: LoopStep, input: LoopInput) {
   const binding = automationUtils.typecastForLooping(loopStep, input)
-  if (!loopStep || !binding) {
-    return 1
+  if (!binding) {
+    return 0
   }
   if (Array.isArray(binding)) {
     return binding.length
@@ -43,7 +43,7 @@ function getLoopIterations(loopStep: LoopStep, input: LoopInput) {
   if (typeof binding === "string") {
     return automationUtils.stringSplit(binding).length
   }
-  return 1
+  return 0
 }
 
 /**
@@ -423,13 +423,25 @@ class Orchestrator {
         }
       }
 
+      if (loopStep && iterations === 0) {
+        loopStep = undefined
+        this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
+          id: step.id,
+          stepId: step.stepId,
+          outputs: { status: AutomationStatus.NO_ITERATIONS, success: true },
+          inputs: {},
+        })
+
+        this._context.steps.splice(loopStepNumber, 1)
+        iterations = 1
+      }
+
       // Delete the step after the loop step as it's irrelevant, since information is included
       // in the loop step
       if (wasLoopStep && !loopStep) {
         this._context.steps.splice(loopStepNumber + 1, 1)
         wasLoopStep = false
       }
-
       if (loopSteps && loopSteps.length) {
         let tempOutput = {
           success: true,

package/src/utilities/csv.ts

@@ -0,0 +1,22 @@
+import csv from "csvtojson"
+
+export async function jsonFromCsvString(csvString: string) {
+  const castedWithEmptyValues = await csv({ ignoreEmpty: true }).fromString(
+    csvString
+  )
+
+  // By default the csvtojson library casts empty values as empty strings. This is causing issues on conversion.
+  // ignoreEmpty will remove the key completly if empty, so creating this empty object will ensure we return the values with the keys but empty values
+  const result = await csv({ ignoreEmpty: false }).fromString(csvString)
+  result.forEach((r, i) => {
+    for (const [key] of Object.entries(r).filter(
+      ([key, value]) => value === ""
+    )) {
+      if (castedWithEmptyValues[i][key] === undefined) {
+        r[key] = null
+      }
+    }
+  })
+
+  return result
+}

package/src/utilities/fileSystem/plugin.ts

@@ -5,6 +5,7 @@ import { join } from "path"
 import { objectStore } from "@budibase/backend-core"
 
 const DATASOURCE_PATH = join(budibaseTempDir(), "datasource")
+const AUTOMATION_PATH = join(budibaseTempDir(), "automation")
 
 export const getPluginMetadata = async (path: string) => {
   let metadata: any = {}
@@ -33,12 +34,12 @@ export const getPluginMetadata = async (path: string) => {
   return { metadata, directory: path }
 }
 
-export const getDatasourcePlugin = async (plugin: Plugin) => {
+async function getPluginImpl(path: string, plugin: Plugin) {
   const hash = plugin.schema?.hash
-  if (!fs.existsSync(DATASOURCE_PATH)) {
-    fs.mkdirSync(DATASOURCE_PATH)
+  if (!fs.existsSync(path)) {
+    fs.mkdirSync(path)
   }
-  const filename = join(DATASOURCE_PATH, plugin.name)
+  const filename = join(path, plugin.name)
   const metadataName = `${filename}.bbmetadata`
   if (fs.existsSync(filename)) {
     const currentHash = fs.readFileSync(metadataName, "utf8")
@@ -62,3 +63,11 @@ export const getDatasourcePlugin = async (plugin: Plugin) => {
 
   return require(filename)
 }
+
+export const getDatasourcePlugin = async (plugin: Plugin) => {
+  return getPluginImpl(DATASOURCE_PATH, plugin)
+}
+
+export const getAutomationPlugin = async (plugin: Plugin) => {
+  return getPluginImpl(AUTOMATION_PATH, plugin)
+}

package/src/utilities/global.ts

@@ -9,6 +9,7 @@ import {
 import env from "../environment"
 import { groups } from "@budibase/pro"
 import { UserCtx, ContextUser, User, UserGroup } from "@budibase/types"
+import { global } from "yargs"
 
 export function updateAppRole(
   user: ContextUser,
@@ -16,7 +17,7 @@ export function updateAppRole(
 ) {
   appId = appId || context.getAppId()
 
-  if (!user || !user.roles) {
+  if (!user || (!user.roles && !user.userGroups)) {
     return user
   }
   // if in an multi-tenancy environment make sure roles are never updated
@@ -27,7 +28,7 @@ export function updateAppRole(
     return user
   }
   // always use the deployed app
-  if (appId) {
+  if (appId && user.roles) {
     user.roleId = user.roles[dbCore.getProdAppID(appId)]
   }
   // if a role wasn't found then either set as admin (builder) or public (everyone else)
@@ -60,7 +61,7 @@ async function checkGroupRoles(
   return user
 }
 
-async function processUser(
+export async function processUser(
   user: ContextUser,
   opts: { appId?: string; groups?: UserGroup[] } = {}
 ) {
@@ -94,16 +95,15 @@ export async function getGlobalUser(userId: string) {
   return processUser(user, { appId })
 }
 
-export async function getGlobalUsers(users?: ContextUser[]) {
+export async function getGlobalUsers(
+  userIds?: string[],
+  opts?: { noProcessing?: boolean }
+) {
   const appId = context.getAppId()
   const db = tenancy.getGlobalDB()
-  const allGroups = await groups.fetch()
   let globalUsers
-  if (users) {
-    const globalIds = users.map(user =>
-      getGlobalIDFromUserMetadataID(user._id!)
-    )
-    globalUsers = (await db.allDocs(getMultiIDParams(globalIds))).rows.map(
+  if (userIds) {
+    globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map(
       row => row.doc
     )
   } else {
@@ -126,15 +126,20 @@ export async function getGlobalUsers(users?: ContextUser[]) {
     return globalUsers
   }
 
-  // pass in the groups, meaning we don't actually need to retrieve them for
-  // each user individually
-  return Promise.all(
-    globalUsers.map(user => processUser(user, { groups: allGroups }))
-  )
+  if (opts?.noProcessing) {
+    return globalUsers
+  } else {
+    // pass in the groups, meaning we don't actually need to retrieve them for
+    // each user individually
+    const allGroups = await groups.fetch()
+    return Promise.all(
+      globalUsers.map(user => processUser(user, { groups: allGroups }))
+    )
+  }
 }
 
 export async function getGlobalUsersFromMetadata(users: ContextUser[]) {
-  const globalUsers = await getGlobalUsers(users)
+  const globalUsers = await getGlobalUsers(users.map(user => user._id!))
   return users.map(user => {
     const globalUser = globalUsers.find(
       globalUser => globalUser && user._id?.includes(globalUser._id)

package/src/utilities/rowProcessor/utils.ts

@@ -1,11 +1,11 @@
 import {
-  FieldTypes,
-  FormulaTypes,
   AutoFieldDefaultNames,
   AutoFieldSubTypes,
+  FieldTypes,
+  FormulaTypes,
 } from "../../constants"
 import { processStringSync } from "@budibase/string-templates"
-import { FieldSchema, Table, Row } from "@budibase/types"
+import { FieldSchema, FieldType, Row, Table } from "@budibase/types"
 
 /**
  * If the subtype has been lost for any reason this works out what
@@ -50,6 +50,7 @@ export function processFormulas(
     const isStatic = schema.formulaType === FormulaTypes.STATIC
     if (
       schema.type !== FieldTypes.FORMULA ||
+      schema.formula == null ||
       (dynamic && isStatic) ||
       (!dynamic && !isStatic)
     ) {
@@ -57,13 +58,11 @@ export function processFormulas(
     }
     // iterate through rows and process formula
     for (let i = 0; i < rowArray.length; i++) {
-      if (schema.formula) {
-        let row = rowArray[i]
-        let context = contextRows ? contextRows[i] : row
-        rowArray[i] = {
-          ...row,
-          [column]: processStringSync(schema.formula, context),
-        }
+      let row = rowArray[i]
+      let context = contextRows ? contextRows[i] : row
+      rowArray[i] = {
+        ...row,
+        [column]: processStringSync(schema.formula, context),
       }
     }
   }