@budibase/server 2.5.6-alpha.2 → 2.5.6-alpha.20

package/src/automations/triggerInfo/rowUpdated.ts

@@ -1,4 +1,7 @@
 import {
+  AutomationCustomIOType,
+  AutomationIOType,
+  AutomationStepType,
   AutomationTriggerSchema,
   AutomationTriggerStepId,
 } from "@budibase/types"
@@ -15,8 +18,8 @@ export const definition: AutomationTriggerSchema = {
     inputs: {
       properties: {
         tableId: {
-          type: "string",
-          customType: "table",
+          type: AutomationIOType.STRING,
+          customType: AutomationCustomIOType.TABLE,
           title: "Table",
         },
       },
@@ -25,21 +28,21 @@ export const definition: AutomationTriggerSchema = {
     outputs: {
       properties: {
         row: {
-          type: "object",
-          customType: "row",
+          type: AutomationIOType.OBJECT,
+          customType: AutomationCustomIOType.ROW,
           description: "The row that was updated",
         },
         id: {
-          type: "string",
+          type: AutomationIOType.STRING,
           description: "Row ID - can be used for updating",
         },
         revision: {
-          type: "string",
+          type: AutomationIOType.STRING,
           description: "Revision of row",
         },
       },
       required: ["row", "id"],
     },
   },
-  type: "TRIGGER",
+  type: AutomationStepType.TRIGGER,
 }

package/src/automations/triggerInfo/webhook.ts

@@ -1,4 +1,7 @@
 import {
+  AutomationCustomIOType,
+  AutomationIOType,
+  AutomationStepType,
   AutomationTriggerSchema,
   AutomationTriggerStepId,
 } from "@budibase/types"
@@ -15,13 +18,13 @@ export const definition: AutomationTriggerSchema = {
     inputs: {
       properties: {
         schemaUrl: {
-          type: "string",
-          customType: "webhookUrl",
+          type: AutomationIOType.STRING,
+          customType: AutomationCustomIOType.WEBHOOK_URL,
           title: "Schema URL",
         },
         triggerUrl: {
-          type: "string",
-          customType: "webhookUrl",
+          type: AutomationIOType.STRING,
+          customType: AutomationCustomIOType.WEBHOOK_URL,
           title: "Trigger URL",
         },
       },
@@ -30,12 +33,12 @@ export const definition: AutomationTriggerSchema = {
     outputs: {
       properties: {
         body: {
-          type: "object",
+          type: AutomationIOType.OBJECT,
           description: "Body of the request which hit the webhook",
         },
       },
       required: ["body"],
     },
   },
-  type: "TRIGGER",
+  type: AutomationStepType.TRIGGER,
 }

package/src/integrations/index.ts

@@ -14,11 +14,11 @@ import firebase from "./firebase"
 import redis from "./redis"
 import snowflake from "./snowflake"
 import oracle from "./oracle"
-import { getPlugins } from "../api/controllers/plugin"
 import { SourceName, Integration, PluginType } from "@budibase/types"
 import { getDatasourcePlugin } from "../utilities/fileSystem"
 import env from "../environment"
 import { cloneDeep } from "lodash"
+import sdk from "../sdk"
 
 const DEFINITIONS: { [key: string]: Integration } = {
   [SourceName.POSTGRES]: postgres.schema,
@@ -79,7 +79,7 @@ export async function getDefinition(source: SourceName): Promise<Integration> {
 export async function getDefinitions() {
   const pluginSchemas: { [key: string]: Integration } = {}
   if (env.SELF_HOSTED) {
-    const plugins = await getPlugins(PluginType.DATASOURCE)
+    const plugins = await sdk.plugins.fetch(PluginType.DATASOURCE)
     // extract the actual schema from each custom
     for (let plugin of plugins) {
       const sourceId = plugin.name
@@ -103,7 +103,7 @@ export async function getIntegration(integration: string) {
     return INTEGRATIONS[integration]
   }
   if (env.SELF_HOSTED) {
-    const plugins = await getPlugins(PluginType.DATASOURCE)
+    const plugins = await sdk.plugins.fetch(PluginType.DATASOURCE)
     for (let plugin of plugins) {
       if (plugin.name === integration) {
         // need to use commonJS require due to its dynamic runtime nature

package/src/migrations/functions/syncQuotas.ts

@@ -2,6 +2,7 @@ import { runQuotaMigration } from "./usageQuotas"
 import * as syncApps from "./usageQuotas/syncApps"
 import * as syncRows from "./usageQuotas/syncRows"
 import * as syncPlugins from "./usageQuotas/syncPlugins"
+import * as syncUsers from "./usageQuotas/syncUsers"
 
 /**
  * Synchronise quotas to the state of the db.
@@ -11,5 +12,6 @@ export const run = async () => {
     await syncApps.run()
     await syncRows.run()
     await syncPlugins.run()
+    await syncUsers.run()
   })
 }

package/src/migrations/functions/usageQuotas/syncApps.ts

@@ -1,4 +1,4 @@
-import { tenancy, db as dbCore } from "@budibase/backend-core"
+import { db as dbCore } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
 
@@ -8,7 +8,6 @@ export const run = async () => {
   const appCount = devApps ? devApps.length : 0
 
   // sync app count
-  const tenantId = tenancy.getTenantId()
   console.log(`Syncing app count: ${appCount}`)
   await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC)
 }

package/src/migrations/functions/usageQuotas/syncUsers.ts

@@ -0,0 +1,9 @@
+import { users } from "@budibase/backend-core"
+import { quotas } from "@budibase/pro"
+import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
+
+export const run = async () => {
+  const userCount = await users.getUserCount()
+  console.log(`Syncing user count: ${userCount}`)
+  await quotas.setUsage(userCount, StaticQuotaName.USERS, QuotaUsageType.STATIC)
+}

package/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts

@@ -0,0 +1,26 @@
+import TestConfig from "../../../../tests/utilities/TestConfiguration"
+import * as syncUsers from "../syncUsers"
+import { quotas } from "@budibase/pro"
+
+describe("syncUsers", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)
+
+  it("syncs users", async () => {
+    return config.doInContext(null, async () => {
+      await config.createUser()
+
+      await syncUsers.run()
+
+      const usageDoc = await quotas.getQuotaUsage()
+      // default + additional user
+      const userCount = 2
+      expect(usageDoc.usageQuota.users).toBe(userCount)
+    })
+  })
+})

package/src/migrations/index.ts

@@ -11,6 +11,7 @@ import env from "../environment"
 // migration functions
 import * as userEmailViewCasing from "./functions/userEmailViewCasing"
 import * as syncQuotas from "./functions/syncQuotas"
+import * as syncUsers from "./functions/usageQuotas/syncUsers"
 import * as appUrls from "./functions/appUrls"
 import * as tableSettings from "./functions/tableSettings"
 import * as backfill from "./functions/backfill"

package/src/sdk/app/backups/exports.ts

@@ -148,41 +148,6 @@ export async function exportApp(appId: string, config?: ExportOpts) {
   }
 }
 
-/**
- * Export all apps + global DB (if supplied) to a single tarball, this includes
- * the attachments for each app as well.
- * @param {object[]} appMetadata The IDs and names of apps to export.
- * @param {string} globalDbContents The contents of the global DB to export as well.
- * @return {string} The path to the tarball.
- */
-export async function exportMultipleApps(
-  appMetadata: { appId: string; name: string }[],
-  globalDbContents?: string
-) {
-  const tmpPath = join(budibaseTempDir(), uuid())
-  fs.mkdirSync(tmpPath)
-  let exportPromises: Promise<void>[] = []
-  // export each app to a directory, then move it into the complete export
-  const exportAndMove = async (appId: string, appName: string) => {
-    const path = await exportApp(appId)
-    await fs.promises.rename(path, join(tmpPath, appName))
-  }
-  for (let metadata of appMetadata) {
-    exportPromises.push(exportAndMove(metadata.appId, metadata.name))
-  }
-  // wait for all exports to finish
-  await Promise.all(exportPromises)
-  // add the global DB contents
-  if (globalDbContents) {
-    fs.writeFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), globalDbContents)
-  }
-  const appNames = appMetadata.map(metadata => metadata.name)
-  const tarPath = tarFilesToTmp(tmpPath, [...appNames, GLOBAL_DB_EXPORT_FILE])
-  // clear up the tmp path now tarball generated
-  fs.rmSync(tmpPath, { recursive: true, force: true })
-  return tarPath
-}
-
 /**
  * Streams a backup of the database state for an app
  * @param {string} appId The ID of the app which is to be backed up.

package/src/sdk/index.ts

@@ -6,6 +6,7 @@ import { default as datasources } from "./app/datasources"
 import { default as queries } from "./app/queries"
 import { default as rows } from "./app/rows"
 import { default as users } from "./users"
+import { default as plugins } from "./plugins"
 
 const sdk = {
   backups,
@@ -16,6 +17,7 @@ const sdk = {
   users,
   datasources,
   queries,
+  plugins,
 }
 
 // default export for TS

package/src/sdk/plugins/index.ts

@@ -0,0 +1,5 @@
+import * as plugins from "./plugins"
+
+export default {
+  ...plugins,
+}

package/src/sdk/plugins/plugins.ts

@@ -0,0 +1,41 @@
+import { FileType, Plugin, PluginSource, PluginType } from "@budibase/types"
+import {
+  db as dbCore,
+  objectStore,
+  plugins as pluginCore,
+  tenancy,
+} from "@budibase/backend-core"
+import { fileUpload } from "../../api/controllers/plugin/file"
+import env from "../../environment"
+import { clientAppSocket } from "../../websockets"
+import { sdk as pro } from "@budibase/pro"
+
+export async function fetch(type?: PluginType) {
+  const db = tenancy.getGlobalDB()
+  const response = await db.allDocs(
+    dbCore.getPluginParams(null, {
+      include_docs: true,
+    })
+  )
+  let plugins = response.rows.map((row: any) => row.doc) as Plugin[]
+  plugins = objectStore.enrichPluginURLs(plugins)
+  if (type) {
+    return plugins.filter((plugin: Plugin) => plugin.schema?.type === type)
+  } else {
+    return plugins
+  }
+}
+
+export async function processUploaded(plugin: FileType, source?: PluginSource) {
+  const { metadata, directory } = await fileUpload(plugin)
+  pluginCore.validate(metadata?.schema)
+
+  // Only allow components in cloud
+  if (!env.SELF_HOSTED && metadata?.schema?.type !== PluginType.COMPONENT) {
+    throw new Error("Only component plugins are supported outside of self-host")
+  }
+
+  const doc = await pro.plugins.storePlugin(metadata, directory, source)
+  clientAppSocket.emit("plugin-update", { name: doc.name, hash: doc.hash })
+  return doc
+}

package/src/tests/utilities/structures.ts

@@ -1,18 +1,22 @@
-import { permissions, roles } from "@budibase/backend-core"
+import { permissions, roles, utils } from "@budibase/backend-core"
 import { createHomeScreen } from "../../constants/screens"
 import { EMPTY_LAYOUT } from "../../constants/layouts"
 import { cloneDeep } from "lodash/fp"
-import { ACTION_DEFINITIONS, TRIGGER_DEFINITIONS } from "../../automations"
+import {
+  BUILTIN_ACTION_DEFINITIONS,
+  TRIGGER_DEFINITIONS,
+} from "../../automations"
 import {
   Automation,
   AutomationActionStepId,
+  AutomationStep,
+  AutomationStepType,
+  AutomationTrigger,
   AutomationTriggerStepId,
   Datasource,
   SourceName,
 } from "@budibase/types"
 
-const { v4: uuidv4 } = require("uuid")
-
 export function basicTable() {
   return {
     name: "TestTable",
@@ -71,19 +75,19 @@ export function view(tableId: string) {
 }
 
 export function automationStep(
-  actionDefinition = ACTION_DEFINITIONS.CREATE_ROW
-) {
+  actionDefinition = BUILTIN_ACTION_DEFINITIONS.CREATE_ROW
+): AutomationStep {
   return {
-    id: uuidv4(),
+    id: utils.newid(),
     ...actionDefinition,
   }
 }
 
 export function automationTrigger(
   triggerDefinition = TRIGGER_DEFINITIONS.ROW_SAVED
-) {
+): AutomationTrigger {
   return {
-    id: uuidv4(),
+    id: utils.newid(),
     ...triggerDefinition,
   }
 }
@@ -106,7 +110,7 @@ export function newAutomation({ steps, trigger }: any = {}) {
   return automation
 }
 
-export function basicAutomation(appId?: string) {
+export function basicAutomation(appId?: string): Automation {
   return {
     name: "My Automation",
     screenId: "kasdkfldsafkl",
@@ -119,18 +123,22 @@ export function basicAutomation(appId?: string) {
         tagline: "test",
         icon: "test",
         description: "test",
-        type: "trigger",
+        type: AutomationStepType.TRIGGER,
         id: "test",
         inputs: {},
         schema: {
-          inputs: {},
-          outputs: {},
+          inputs: {
+            properties: {},
+          },
+          outputs: {
+            properties: {},
+          },
         },
       },
       steps: [],
     },
     type: "automation",
-    appId,
+    appId: appId!,
   }
 }
 
@@ -154,7 +162,7 @@ export function loopAutomation(tableId: string, loopOpts?: any): Automation {
           inputs: {
             tableId,
           },
-          schema: ACTION_DEFINITIONS.QUERY_ROWS.schema,
+          schema: BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS.schema,
         },
         {
           id: "c",
@@ -163,7 +171,7 @@ export function loopAutomation(tableId: string, loopOpts?: any): Automation {
           internal: true,
           inputs: loopOpts,
           blockToLoop: "d",
-          schema: ACTION_DEFINITIONS.LOOP.schema,
+          schema: BUILTIN_ACTION_DEFINITIONS.LOOP.schema,
         },
         {
           id: "d",
@@ -173,7 +181,7 @@ export function loopAutomation(tableId: string, loopOpts?: any): Automation {
           inputs: {
             text: "log statement",
           },
-          schema: ACTION_DEFINITIONS.SERVER_LOG.schema,
+          schema: BUILTIN_ACTION_DEFINITIONS.SERVER_LOG.schema,
         },
       ],
       trigger: {

package/src/threads/automation.ts

@@ -27,8 +27,8 @@ import { processObject } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
 import * as sdkUtils from "../sdk/utils"
 import env from "../environment"
-const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
-const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
+const FILTER_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.FILTER.stepId
+const LOOP_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.LOOP.stepId
 const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED }
 

package/src/utilities/fileSystem/plugin.ts

@@ -5,6 +5,7 @@ import { join } from "path"
 import { objectStore } from "@budibase/backend-core"
 
 const DATASOURCE_PATH = join(budibaseTempDir(), "datasource")
+const AUTOMATION_PATH = join(budibaseTempDir(), "automation")
 
 export const getPluginMetadata = async (path: string) => {
   let metadata: any = {}
@@ -33,12 +34,12 @@ export const getPluginMetadata = async (path: string) => {
   return { metadata, directory: path }
 }
 
-export const getDatasourcePlugin = async (plugin: Plugin) => {
+async function getPluginImpl(path: string, plugin: Plugin) {
   const hash = plugin.schema?.hash
-  if (!fs.existsSync(DATASOURCE_PATH)) {
-    fs.mkdirSync(DATASOURCE_PATH)
+  if (!fs.existsSync(path)) {
+    fs.mkdirSync(path)
   }
-  const filename = join(DATASOURCE_PATH, plugin.name)
+  const filename = join(path, plugin.name)
   const metadataName = `${filename}.bbmetadata`
   if (fs.existsSync(filename)) {
     const currentHash = fs.readFileSync(metadataName, "utf8")
@@ -62,3 +63,11 @@ export const getDatasourcePlugin = async (plugin: Plugin) => {
 
   return require(filename)
 }
+
+export const getDatasourcePlugin = async (plugin: Plugin) => {
+  return getPluginImpl(DATASOURCE_PATH, plugin)
+}
+
+export const getAutomationPlugin = async (plugin: Plugin) => {
+  return getPluginImpl(AUTOMATION_PATH, plugin)
+}

package/src/watch.ts

@@ -3,7 +3,7 @@ import env from "./environment"
 import chokidar from "chokidar"
 import fs from "fs"
 import { constants, tenancy } from "@budibase/backend-core"
-import { processUploadedPlugin } from "./api/controllers/plugin"
+import pluginsSdk from "./sdk/plugins"
 
 export function watch() {
   const watchPath = path.join(env.PLUGINS_DIR, "./**/*.tar.gz")
@@ -27,7 +27,7 @@ export function watch() {
           const split = path.split("/")
           const name = split[split.length - 1]
           console.log("Importing plugin:", path)
-          await processUploadedPlugin({ name, path })
+          await pluginsSdk.processUploaded({ name, path })
         } catch (err: any) {
           const message = err?.message ? err?.message : err
           console.error("Failed to import plugin:", message)

package/src/websockets/client.ts

@@ -0,0 +1,11 @@
+import Socket from "./websocket"
+import authorized from "../middleware/authorized"
+import http from "http"
+import Koa from "koa"
+import { permissions } from "@budibase/backend-core"
+
+export default class ClientAppWebsocket extends Socket {
+  constructor(app: Koa, server: http.Server) {
+    super(app, server, "/socket/client", [authorized(permissions.BUILDER)])
+  }
+}

package/src/websockets/grid.ts

@@ -0,0 +1,55 @@
+import authorized from "../middleware/authorized"
+import Socket from "./websocket"
+import { permissions } from "@budibase/backend-core"
+import http from "http"
+import Koa from "koa"
+
+export default class GridSocket extends Socket {
+  constructor(app: Koa, server: http.Server) {
+    super(app, server, "/socket/grid", [authorized(permissions.BUILDER)])
+
+    this.io.on("connection", socket => {
+      const user = socket.data.user
+      console.log(`Spreadsheet user connected: ${user?.id}`)
+
+      // Socket state
+      let currentRoom: string
+
+      // Initial identification of connected spreadsheet
+      socket.on("select-table", async (tableId, callback) => {
+        // Leave current room
+        if (currentRoom) {
+          socket.to(currentRoom).emit("user-disconnect", socket.data.user)
+          socket.leave(currentRoom)
+        }
+
+        // Join new room
+        currentRoom = tableId
+        socket.join(currentRoom)
+        socket.to(currentRoom).emit("user-update", socket.data.user)
+
+        // Reply with all users in current room
+        const sockets = await this.io.in(currentRoom).fetchSockets()
+        callback({
+          users: sockets.map(socket => socket.data.user),
+          id: user.id,
+        })
+      })
+
+      // Handle users selecting a new cell
+      socket.on("select-cell", cellId => {
+        socket.data.user.selectedCellId = cellId
+        if (currentRoom) {
+          socket.to(currentRoom).emit("user-update", socket.data.user)
+        }
+      })
+
+      // Disconnection cleanup
+      socket.on("disconnect", () => {
+        if (currentRoom) {
+          socket.to(currentRoom).emit("user-disconnect", socket.data.user)
+        }
+      })
+    })
+  }
+}

package/src/websockets/index.ts

@@ -0,0 +1,14 @@
+import http from "http"
+import Koa from "koa"
+import GridSocket from "./grid"
+import ClientAppSocket from "./client"
+
+let clientAppSocket: ClientAppSocket
+let gridSocket: GridSocket
+
+export const initialise = (app: Koa, server: http.Server) => {
+  clientAppSocket = new ClientAppSocket(app, server)
+  gridSocket = new GridSocket(app, server)
+}
+
+export { clientAppSocket, gridSocket }

package/src/websockets/websocket.ts

@@ -0,0 +1,83 @@
+import { Server } from "socket.io"
+import http from "http"
+import Koa from "koa"
+import Cookies from "cookies"
+import { userAgent } from "koa-useragent"
+import { auth } from "@budibase/backend-core"
+import currentApp from "../middleware/currentapp"
+
+export default class Socket {
+  io: Server
+
+  constructor(
+    app: Koa,
+    server: http.Server,
+    path: string,
+    additionalMiddlewares?: any[]
+  ) {
+    this.io = new Server(server, {
+      path,
+    })
+
+    // Attach default middlewares
+    const authenticate = auth.buildAuthMiddleware([], {
+      publicAllowed: true,
+    })
+    const middlewares = [
+      userAgent,
+      authenticate,
+      currentApp,
+      ...(additionalMiddlewares || []),
+    ]
+
+    // Apply middlewares
+    this.io.use(async (socket, next) => {
+      // Build fake koa context
+      const res = new http.ServerResponse(socket.request)
+      const ctx: any = {
+        ...app.createContext(socket.request, res),
+
+        // Additional overrides needed to make our middlewares work with this
+        // fake koa context
+        cookies: new Cookies(socket.request, res),
+        get: (field: string) => socket.request.headers[field],
+        throw: (code: number, message: string) => {
+          throw new Error(message)
+        },
+
+        // Needed for koa-useragent middleware
+        headers: socket.request.headers,
+        header: socket.request.headers,
+
+        // We don't really care about the path since it will never contain
+        // an app ID
+        path: "/socket",
+      }
+
+      // Run all koa middlewares
+      try {
+        for (let [idx, middleware] of middlewares.entries()) {
+          await middleware(ctx, () => {
+            if (idx === middlewares.length - 1) {
+              // Middlewares are finished.
+              // Extract some data from our enriched koa context to persist
+              // as metadata for the socket
+              socket.data.user = {
+                id: ctx.user._id,
+                email: ctx.user.email,
+              }
+              next()
+            }
+          })
+        }
+      } catch (error: any) {
+        next(error)
+      }
+    })
+  }
+
+  // Emit an event to all sockets
+  emit(event: string, payload: any) {
+    this.io.sockets.emit(event, payload)
+  }
+}