@budibase/server 2.5.6-alpha.4 → 2.5.6-alpha.41

package/src/automations/tests/utilities/index.ts

@@ -1,6 +1,6 @@
 import TestConfig from "../../../tests/utilities/TestConfiguration"
 import { context } from "@budibase/backend-core"
-import { ACTION_DEFINITIONS, getAction } from "../../actions"
+import { BUILTIN_ACTION_DEFINITIONS, getAction } from "../../actions"
 import emitter from "../../../events/index"
 import env from "../../../environment"
 
@@ -57,4 +57,4 @@ export async function runStep(stepId: string, inputs: any, stepContext?: any) {
 }
 
 export const apiKey = "test"
-export const actions = ACTION_DEFINITIONS
+export const actions = BUILTIN_ACTION_DEFINITIONS

package/src/automations/triggerInfo/app.ts

@@ -1,4 +1,7 @@
 import {
+  AutomationCustomIOType,
+  AutomationIOType,
+  AutomationStepType,
   AutomationTriggerSchema,
   AutomationTriggerStepId,
 } from "@budibase/types"
@@ -15,8 +18,8 @@ export const definition: AutomationTriggerSchema = {
     inputs: {
       properties: {
         fields: {
-          type: "object",
-          customType: "triggerSchema",
+          type: AutomationIOType.OBJECT,
+          customType: AutomationCustomIOType.TRIGGER_SCHEMA,
           title: "Fields",
         },
       },
@@ -25,13 +28,13 @@ export const definition: AutomationTriggerSchema = {
     outputs: {
       properties: {
         fields: {
-          type: "object",
+          type: AutomationIOType.OBJECT,
           description: "Fields submitted from the app frontend",
-          customType: "triggerSchema",
+          customType: AutomationCustomIOType.TRIGGER_SCHEMA,
         },
       },
       required: ["fields"],
     },
   },
-  type: "TRIGGER",
+  type: AutomationStepType.TRIGGER,
 }

package/src/automations/triggerInfo/cron.ts

@@ -1,4 +1,7 @@
 import {
+  AutomationCustomIOType,
+  AutomationIOType,
+  AutomationStepType,
   AutomationTriggerSchema,
   AutomationTriggerStepId,
 } from "@budibase/types"
@@ -15,8 +18,8 @@ export const definition: AutomationTriggerSchema = {
     inputs: {
       properties: {
         cron: {
-          type: "string",
-          customType: "cron",
+          type: AutomationIOType.STRING,
+          customType: AutomationCustomIOType.CRON,
           title: "Expression",
         },
       },
@@ -25,12 +28,12 @@ export const definition: AutomationTriggerSchema = {
     outputs: {
       properties: {
         timestamp: {
-          type: "number",
+          type: AutomationIOType.NUMBER,
           description: "Timestamp the cron was executed",
         },
       },
       required: ["timestamp"],
     },
   },
-  type: "TRIGGER",
+  type: AutomationStepType.TRIGGER,
 }

package/src/automations/triggerInfo/rowDeleted.ts

@@ -1,4 +1,7 @@
 import {
+  AutomationCustomIOType,
+  AutomationIOType,
+  AutomationStepType,
   AutomationTriggerSchema,
   AutomationTriggerStepId,
 } from "@budibase/types"
@@ -15,8 +18,8 @@ export const definition: AutomationTriggerSchema = {
     inputs: {
       properties: {
         tableId: {
-          type: "string",
-          customType: "table",
+          type: AutomationIOType.STRING,
+          customType: AutomationCustomIOType.TABLE,
           title: "Table",
         },
       },
@@ -25,13 +28,13 @@ export const definition: AutomationTriggerSchema = {
     outputs: {
       properties: {
         row: {
-          type: "object",
-          customType: "row",
+          type: AutomationIOType.OBJECT,
+          customType: AutomationCustomIOType.ROW,
           description: "The row that was deleted",
         },
       },
       required: ["row"],
     },
   },
-  type: "TRIGGER",
+  type: AutomationStepType.TRIGGER,
 }

package/src/automations/triggerInfo/rowSaved.ts

@@ -1,4 +1,7 @@
 import {
+  AutomationCustomIOType,
+  AutomationIOType,
+  AutomationStepType,
   AutomationTriggerSchema,
   AutomationTriggerStepId,
 } from "@budibase/types"
@@ -15,8 +18,8 @@ export const definition: AutomationTriggerSchema = {
     inputs: {
       properties: {
         tableId: {
-          type: "string",
-          customType: "table",
+          type: AutomationIOType.STRING,
+          customType: AutomationCustomIOType.TABLE,
           title: "Table",
         },
       },
@@ -25,21 +28,21 @@ export const definition: AutomationTriggerSchema = {
     outputs: {
       properties: {
         row: {
-          type: "object",
-          customType: "row",
+          type: AutomationIOType.OBJECT,
+          customType: AutomationCustomIOType.ROW,
           description: "The new row that was created",
         },
         id: {
-          type: "string",
+          type: AutomationIOType.STRING,
           description: "Row ID - can be used for updating",
         },
         revision: {
-          type: "string",
+          type: AutomationIOType.STRING,
           description: "Revision of row",
         },
       },
       required: ["row", "id"],
     },
   },
-  type: "TRIGGER",
+  type: AutomationStepType.TRIGGER,
 }

package/src/automations/triggerInfo/rowUpdated.ts

@@ -1,4 +1,7 @@
 import {
+  AutomationCustomIOType,
+  AutomationIOType,
+  AutomationStepType,
   AutomationTriggerSchema,
   AutomationTriggerStepId,
 } from "@budibase/types"
@@ -15,8 +18,8 @@ export const definition: AutomationTriggerSchema = {
     inputs: {
       properties: {
         tableId: {
-          type: "string",
-          customType: "table",
+          type: AutomationIOType.STRING,
+          customType: AutomationCustomIOType.TABLE,
           title: "Table",
         },
       },
@@ -25,21 +28,21 @@ export const definition: AutomationTriggerSchema = {
     outputs: {
       properties: {
         row: {
-          type: "object",
-          customType: "row",
+          type: AutomationIOType.OBJECT,
+          customType: AutomationCustomIOType.ROW,
           description: "The row that was updated",
         },
         id: {
-          type: "string",
+          type: AutomationIOType.STRING,
           description: "Row ID - can be used for updating",
         },
         revision: {
-          type: "string",
+          type: AutomationIOType.STRING,
           description: "Revision of row",
         },
       },
       required: ["row", "id"],
     },
   },
-  type: "TRIGGER",
+  type: AutomationStepType.TRIGGER,
 }

package/src/automations/triggerInfo/webhook.ts

@@ -1,4 +1,7 @@
 import {
+  AutomationCustomIOType,
+  AutomationIOType,
+  AutomationStepType,
   AutomationTriggerSchema,
   AutomationTriggerStepId,
 } from "@budibase/types"
@@ -15,13 +18,13 @@ export const definition: AutomationTriggerSchema = {
     inputs: {
       properties: {
         schemaUrl: {
-          type: "string",
-          customType: "webhookUrl",
+          type: AutomationIOType.STRING,
+          customType: AutomationCustomIOType.WEBHOOK_URL,
           title: "Schema URL",
         },
         triggerUrl: {
-          type: "string",
-          customType: "webhookUrl",
+          type: AutomationIOType.STRING,
+          customType: AutomationCustomIOType.WEBHOOK_URL,
           title: "Trigger URL",
         },
       },
@@ -30,12 +33,12 @@ export const definition: AutomationTriggerSchema = {
     outputs: {
       properties: {
         body: {
-          type: "object",
+          type: AutomationIOType.OBJECT,
           description: "Body of the request which hit the webhook",
         },
       },
       required: ["body"],
     },
   },
-  type: "TRIGGER",
+  type: AutomationStepType.TRIGGER,
 }

package/src/db/utils.ts

@@ -27,6 +27,7 @@ export const isProdAppID = dbCore.isProdAppID
 export const USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}`
 export const LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}`
 export const TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}`
+export const AUTOMATION_LOG_PREFIX = `${DocumentType.AUTOMATION_LOG}${SEPARATOR}`
 export const ViewName = dbCore.ViewName
 export const InternalTables = dbCore.InternalTable
 export const UNICODE_MAX = dbCore.UNICODE_MAX

package/src/integration-test/postgres.spec.ts

@@ -349,7 +349,7 @@ describe("row api - postgres", () => {
       },
       plus: true,
       source: "POSTGRES",
-      type: "datasource",
+      type: "datasource_plus",
       _id: expect.any(String),
       _rev: expect.any(String),
       createdAt: expect.any(String),
@@ -921,6 +921,7 @@ describe("row api - postgres", () => {
             [m2mFieldName]: [
               {
                 _id: row._id,
+                primaryDisplay: "Invalid display column",
               },
             ],
           })
@@ -929,6 +930,7 @@ describe("row api - postgres", () => {
             [m2mFieldName]: [
               {
                 _id: row._id,
+                primaryDisplay: "Invalid display column",
               },
             ],
           })

package/src/integrations/base/sqlTable.ts

@@ -79,10 +79,17 @@ function generateSchema(
           if (!relatedTable) {
             throw "Referenced table doesn't exist"
           }
-          schema.integer(column.foreignKey).unsigned()
+          const relatedPrimary = relatedTable.primary[0]
+          const externalType = relatedTable.schema[relatedPrimary].externalType
+          if (externalType) {
+            schema.specificType(column.foreignKey, externalType)
+          } else {
+            schema.integer(column.foreignKey).unsigned()
+          }
+
           schema
             .foreign(column.foreignKey)
-            .references(`${tableName}.${relatedTable.primary[0]}`)
+            .references(`${tableName}.${relatedPrimary}`)
         }
         break
     }

package/src/integrations/index.ts

@@ -14,11 +14,11 @@ import firebase from "./firebase"
 import redis from "./redis"
 import snowflake from "./snowflake"
 import oracle from "./oracle"
-import { getPlugins } from "../api/controllers/plugin"
 import { SourceName, Integration, PluginType } from "@budibase/types"
 import { getDatasourcePlugin } from "../utilities/fileSystem"
 import env from "../environment"
 import { cloneDeep } from "lodash"
+import sdk from "../sdk"
 
 const DEFINITIONS: { [key: string]: Integration } = {
   [SourceName.POSTGRES]: postgres.schema,
@@ -79,7 +79,7 @@ export async function getDefinition(source: SourceName): Promise<Integration> {
 export async function getDefinitions() {
   const pluginSchemas: { [key: string]: Integration } = {}
   if (env.SELF_HOSTED) {
-    const plugins = await getPlugins(PluginType.DATASOURCE)
+    const plugins = await sdk.plugins.fetch(PluginType.DATASOURCE)
     // extract the actual schema from each custom
     for (let plugin of plugins) {
       const sourceId = plugin.name
@@ -103,7 +103,7 @@ export async function getIntegration(integration: string) {
     return INTEGRATIONS[integration]
   }
   if (env.SELF_HOSTED) {
-    const plugins = await getPlugins(PluginType.DATASOURCE)
+    const plugins = await sdk.plugins.fetch(PluginType.DATASOURCE)
     for (let plugin of plugins) {
       if (plugin.name === integration) {
         // need to use commonJS require due to its dynamic runtime nature

package/src/migrations/functions/syncQuotas.ts

@@ -2,6 +2,7 @@ import { runQuotaMigration } from "./usageQuotas"
 import * as syncApps from "./usageQuotas/syncApps"
 import * as syncRows from "./usageQuotas/syncRows"
 import * as syncPlugins from "./usageQuotas/syncPlugins"
+import * as syncUsers from "./usageQuotas/syncUsers"
 
 /**
  * Synchronise quotas to the state of the db.
@@ -11,5 +12,6 @@ export const run = async () => {
     await syncApps.run()
     await syncRows.run()
     await syncPlugins.run()
+    await syncUsers.run()
   })
 }

package/src/migrations/functions/usageQuotas/syncApps.ts

@@ -1,4 +1,4 @@
-import { tenancy, db as dbCore } from "@budibase/backend-core"
+import { db as dbCore } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
 
@@ -8,7 +8,6 @@ export const run = async () => {
   const appCount = devApps ? devApps.length : 0
 
   // sync app count
-  const tenantId = tenancy.getTenantId()
   console.log(`Syncing app count: ${appCount}`)
   await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC)
 }

package/src/migrations/functions/usageQuotas/syncUsers.ts

@@ -0,0 +1,9 @@
+import { users } from "@budibase/backend-core"
+import { quotas } from "@budibase/pro"
+import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
+
+export const run = async () => {
+  const userCount = await users.getUserCount()
+  console.log(`Syncing user count: ${userCount}`)
+  await quotas.setUsage(userCount, StaticQuotaName.USERS, QuotaUsageType.STATIC)
+}

package/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts

@@ -0,0 +1,26 @@
+import TestConfig from "../../../../tests/utilities/TestConfiguration"
+import * as syncUsers from "../syncUsers"
+import { quotas } from "@budibase/pro"
+
+describe("syncUsers", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)
+
+  it("syncs users", async () => {
+    return config.doInContext(null, async () => {
+      await config.createUser()
+
+      await syncUsers.run()
+
+      const usageDoc = await quotas.getQuotaUsage()
+      // default + additional user
+      const userCount = 2
+      expect(usageDoc.usageQuota.users).toBe(userCount)
+    })
+  })
+})

package/src/migrations/index.ts

@@ -11,6 +11,7 @@ import env from "../environment"
 // migration functions
 import * as userEmailViewCasing from "./functions/userEmailViewCasing"
 import * as syncQuotas from "./functions/syncQuotas"
+import * as syncUsers from "./functions/usageQuotas/syncUsers"
 import * as appUrls from "./functions/appUrls"
 import * as tableSettings from "./functions/tableSettings"
 import * as backfill from "./functions/backfill"

package/src/sdk/app/backups/exports.ts

@@ -3,6 +3,7 @@ import { budibaseTempDir } from "../../../utilities/budibaseDir"
 import { streamFile, createTempFolder } from "../../../utilities/fileSystem"
 import { ObjectStoreBuckets } from "../../../constants"
 import {
+  AUTOMATION_LOG_PREFIX,
   LINK_USER_METADATA_PREFIX,
   TABLE_ROW_PREFIX,
   USER_METDATA_PREFIX,
@@ -20,11 +21,15 @@ const uuid = require("uuid/v4")
 const tar = require("tar")
 const MemoryStream = require("memorystream")
 
-type ExportOpts = {
+interface DBDumpOpts {
   filter?: any
   exportPath?: string
+}
+
+interface ExportOpts extends DBDumpOpts {
   tar?: boolean
   excludeRows?: boolean
+  excludeLogs?: boolean
 }
 
 function tarFilesToTmp(tmpDir: string, files: string[]) {
@@ -49,7 +54,7 @@ function tarFilesToTmp(tmpDir: string, files: string[]) {
  * a filter function or the name of the export.
  * @return {*} either a readable stream or a string
  */
-export async function exportDB(dbName: string, opts: ExportOpts = {}) {
+export async function exportDB(dbName: string, opts: DBDumpOpts = {}) {
   const exportOpts = {
     filter: opts?.filter,
     batch_size: 1000,
@@ -76,11 +81,14 @@ export async function exportDB(dbName: string, opts: ExportOpts = {}) {
   })
 }
 
-function defineFilter(excludeRows?: boolean) {
+function defineFilter(excludeRows?: boolean, excludeLogs?: boolean) {
   const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX]
   if (excludeRows) {
     ids.push(TABLE_ROW_PREFIX)
   }
+  if (excludeLogs) {
+    ids.push(AUTOMATION_LOG_PREFIX)
+  }
   return (doc: any) =>
     !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr)
 }
@@ -130,8 +138,7 @@ export async function exportApp(appId: string, config?: ExportOpts) {
   // enforce an export of app DB to the tmp path
   const dbPath = join(tmpPath, DB_EXPORT_FILE)
   await exportDB(appId, {
-    ...config,
-    filter: defineFilter(config?.excludeRows),
+    filter: defineFilter(config?.excludeRows, config?.excludeLogs),
     exportPath: dbPath,
   })
   // if tar requested, return where the tarball is
@@ -148,41 +155,6 @@ export async function exportApp(appId: string, config?: ExportOpts) {
   }
 }
 
-/**
- * Export all apps + global DB (if supplied) to a single tarball, this includes
- * the attachments for each app as well.
- * @param {object[]} appMetadata The IDs and names of apps to export.
- * @param {string} globalDbContents The contents of the global DB to export as well.
- * @return {string} The path to the tarball.
- */
-export async function exportMultipleApps(
-  appMetadata: { appId: string; name: string }[],
-  globalDbContents?: string
-) {
-  const tmpPath = join(budibaseTempDir(), uuid())
-  fs.mkdirSync(tmpPath)
-  let exportPromises: Promise<void>[] = []
-  // export each app to a directory, then move it into the complete export
-  const exportAndMove = async (appId: string, appName: string) => {
-    const path = await exportApp(appId)
-    await fs.promises.rename(path, join(tmpPath, appName))
-  }
-  for (let metadata of appMetadata) {
-    exportPromises.push(exportAndMove(metadata.appId, metadata.name))
-  }
-  // wait for all exports to finish
-  await Promise.all(exportPromises)
-  // add the global DB contents
-  if (globalDbContents) {
-    fs.writeFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), globalDbContents)
-  }
-  const appNames = appMetadata.map(metadata => metadata.name)
-  const tarPath = tarFilesToTmp(tmpPath, [...appNames, GLOBAL_DB_EXPORT_FILE])
-  // clear up the tmp path now tarball generated
-  fs.rmSync(tmpPath, { recursive: true, force: true })
-  return tarPath
-}
-
 /**
  * Streams a backup of the database state for an app
  * @param {string} appId The ID of the app which is to be backed up.
@@ -190,6 +162,10 @@ export async function exportMultipleApps(
  * @returns {*} a readable stream of the backup which is written in real time
  */
 export async function streamExportApp(appId: string, excludeRows: boolean) {
-  const tmpPath = await exportApp(appId, { excludeRows, tar: true })
+  const tmpPath = await exportApp(appId, {
+    excludeRows,
+    excludeLogs: true,
+    tar: true,
+  })
   return streamFile(tmpPath)
 }

package/src/sdk/index.ts

@@ -6,6 +6,7 @@ import { default as datasources } from "./app/datasources"
 import { default as queries } from "./app/queries"
 import { default as rows } from "./app/rows"
 import { default as users } from "./users"
+import { default as plugins } from "./plugins"
 
 const sdk = {
   backups,
@@ -16,6 +17,7 @@ const sdk = {
   users,
   datasources,
   queries,
+  plugins,
 }
 
 // default export for TS

package/src/sdk/plugins/index.ts

@@ -0,0 +1,5 @@
+import * as plugins from "./plugins"
+
+export default {
+  ...plugins,
+}

package/src/sdk/plugins/plugins.ts

@@ -0,0 +1,41 @@
+import { FileType, Plugin, PluginSource, PluginType } from "@budibase/types"
+import {
+  db as dbCore,
+  objectStore,
+  plugins as pluginCore,
+  tenancy,
+} from "@budibase/backend-core"
+import { fileUpload } from "../../api/controllers/plugin/file"
+import env from "../../environment"
+import { clientAppSocket } from "../../websockets"
+import { sdk as pro } from "@budibase/pro"
+
+export async function fetch(type?: PluginType) {
+  const db = tenancy.getGlobalDB()
+  const response = await db.allDocs(
+    dbCore.getPluginParams(null, {
+      include_docs: true,
+    })
+  )
+  let plugins = response.rows.map((row: any) => row.doc) as Plugin[]
+  plugins = objectStore.enrichPluginURLs(plugins)
+  if (type) {
+    return plugins.filter((plugin: Plugin) => plugin.schema?.type === type)
+  } else {
+    return plugins
+  }
+}
+
+export async function processUploaded(plugin: FileType, source?: PluginSource) {
+  const { metadata, directory } = await fileUpload(plugin)
+  pluginCore.validate(metadata?.schema)
+
+  // Only allow components in cloud
+  if (!env.SELF_HOSTED && metadata?.schema?.type !== PluginType.COMPONENT) {
+    throw new Error("Only component plugins are supported outside of self-host")
+  }
+
+  const doc = await pro.plugins.storePlugin(metadata, directory, source)
+  clientAppSocket.emit("plugin-update", { name: doc.name, hash: doc.hash })
+  return doc
+}