@budibase/server 2.5.9 → 2.5.10-alpha.0

package/src/automations/tests/utilities/index.ts

@@ -1,6 +1,6 @@
 import TestConfig from "../../../tests/utilities/TestConfiguration"
 import { context } from "@budibase/backend-core"
-import { ACTION_DEFINITIONS, getAction } from "../../actions"
+import { BUILTIN_ACTION_DEFINITIONS, getAction } from "../../actions"
 import emitter from "../../../events/index"
 import env from "../../../environment"
 
@@ -57,4 +57,4 @@ export async function runStep(stepId: string, inputs: any, stepContext?: any) {
 }
 
 export const apiKey = "test"
-export const actions = ACTION_DEFINITIONS
+export const actions = BUILTIN_ACTION_DEFINITIONS

package/src/automations/triggerInfo/app.ts

@@ -1,4 +1,7 @@
 import {
+  AutomationCustomIOType,
+  AutomationIOType,
+  AutomationStepType,
   AutomationTriggerSchema,
   AutomationTriggerStepId,
 } from "@budibase/types"
@@ -15,8 +18,8 @@ export const definition: AutomationTriggerSchema = {
     inputs: {
       properties: {
         fields: {
-          type: "object",
-          customType: "triggerSchema",
+          type: AutomationIOType.OBJECT,
+          customType: AutomationCustomIOType.TRIGGER_SCHEMA,
           title: "Fields",
         },
       },
@@ -25,13 +28,13 @@ export const definition: AutomationTriggerSchema = {
     outputs: {
       properties: {
         fields: {
-          type: "object",
+          type: AutomationIOType.OBJECT,
           description: "Fields submitted from the app frontend",
-          customType: "triggerSchema",
+          customType: AutomationCustomIOType.TRIGGER_SCHEMA,
         },
       },
       required: ["fields"],
     },
   },
-  type: "TRIGGER",
+  type: AutomationStepType.TRIGGER,
 }

package/src/automations/triggerInfo/cron.ts

@@ -1,4 +1,7 @@
 import {
+  AutomationCustomIOType,
+  AutomationIOType,
+  AutomationStepType,
   AutomationTriggerSchema,
   AutomationTriggerStepId,
 } from "@budibase/types"
@@ -15,8 +18,8 @@ export const definition: AutomationTriggerSchema = {
     inputs: {
       properties: {
         cron: {
-          type: "string",
-          customType: "cron",
+          type: AutomationIOType.STRING,
+          customType: AutomationCustomIOType.CRON,
           title: "Expression",
         },
       },
@@ -25,12 +28,12 @@ export const definition: AutomationTriggerSchema = {
     outputs: {
       properties: {
         timestamp: {
-          type: "number",
+          type: AutomationIOType.NUMBER,
           description: "Timestamp the cron was executed",
         },
       },
       required: ["timestamp"],
     },
   },
-  type: "TRIGGER",
+  type: AutomationStepType.TRIGGER,
 }

package/src/automations/triggerInfo/rowDeleted.ts

@@ -1,4 +1,7 @@
 import {
+  AutomationCustomIOType,
+  AutomationIOType,
+  AutomationStepType,
   AutomationTriggerSchema,
   AutomationTriggerStepId,
 } from "@budibase/types"
@@ -15,8 +18,8 @@ export const definition: AutomationTriggerSchema = {
     inputs: {
       properties: {
         tableId: {
-          type: "string",
-          customType: "table",
+          type: AutomationIOType.STRING,
+          customType: AutomationCustomIOType.TABLE,
           title: "Table",
         },
       },
@@ -25,13 +28,13 @@ export const definition: AutomationTriggerSchema = {
     outputs: {
       properties: {
         row: {
-          type: "object",
-          customType: "row",
+          type: AutomationIOType.OBJECT,
+          customType: AutomationCustomIOType.ROW,
           description: "The row that was deleted",
         },
       },
       required: ["row"],
     },
   },
-  type: "TRIGGER",
+  type: AutomationStepType.TRIGGER,
 }

package/src/automations/triggerInfo/rowSaved.ts

@@ -1,4 +1,7 @@
 import {
+  AutomationCustomIOType,
+  AutomationIOType,
+  AutomationStepType,
   AutomationTriggerSchema,
   AutomationTriggerStepId,
 } from "@budibase/types"
@@ -15,8 +18,8 @@ export const definition: AutomationTriggerSchema = {
     inputs: {
       properties: {
         tableId: {
-          type: "string",
-          customType: "table",
+          type: AutomationIOType.STRING,
+          customType: AutomationCustomIOType.TABLE,
           title: "Table",
         },
       },
@@ -25,21 +28,21 @@ export const definition: AutomationTriggerSchema = {
     outputs: {
       properties: {
         row: {
-          type: "object",
-          customType: "row",
+          type: AutomationIOType.OBJECT,
+          customType: AutomationCustomIOType.ROW,
           description: "The new row that was created",
         },
         id: {
-          type: "string",
+          type: AutomationIOType.STRING,
           description: "Row ID - can be used for updating",
         },
         revision: {
-          type: "string",
+          type: AutomationIOType.STRING,
           description: "Revision of row",
         },
       },
       required: ["row", "id"],
     },
   },
-  type: "TRIGGER",
+  type: AutomationStepType.TRIGGER,
 }

package/src/automations/triggerInfo/rowUpdated.ts

@@ -1,4 +1,7 @@
 import {
+  AutomationCustomIOType,
+  AutomationIOType,
+  AutomationStepType,
   AutomationTriggerSchema,
   AutomationTriggerStepId,
 } from "@budibase/types"
@@ -15,8 +18,8 @@ export const definition: AutomationTriggerSchema = {
     inputs: {
       properties: {
         tableId: {
-          type: "string",
-          customType: "table",
+          type: AutomationIOType.STRING,
+          customType: AutomationCustomIOType.TABLE,
           title: "Table",
         },
       },
@@ -25,21 +28,21 @@ export const definition: AutomationTriggerSchema = {
     outputs: {
       properties: {
         row: {
-          type: "object",
-          customType: "row",
+          type: AutomationIOType.OBJECT,
+          customType: AutomationCustomIOType.ROW,
           description: "The row that was updated",
         },
         id: {
-          type: "string",
+          type: AutomationIOType.STRING,
           description: "Row ID - can be used for updating",
         },
         revision: {
-          type: "string",
+          type: AutomationIOType.STRING,
           description: "Revision of row",
         },
       },
       required: ["row", "id"],
     },
   },
-  type: "TRIGGER",
+  type: AutomationStepType.TRIGGER,
 }

package/src/automations/triggerInfo/webhook.ts

@@ -1,4 +1,7 @@
 import {
+  AutomationCustomIOType,
+  AutomationIOType,
+  AutomationStepType,
   AutomationTriggerSchema,
   AutomationTriggerStepId,
 } from "@budibase/types"
@@ -15,13 +18,13 @@ export const definition: AutomationTriggerSchema = {
     inputs: {
       properties: {
         schemaUrl: {
-          type: "string",
-          customType: "webhookUrl",
+          type: AutomationIOType.STRING,
+          customType: AutomationCustomIOType.WEBHOOK_URL,
           title: "Schema URL",
         },
         triggerUrl: {
-          type: "string",
-          customType: "webhookUrl",
+          type: AutomationIOType.STRING,
+          customType: AutomationCustomIOType.WEBHOOK_URL,
           title: "Trigger URL",
         },
       },
@@ -30,12 +33,12 @@ export const definition: AutomationTriggerSchema = {
     outputs: {
       properties: {
         body: {
-          type: "object",
+          type: AutomationIOType.OBJECT,
           description: "Body of the request which hit the webhook",
         },
       },
       required: ["body"],
     },
   },
-  type: "TRIGGER",
+  type: AutomationStepType.TRIGGER,
 }

package/src/db/utils.ts

@@ -27,6 +27,7 @@ export const isProdAppID = dbCore.isProdAppID
 export const USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}`
 export const LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}`
 export const TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}`
+export const AUTOMATION_LOG_PREFIX = `${DocumentType.AUTOMATION_LOG}${SEPARATOR}`
 export const ViewName = dbCore.ViewName
 export const InternalTables = dbCore.InternalTable
 export const UNICODE_MAX = dbCore.UNICODE_MAX

package/src/environment.ts

@@ -62,7 +62,6 @@ const environment = {
   // minor
   SALT_ROUNDS: process.env.SALT_ROUNDS,
   LOGGER: process.env.LOGGER,
-  LOG_LEVEL: process.env.LOG_LEVEL,
   ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
   AUTOMATION_MAX_ITERATIONS:
     parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) || 200,

package/src/events/docUpdates/index.ts

@@ -0,0 +1 @@
+export * from "./processors"

package/src/events/docUpdates/processors.ts

@@ -0,0 +1,14 @@
+import userGroupProcessor from "./syncUsers"
+import { docUpdates } from "@budibase/backend-core"
+
+export type UpdateCallback = (docId: string) => void
+let started = false
+
+export function init(updateCb?: UpdateCallback) {
+  if (started) {
+    return
+  }
+  const processors = [userGroupProcessor(updateCb)]
+  docUpdates.init(processors)
+  started = true
+}

package/src/events/docUpdates/syncUsers.ts

@@ -0,0 +1,35 @@
+import { constants, logging } from "@budibase/backend-core"
+import { sdk as proSdk } from "@budibase/pro"
+import { DocUpdateEvent, UserGroupSyncEvents } from "@budibase/types"
+import { syncUsersToAllApps } from "../../sdk/app/applications/sync"
+import { UpdateCallback } from "./processors"
+
+export default function process(updateCb?: UpdateCallback) {
+  const processor = async (update: DocUpdateEvent) => {
+    try {
+      const docId = update.id
+      const isGroup = docId.startsWith(constants.DocumentType.GROUP)
+      let userIds: string[]
+      if (isGroup) {
+        const group = await proSdk.groups.get(docId)
+        userIds = group.users?.map(user => user._id) || []
+      } else {
+        userIds = [docId]
+      }
+      if (userIds.length > 0) {
+        await syncUsersToAllApps(userIds)
+      }
+      if (updateCb) {
+        updateCb(docId)
+      }
+    } catch (err: any) {
+      // if something not found - no changes to perform
+      if (err?.status === 404) {
+        return
+      } else {
+        logging.logAlert("Failed to perform user/group app sync", err)
+      }
+    }
+  }
+  return { events: UserGroupSyncEvents, processor }
+}

package/src/events/index.ts

@@ -2,4 +2,5 @@ import BudibaseEmitter from "./BudibaseEmitter"
 
 const emitter = new BudibaseEmitter()
 
+export { init } from "./docUpdates"
 export default emitter

package/src/integration-test/postgres.spec.ts

@@ -349,7 +349,7 @@ describe("row api - postgres", () => {
       },
       plus: true,
       source: "POSTGRES",
-      type: "datasource",
+      type: "datasource_plus",
       _id: expect.any(String),
       _rev: expect.any(String),
       createdAt: expect.any(String),
@@ -921,6 +921,7 @@ describe("row api - postgres", () => {
             [m2mFieldName]: [
               {
                 _id: row._id,
+                primaryDisplay: "Invalid display column",
               },
             ],
           })
@@ -929,6 +930,7 @@ describe("row api - postgres", () => {
             [m2mFieldName]: [
               {
                 _id: row._id,
+                primaryDisplay: "Invalid display column",
               },
             ],
           })

package/src/integrations/base/sqlTable.ts

@@ -79,10 +79,17 @@ function generateSchema(
           if (!relatedTable) {
             throw "Referenced table doesn't exist"
           }
-          schema.integer(column.foreignKey).unsigned()
+          const relatedPrimary = relatedTable.primary[0]
+          const externalType = relatedTable.schema[relatedPrimary].externalType
+          if (externalType) {
+            schema.specificType(column.foreignKey, externalType)
+          } else {
+            schema.integer(column.foreignKey).unsigned()
+          }
+
           schema
             .foreign(column.foreignKey)
-            .references(`${tableName}.${relatedTable.primary[0]}`)
+            .references(`${tableName}.${relatedPrimary}`)
         }
         break
     }

package/src/integrations/index.ts

@@ -14,11 +14,11 @@ import firebase from "./firebase"
 import redis from "./redis"
 import snowflake from "./snowflake"
 import oracle from "./oracle"
-import { getPlugins } from "../api/controllers/plugin"
 import { SourceName, Integration, PluginType } from "@budibase/types"
 import { getDatasourcePlugin } from "../utilities/fileSystem"
 import env from "../environment"
 import { cloneDeep } from "lodash"
+import sdk from "../sdk"
 
 const DEFINITIONS: { [key: string]: Integration } = {
   [SourceName.POSTGRES]: postgres.schema,
@@ -79,7 +79,7 @@ export async function getDefinition(source: SourceName): Promise<Integration> {
 export async function getDefinitions() {
   const pluginSchemas: { [key: string]: Integration } = {}
   if (env.SELF_HOSTED) {
-    const plugins = await getPlugins(PluginType.DATASOURCE)
+    const plugins = await sdk.plugins.fetch(PluginType.DATASOURCE)
     // extract the actual schema from each custom
     for (let plugin of plugins) {
       const sourceId = plugin.name
@@ -103,7 +103,7 @@ export async function getIntegration(integration: string) {
     return INTEGRATIONS[integration]
   }
   if (env.SELF_HOSTED) {
-    const plugins = await getPlugins(PluginType.DATASOURCE)
+    const plugins = await sdk.plugins.fetch(PluginType.DATASOURCE)
     for (let plugin of plugins) {
       if (plugin.name === integration) {
         // need to use commonJS require due to its dynamic runtime nature

package/src/integrations/microsoftSqlServer.ts

@@ -243,11 +243,14 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
         if (typeof name !== "string") {
           continue
         }
+        const hasDefault = def.COLUMN_DEFAULT
+        const isAuto = !!autoColumns.find(col => col === name)
+        const required = !!requiredColumns.find(col => col === name)
         schema[name] = {
-          autocolumn: !!autoColumns.find(col => col === name),
+          autocolumn: isAuto,
           name: name,
           constraints: {
-            presence: requiredColumns.find(col => col === name),
+            presence: required && !isAuto && !hasDefault,
           },
           ...convertSqlType(def.DATA_TYPE),
           externalType: def.DATA_TYPE,

package/src/integrations/mysql.ts

@@ -229,13 +229,15 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
           if (column.Key === "PRI" && primaryKeys.indexOf(column.Key) === -1) {
             primaryKeys.push(columnName)
           }
-          const constraints = {
-            presence: column.Null !== "YES",
-          }
+          const hasDefault = column.Default != null
           const isAuto: boolean =
             typeof column.Extra === "string" &&
             (column.Extra === "auto_increment" ||
               column.Extra.toLowerCase().includes("generated"))
+          const required = column.Null !== "YES"
+          const constraints = {
+            presence: required && !isAuto && !hasDefault,
+          }
           schema[columnName] = {
             name: columnName,
             autocolumn: isAuto,

package/src/integrations/postgres.ts

@@ -262,15 +262,17 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
           column.identity_start ||
           column.identity_increment
         )
-        const constraints = {
-          presence: column.is_nullable === "NO",
-        }
-        const hasDefault =
+        const hasDefault = column.column_default != null
+        const hasNextVal =
           typeof column.column_default === "string" &&
           column.column_default.startsWith("nextval")
         const isGenerated =
           column.is_generated && column.is_generated !== "NEVER"
-        const isAuto: boolean = hasDefault || identity || isGenerated
+        const isAuto: boolean = hasNextVal || identity || isGenerated
+        const required = column.is_nullable === "NO"
+        const constraints = {
+          presence: required && !hasDefault && !isGenerated,
+        }
         tables[tableName].schema[columnName] = {
           autocolumn: isAuto,
           name: columnName,

package/src/integrations/redis.ts

@@ -6,6 +6,7 @@ interface RedisConfig {
   port: number
   username: string
   password?: string
+  db?: number
 }
 
 const SCHEMA: Integration = {
@@ -32,6 +33,12 @@ const SCHEMA: Integration = {
       type: "password",
       required: false,
     },
+    db: {
+      type: "number",
+      required: false,
+      display: "DB",
+      default: 0,
+    },
   },
   query: {
     create: {
@@ -88,6 +95,7 @@ class RedisIntegration {
       port: this.config.port,
       username: this.config.username,
       password: this.config.password,
+      db: this.config.db,
     })
   }
 

package/src/integrations/rest.ts

@@ -151,6 +151,9 @@ class RestIntegration implements IntegrationBase {
           data = data[keys[0]]
         }
         raw = rawXml
+      } else if (contentType.includes("application/pdf")) {
+        data = await response.arrayBuffer() // Save PDF as ArrayBuffer
+        raw = Buffer.from(data)
       } else {
         data = await response.text()
         raw = data

package/src/migrations/functions/syncQuotas.ts

@@ -2,6 +2,7 @@ import { runQuotaMigration } from "./usageQuotas"
 import * as syncApps from "./usageQuotas/syncApps"
 import * as syncRows from "./usageQuotas/syncRows"
 import * as syncPlugins from "./usageQuotas/syncPlugins"
+import * as syncUsers from "./usageQuotas/syncUsers"
 
 /**
  * Synchronise quotas to the state of the db.
@@ -11,5 +12,6 @@ export const run = async () => {
     await syncApps.run()
     await syncRows.run()
     await syncPlugins.run()
+    await syncUsers.run()
   })
 }

package/src/migrations/functions/usageQuotas/syncApps.ts

@@ -1,4 +1,4 @@
-import { tenancy, db as dbCore } from "@budibase/backend-core"
+import { db as dbCore } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
 
@@ -8,7 +8,6 @@ export const run = async () => {
   const appCount = devApps ? devApps.length : 0
 
   // sync app count
-  const tenantId = tenancy.getTenantId()
-  console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
+  console.log(`Syncing app count: ${appCount}`)
   await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC)
 }

package/src/migrations/functions/usageQuotas/syncRows.ts

@@ -1,4 +1,4 @@
-import { tenancy, db as dbCore } from "@budibase/backend-core"
+import { db as dbCore } from "@budibase/backend-core"
 import { getUniqueRows } from "../../../utilities/usageQuota/rows"
 import { quotas } from "@budibase/pro"
 import { StaticQuotaName, QuotaUsageType, App } from "@budibase/types"
@@ -18,8 +18,7 @@ export const run = async () => {
   })
 
   // sync row count
-  const tenantId = tenancy.getTenantId()
-  console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
+  console.log(`Syncing row count: ${rowCount}`)
   await quotas.setUsagePerApp(
     counts,
     StaticQuotaName.ROWS,

package/src/migrations/functions/usageQuotas/syncUsers.ts

@@ -0,0 +1,9 @@
+import { users } from "@budibase/backend-core"
+import { quotas } from "@budibase/pro"
+import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
+
+export const run = async () => {
+  const userCount = await users.getUserCount()
+  console.log(`Syncing user count: ${userCount}`)
+  await quotas.setUsage(userCount, StaticQuotaName.USERS, QuotaUsageType.STATIC)
+}

package/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts

@@ -24,7 +24,7 @@ describe("syncRows", () => {
 
       // app 1
       const app1 = config.app
-      await context.doInAppContext(app1.appId, async () => {
+      await context.doInAppContext(app1!.appId, async () => {
         await config.createTable()
         await config.createRow()
       })
@@ -43,7 +43,7 @@ describe("syncRows", () => {
       usageDoc = await quotas.getQuotaUsage()
       expect(usageDoc.usageQuota.rows).toEqual(3)
       expect(
-        usageDoc.apps?.[dbCore.getProdAppID(app1.appId)].usageQuota.rows
+        usageDoc.apps?.[dbCore.getProdAppID(app1!.appId)].usageQuota.rows
       ).toEqual(1)
       expect(
         usageDoc.apps?.[dbCore.getProdAppID(app2.appId)].usageQuota.rows

package/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts

@@ -0,0 +1,26 @@
+import TestConfig from "../../../../tests/utilities/TestConfiguration"
+import * as syncUsers from "../syncUsers"
+import { quotas } from "@budibase/pro"
+
+describe("syncUsers", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)
+
+  it("syncs users", async () => {
+    return config.doInContext(null, async () => {
+      await config.createUser()
+
+      await syncUsers.run()
+
+      const usageDoc = await quotas.getQuotaUsage()
+      // default + additional user
+      const userCount = 2
+      expect(usageDoc.usageQuota.users).toBe(userCount)
+    })
+  })
+})

package/src/migrations/index.ts

@@ -11,6 +11,7 @@ import env from "../environment"
 // migration functions
 import * as userEmailViewCasing from "./functions/userEmailViewCasing"
 import * as syncQuotas from "./functions/syncQuotas"
+import * as syncUsers from "./functions/usageQuotas/syncUsers"
 import * as appUrls from "./functions/appUrls"
 import * as tableSettings from "./functions/tableSettings"
 import * as backfill from "./functions/backfill"