@budibase/server 2.5.6-alpha.4 → 2.5.6-alpha.41

package/src/api/controllers/row/ExternalRequest.ts

@@ -1,31 +1,32 @@
 import {
+  Datasource,
+  FieldSchema,
+  FieldType,
   FilterType,
   IncludeRelationship,
   Operation,
   PaginationJson,
   RelationshipsJson,
+  RelationshipTypes,
+  Row,
   SearchFilters,
   SortJson,
-  Datasource,
-  FieldSchema,
-  Row,
-  Table,
-  RelationshipTypes,
-  FieldType,
   SortType,
+  Table,
 } from "@budibase/types"
 import {
+  breakExternalTableId,
   breakRowIdField,
+  convertRowId,
   generateRowIdField,
   isRowId,
-  convertRowId,
+  isSQL,
 } from "../../../integrations/utils"
 import { getDatasourceAndQuery } from "./utils"
 import { FieldTypes } from "../../../constants"
-import { breakExternalTableId, isSQL } from "../../../integrations/utils"
 import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
-import { processFormulas, processDates } from "../../../utilities/rowProcessor"
+import { processDates, processFormulas } from "../../../utilities/rowProcessor"
 import { db as dbCore } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 
@@ -382,10 +383,18 @@ export class ExternalRequest {
       }
       const display = linkedTable.primaryDisplay
       for (let key of Object.keys(row[relationship.column])) {
-        const related: Row = row[relationship.column][key]
+        let relatedRow: Row = row[relationship.column][key]
+        // add this row as context for the relationship
+        for (let col of Object.values(linkedTable.schema)) {
+          if (col.type === FieldType.LINK && col.tableId === table._id) {
+            relatedRow[col.name] = [row]
+          }
+        }
+        relatedRow = processFormulas(linkedTable, relatedRow)
+        const relatedDisplay = display ? relatedRow[display] : undefined
         row[relationship.column][key] = {
-          primaryDisplay: display ? related[display] : undefined,
-          _id: related._id,
+          primaryDisplay: relatedDisplay || "Invalid display column",
+          _id: relatedRow._id,
         }
       }
     }

package/src/api/controllers/table/utils.ts

@@ -110,21 +110,28 @@ export function importToRows(
   table: Table,
   user: ContextUser | null = null
 ) {
+  let originalTable = table
   let finalData: any = []
   for (let i = 0; i < data.length; i++) {
     let row = data[i]
     row._id = generateRowID(table._id!)
     row.tableId = table._id
+
+    // We use a reference to table here and update it after input processing,
+    // so that we can auto increment auto IDs in imported data properly
     const processed = inputProcessing(user, table, row, {
       noAutoRelationships: true,
     })
     row = processed.row
     table = processed.table
 
-    for (const [fieldName, schema] of Object.entries(table.schema)) {
-      // check whether the options need to be updated for inclusion as part of the data import
+    // However here we must reference the original table, as we want to mutate
+    // the real schema of the table passed in, not the clone used for
+    // incrementing auto IDs
+    for (const [fieldName, schema] of Object.entries(originalTable.schema)) {
       if (
-        schema.type === FieldTypes.OPTIONS &&
+        (schema.type === FieldTypes.OPTIONS ||
+          schema.type === FieldTypes.ARRAY) &&
         row[fieldName] &&
         (!schema.constraints!.inclusion ||
           schema.constraints!.inclusion.indexOf(row[fieldName]) === -1)

package/src/api/index.ts

@@ -1,10 +1,8 @@
 import Router from "@koa/router"
-import { auth, middleware } from "@budibase/backend-core"
+import { auth, middleware, env as envCore } from "@budibase/backend-core"
 import currentApp from "../middleware/currentapp"
 import zlib from "zlib"
 import { mainRoutes, staticRoutes, publicRoutes } from "./routes"
-import pkg from "../../package.json"
-import env from "../environment"
 import { middleware as pro } from "@budibase/pro"
 export { shutdown } from "./routes/public"
 const compress = require("koa-compress")
@@ -12,7 +10,7 @@ const compress = require("koa-compress")
 export const router: Router = new Router()
 
 router.get("/health", ctx => (ctx.status = 200))
-router.get("/version", ctx => (ctx.body = pkg.version))
+router.get("/version", ctx => (ctx.body = envCore.VERSION))
 
 router.use(middleware.errorHandling)
 

package/src/api/routes/index.ts

@@ -22,7 +22,6 @@ import queryRoutes from "./query"
 import backupRoutes from "./backup"
 import metadataRoutes from "./metadata"
 import devRoutes from "./dev"
-import cloudRoutes from "./cloud"
 import migrationRoutes from "./migrations"
 import pluginRoutes from "./plugin"
 import opsRoutes from "./ops"
@@ -60,7 +59,6 @@ export const mainRoutes: Router[] = [
   queryRoutes,
   metadataRoutes,
   devRoutes,
-  cloudRoutes,
   rowRoutes,
   migrationRoutes,
   pluginRoutes,

package/src/api/routes/tests/automation.spec.js

@@ -7,7 +7,7 @@ const {
 const setup = require("./utilities")
 const { basicAutomation, newAutomation, automationTrigger, automationStep } = setup.structures
 const MAX_RETRIES = 4
-const { TRIGGER_DEFINITIONS, ACTION_DEFINITIONS } = require("../../../automations")
+const { TRIGGER_DEFINITIONS, BUILTIN_ACTION_DEFINITIONS } = require("../../../automations")
 const { events } = require("@budibase/backend-core")
 
 
@@ -55,7 +55,7 @@ describe("/automations", () => {
         .expect('Content-Type', /json/)
         .expect(200)
 
-      let definitionsLength = Object.keys(ACTION_DEFINITIONS).length
+      let definitionsLength = Object.keys(BUILTIN_ACTION_DEFINITIONS).length
       definitionsLength-- // OUTGOING_WEBHOOK is deprecated
 
       expect(Object.keys(res.body.action).length).toBeGreaterThanOrEqual(definitionsLength)

package/src/app.ts

@@ -15,8 +15,8 @@ import * as api from "./api"
 import * as automations from "./automations"
 import { Thread } from "./threads"
 import * as redis from "./utilities/redis"
+import { initialise as initialiseWebsockets } from "./websockets"
 import { events, logging, middleware, timers } from "@budibase/backend-core"
-import { initialise as initialiseWebsockets } from "./websocket"
 import { startup } from "./startup"
 const Sentry = require("@sentry/node")
 const destroyable = require("server-destroy")
@@ -61,7 +61,7 @@ if (env.isProd()) {
 
 const server = http.createServer(app.callback())
 destroyable(server)
-initialiseWebsockets(server)
+initialiseWebsockets(app, server)
 
 let shuttingDown = false,
   errCode = 0

package/src/automations/actions.ts

@@ -15,7 +15,14 @@ import * as delay from "./steps/delay"
 import * as queryRow from "./steps/queryRows"
 import * as loop from "./steps/loop"
 import env from "../environment"
-import { AutomationStepSchema, AutomationStepInput } from "@budibase/types"
+import {
+  AutomationStepSchema,
+  AutomationStepInput,
+  PluginType,
+  AutomationStep,
+} from "@budibase/types"
+import sdk from "../sdk"
+import { getAutomationPlugin } from "../utilities/fileSystem"
 
 const ACTION_IMPLS: Record<
   string,
@@ -38,25 +45,26 @@ const ACTION_IMPLS: Record<
   zapier: zapier.run,
   integromat: integromat.run,
 }
-export const ACTION_DEFINITIONS: Record<string, AutomationStepSchema> = {
-  SEND_EMAIL_SMTP: sendSmtpEmail.definition,
-  CREATE_ROW: createRow.definition,
-  UPDATE_ROW: updateRow.definition,
-  DELETE_ROW: deleteRow.definition,
-  OUTGOING_WEBHOOK: outgoingWebhook.definition,
-  EXECUTE_SCRIPT: executeScript.definition,
-  EXECUTE_QUERY: executeQuery.definition,
-  SERVER_LOG: serverLog.definition,
-  DELAY: delay.definition,
-  FILTER: filter.definition,
-  QUERY_ROWS: queryRow.definition,
-  LOOP: loop.definition,
-  // these used to be lowercase step IDs, maintain for backwards compat
-  discord: discord.definition,
-  slack: slack.definition,
-  zapier: zapier.definition,
-  integromat: integromat.definition,
-}
+export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
+  {
+    SEND_EMAIL_SMTP: sendSmtpEmail.definition,
+    CREATE_ROW: createRow.definition,
+    UPDATE_ROW: updateRow.definition,
+    DELETE_ROW: deleteRow.definition,
+    OUTGOING_WEBHOOK: outgoingWebhook.definition,
+    EXECUTE_SCRIPT: executeScript.definition,
+    EXECUTE_QUERY: executeQuery.definition,
+    SERVER_LOG: serverLog.definition,
+    DELAY: delay.definition,
+    FILTER: filter.definition,
+    QUERY_ROWS: queryRow.definition,
+    LOOP: loop.definition,
+    // these used to be lowercase step IDs, maintain for backwards compat
+    discord: discord.definition,
+    slack: slack.definition,
+    zapier: zapier.definition,
+    integromat: integromat.definition,
+  }
 
 // don't add the bash script/definitions unless in self host
 // the fact this isn't included in any definitions means it cannot be
@@ -66,12 +74,36 @@ if (env.SELF_HOSTED) {
   // @ts-ignore
   ACTION_IMPLS["EXECUTE_BASH"] = bash.run
   // @ts-ignore
-  ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
+  BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
+}
+
+export async function getActionDefinitions() {
+  const actionDefinitions = BUILTIN_ACTION_DEFINITIONS
+  if (env.SELF_HOSTED) {
+    const plugins = await sdk.plugins.fetch(PluginType.AUTOMATION)
+    for (let plugin of plugins) {
+      const schema = plugin.schema.schema as AutomationStep
+      actionDefinitions[schema.stepId] = {
+        ...schema,
+        custom: true,
+      }
+    }
+  }
+  return actionDefinitions
 }
 
 /* istanbul ignore next */
-export async function getAction(actionName: string) {
-  if (ACTION_IMPLS[actionName] != null) {
-    return ACTION_IMPLS[actionName]
+export async function getAction(stepId: string) {
+  if (ACTION_IMPLS[stepId] != null) {
+    return ACTION_IMPLS[stepId]
+  }
+  // must be a plugin
+  if (env.SELF_HOSTED) {
+    const plugins = await sdk.plugins.fetch(PluginType.AUTOMATION)
+    const found = plugins.find(plugin => plugin.schema.schema.stepId === stepId)
+    if (!found) {
+      throw new Error(`Unable to find action implementation for "${stepId}"`)
+    }
+    return (await getAutomationPlugin(found)).action
   }
 }

package/src/automations/index.ts

@@ -6,7 +6,7 @@ import BullQueue from "bull"
 export { automationQueue } from "./bullboard"
 export { shutdown } from "./bullboard"
 export { TRIGGER_DEFINITIONS } from "./triggers"
-export { ACTION_DEFINITIONS } from "./actions"
+export { BUILTIN_ACTION_DEFINITIONS, getActionDefinitions } from "./actions"
 
 /**
  * This module is built purely to kick off the worker farm and manage the inputs/outputs

package/src/automations/steps/bash.ts

@@ -4,8 +4,11 @@ import * as automationUtils from "../automationUtils"
 import environment from "../../environment"
 import {
   AutomationActionStepId,
-  AutomationStepSchema,
+  AutomationCustomIOType,
+  AutomationIOType,
   AutomationStepInput,
+  AutomationStepSchema,
+  AutomationStepType,
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
@@ -13,7 +16,7 @@ export const definition: AutomationStepSchema = {
   tagline: "Execute a bash command",
   icon: "JourneyEvent",
   description: "Run a bash script",
-  type: "ACTION",
+  type: AutomationStepType.ACTION,
   internal: true,
   stepId: AutomationActionStepId.EXECUTE_BASH,
   inputs: {},
@@ -21,8 +24,8 @@ export const definition: AutomationStepSchema = {
     inputs: {
       properties: {
         code: {
-          type: "string",
-          customType: "code",
+          type: AutomationIOType.STRING,
+          customType: AutomationCustomIOType.CODE,
           title: "Code",
         },
       },
@@ -31,16 +34,16 @@ export const definition: AutomationStepSchema = {
     outputs: {
       properties: {
         stdout: {
-          type: "string",
+          type: AutomationIOType.STRING,
           description: "Standard output of your bash command or script",
         },
         success: {
-          type: "boolean",
+          type: AutomationIOType.BOOLEAN,
           description: "Whether the command was successful",
         },
       },
+      required: ["stdout"],
     },
-    required: ["stdout"],
   },
 }
 

package/src/automations/steps/createRow.ts

@@ -3,8 +3,11 @@ import { cleanUpRow, getError } from "../automationUtils"
 import { buildCtx } from "./utils"
 import {
   AutomationActionStepId,
-  AutomationStepSchema,
+  AutomationCustomIOType,
+  AutomationIOType,
   AutomationStepInput,
+  AutomationStepSchema,
+  AutomationStepType,
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
@@ -12,7 +15,7 @@ export const definition: AutomationStepSchema = {
   tagline: "Create a {{inputs.enriched.table.name}} row",
   icon: "TableRowAddBottom",
   description: "Add a row to your database",
-  type: "ACTION",
+  type: AutomationStepType.ACTION,
   internal: true,
   stepId: AutomationActionStepId.CREATE_ROW,
   inputs: {},
@@ -20,14 +23,14 @@ export const definition: AutomationStepSchema = {
     inputs: {
       properties: {
         row: {
-          type: "object",
+          type: AutomationIOType.OBJECT,
           properties: {
             tableId: {
-              type: "string",
-              customType: "table",
+              type: AutomationIOType.STRING,
+              customType: AutomationCustomIOType.TABLE,
             },
           },
-          customType: "row",
+          customType: AutomationCustomIOType.ROW,
           title: "Table",
           required: ["tableId"],
         },
@@ -37,24 +40,24 @@ export const definition: AutomationStepSchema = {
     outputs: {
       properties: {
         row: {
-          type: "object",
-          customType: "row",
+          type: AutomationIOType.OBJECT,
+          customType: AutomationCustomIOType.ROW,
           description: "The new row",
         },
         response: {
-          type: "object",
+          type: AutomationIOType.OBJECT,
           description: "The response from the table",
         },
         success: {
-          type: "boolean",
+          type: AutomationIOType.BOOLEAN,
           description: "Whether the row creation was successful",
         },
         id: {
-          type: "string",
+          type: AutomationIOType.STRING,
           description: "The identifier of the new row",
         },
         revision: {
-          type: "string",
+          type: AutomationIOType.STRING,
           description: "The revision of the new row",
         },
       },

package/src/automations/steps/delay.ts

@@ -1,8 +1,10 @@
 import { wait } from "../../utilities"
 import {
   AutomationActionStepId,
-  AutomationStepSchema,
+  AutomationIOType,
   AutomationStepInput,
+  AutomationStepSchema,
+  AutomationStepType,
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
@@ -17,7 +19,7 @@ export const definition: AutomationStepSchema = {
     inputs: {
       properties: {
         time: {
-          type: "number",
+          type: AutomationIOType.NUMBER,
           title: "Delay in milliseconds",
         },
       },
@@ -26,14 +28,14 @@ export const definition: AutomationStepSchema = {
     outputs: {
       properties: {
         success: {
-          type: "boolean",
+          type: AutomationIOType.BOOLEAN,
           description: "Whether the delay was successful",
         },
       },
       required: ["success"],
     },
   },
-  type: "LOGIC",
+  type: AutomationStepType.LOGIC,
 }
 
 export async function run({ inputs }: AutomationStepInput) {

package/src/automations/steps/deleteRow.ts

@@ -3,8 +3,11 @@ import { buildCtx } from "./utils"
 import { getError } from "../automationUtils"
 import {
   AutomationActionStepId,
-  AutomationStepSchema,
   AutomationStepInput,
+  AutomationStepSchema,
+  AutomationStepType,
+  AutomationIOType,
+  AutomationCustomIOType,
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
@@ -12,7 +15,7 @@ export const definition: AutomationStepSchema = {
   icon: "TableRowRemoveCenter",
   name: "Delete Row",
   tagline: "Delete a {{inputs.enriched.table.name}} row",
-  type: "ACTION",
+  type: AutomationStepType.ACTION,
   stepId: AutomationActionStepId.DELETE_ROW,
   internal: true,
   inputs: {},
@@ -20,12 +23,12 @@ export const definition: AutomationStepSchema = {
     inputs: {
       properties: {
         tableId: {
-          type: "string",
-          customType: "table",
+          type: AutomationIOType.STRING,
+          customType: AutomationCustomIOType.TABLE,
           title: "Table",
         },
         id: {
-          type: "string",
+          type: AutomationIOType.STRING,
           title: "Row ID",
         },
       },
@@ -34,16 +37,16 @@ export const definition: AutomationStepSchema = {
     outputs: {
       properties: {
         row: {
-          type: "object",
-          customType: "row",
+          type: AutomationIOType.OBJECT,
+          customType: AutomationCustomIOType.ROW,
           description: "The deleted row",
         },
         response: {
-          type: "object",
+          type: AutomationIOType.OBJECT,
           description: "The response from the table",
         },
         success: {
-          type: "boolean",
+          type: AutomationIOType.BOOLEAN,
           description: "Whether the deletion was successful",
         },
       },

package/src/automations/steps/discord.ts

@@ -4,6 +4,8 @@ import {
   AutomationActionStepId,
   AutomationStepSchema,
   AutomationStepInput,
+  AutomationStepType,
+  AutomationIOType,
 } from "@budibase/types"
 
 const DEFAULT_USERNAME = "Budibase Automate"
@@ -15,26 +17,26 @@ export const definition: AutomationStepSchema = {
   description: "Send a message to a Discord server",
   icon: "ri-discord-line",
   stepId: AutomationActionStepId.discord,
-  type: "ACTION",
+  type: AutomationStepType.ACTION,
   internal: false,
   inputs: {},
   schema: {
     inputs: {
       properties: {
         url: {
-          type: "string",
+          type: AutomationIOType.STRING,
           title: "Discord Webhook URL",
         },
         username: {
-          type: "string",
+          type: AutomationIOType.STRING,
           title: "Bot Name",
         },
         avatar_url: {
-          type: "string",
+          type: AutomationIOType.STRING,
           title: "Bot Avatar URL",
         },
         content: {
-          type: "string",
+          type: AutomationIOType.STRING,
           title: "Message",
         },
       },
@@ -43,15 +45,15 @@ export const definition: AutomationStepSchema = {
     outputs: {
       properties: {
         httpStatus: {
-          type: "number",
+          type: AutomationIOType.NUMBER,
           description: "The HTTP status code of the request",
         },
         response: {
-          type: "string",
+          type: AutomationIOType.STRING,
           description: "The response from the Discord Webhook",
         },
         success: {
-          type: "boolean",
+          type: AutomationIOType.BOOLEAN,
           description: "Whether the message sent successfully",
         },
       },

package/src/automations/steps/executeQuery.ts

@@ -3,8 +3,11 @@ import { buildCtx } from "./utils"
 import * as automationUtils from "../automationUtils"
 import {
   AutomationActionStepId,
-  AutomationStepSchema,
+  AutomationCustomIOType,
+  AutomationIOType,
   AutomationStepInput,
+  AutomationStepSchema,
+  AutomationStepType,
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
@@ -12,7 +15,7 @@ export const definition: AutomationStepSchema = {
   tagline: "Execute Data Connector",
   icon: "Data",
   description: "Execute a query in an external data connector",
-  type: "ACTION",
+  type: AutomationStepType.ACTION,
   stepId: AutomationActionStepId.EXECUTE_QUERY,
   internal: true,
   inputs: {},
@@ -20,14 +23,14 @@ export const definition: AutomationStepSchema = {
     inputs: {
       properties: {
         query: {
-          type: "object",
+          type: AutomationIOType.OBJECT,
           properties: {
             queryId: {
-              type: "string",
-              customType: "query",
+              type: AutomationIOType.STRING,
+              customType: AutomationCustomIOType.QUERY,
             },
           },
-          customType: "queryParams",
+          customType: AutomationCustomIOType.QUERY_PARAMS,
           title: "Parameters",
           required: ["queryId"],
         },
@@ -37,21 +40,21 @@ export const definition: AutomationStepSchema = {
     outputs: {
       properties: {
         response: {
-          type: "object",
+          type: AutomationIOType.OBJECT,
           description: "The response from the datasource execution",
         },
         info: {
-          type: "object",
+          type: AutomationIOType.OBJECT,
           description:
             "Some query types may return extra data, like headers from a REST query",
         },
         success: {
-          type: "boolean",
+          type: AutomationIOType.BOOLEAN,
           description: "Whether the action was successful",
         },
       },
+      required: ["response", "success"],
     },
-    required: ["response", "success"],
   },
 }