@budibase/server 2.6.16-alpha.5 → 2.6.17

package/jest.config.ts

@@ -20,9 +20,9 @@ const baseConfig: Config.InitialProjectOptions = {
 }
 
 // add pro sources if they exist
-if (fs.existsSync("../pro/packages")) {
-  baseConfig.moduleNameMapper!["@budibase/pro"] =
-    "<rootDir>/../pro/packages/pro/src"
+if (fs.existsSync("../../../budibase-pro")) {
+  baseConfig.moduleNameMapper["@budibase/pro"] =
+    "<rootDir>/../../../budibase-pro/packages/pro/src"
 }
 
 const config: Config.InitialOptions = {

package/nodemon.json

@@ -1,10 +1,6 @@
 {
-  "watch": ["src", "../backend-core", "../pro/packages/pro"],
+  "watch": ["src", "../backend-core", "../../../budibase-pro/packages/pro"],
   "ext": "js,ts,json",
-  "ignore": [
-    "src/**/*.spec.ts",
-    "src/**/*.spec.js",
-    "../backend-core/dist/**/*"
-  ],
+  "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../backend-core/dist/**/*"],
   "exec": "ts-node src/index.ts"
-}
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.6.16-alpha.5",
+  "version": "2.6.17",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -45,12 +45,12 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.6.16-alpha.5",
-    "@budibase/client": "2.6.16-alpha.5",
-    "@budibase/pro": "2.6.16-alpha.5",
-    "@budibase/shared-core": "2.6.16-alpha.5",
-    "@budibase/string-templates": "2.6.16-alpha.5",
-    "@budibase/types": "2.6.16-alpha.5",
+    "@budibase/backend-core": "^2.6.17",
+    "@budibase/client": "^2.6.17",
+    "@budibase/pro": "2.6.16",
+    "@budibase/shared-core": "^2.6.17",
+    "@budibase/string-templates": "^2.6.17",
+    "@budibase/types": "^2.6.17",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -99,7 +99,7 @@
     "mysql2": "2.3.3",
     "node-fetch": "2.6.7",
     "open": "8.4.0",
-    "pg": "8.10.0",
+    "pg": "8.5.1",
     "posthog-node": "1.3.0",
     "pouchdb": "7.3.0",
     "pouchdb-adapter-memory": "7.2.2",
@@ -141,7 +141,6 @@
     "@types/node": "14.18.20",
     "@types/node-fetch": "2.6.1",
     "@types/oracledb": "5.2.2",
-    "@types/pg": "8.6.6",
     "@types/pouchdb": "6.4.0",
     "@types/redis": "4.0.11",
     "@types/server-destroy": "1.0.1",
@@ -177,5 +176,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "bbb516f26ae966668b5fa65418d81fbccf3dce37"
+  "gitHead": "2a0993fe5e3cab1ec779603cc9228f824e2cea4d"
 }

package/src/api/controllers/datasource.ts

@@ -18,71 +18,11 @@ import {
   Row,
   CreateDatasourceResponse,
   UpdateDatasourceResponse,
+  UpdateDatasourceRequest,
   CreateDatasourceRequest,
-  VerifyDatasourceRequest,
-  VerifyDatasourceResponse,
-  IntegrationBase,
-  DatasourcePlus,
 } from "@budibase/types"
 import sdk from "../../sdk"
 
-function getErrorTables(errors: any, errorType: string) {
-  return Object.entries(errors)
-    .filter(entry => entry[1] === errorType)
-    .map(([name]) => name)
-}
-
-function updateError(error: any, newError: any, tables: string[]) {
-  if (!error) {
-    error = ""
-  }
-  if (error.length > 0) {
-    error += "\n"
-  }
-  error += `${newError} ${tables.join(", ")}`
-  return error
-}
-
-async function getConnector(
-  datasource: Datasource
-): Promise<IntegrationBase | DatasourcePlus> {
-  const Connector = await getIntegration(datasource.source)
-  // can't enrich if it doesn't have an ID yet
-  if (datasource._id) {
-    datasource = await sdk.datasources.enrich(datasource)
-  }
-  // Connect to the DB and build the schema
-  return new Connector(datasource.config)
-}
-
-async function buildSchemaHelper(datasource: Datasource) {
-  const connector = (await getConnector(datasource)) as DatasourcePlus
-  await connector.buildSchema(datasource._id!, datasource.entities!)
-
-  const errors = connector.schemaErrors
-  let error = null
-  if (errors && Object.keys(errors).length > 0) {
-    const noKey = getErrorTables(errors, BuildSchemaErrors.NO_KEY)
-    const invalidCol = getErrorTables(errors, BuildSchemaErrors.INVALID_COLUMN)
-    if (noKey.length) {
-      error = updateError(
-        error,
-        "No primary key constraint found for the following:",
-        noKey
-      )
-    }
-    if (invalidCol.length) {
-      const invalidCols = Object.values(InvalidColumns).join(", ")
-      error = updateError(
-        error,
-        `Cannot use columns ${invalidCols} found in following:`,
-        invalidCol
-      )
-    }
-  }
-  return { tables: connector.tables, error }
-}
-
 export async function fetch(ctx: UserCtx) {
   // Get internal tables
   const db = context.getAppDB()
@@ -126,33 +66,6 @@ export async function fetch(ctx: UserCtx) {
   ctx.body = [bbInternalDb, ...datasources]
 }
 
-export async function verify(
-  ctx: UserCtx<VerifyDatasourceRequest, VerifyDatasourceResponse>
-) {
-  const { datasource } = ctx.request.body
-  let existingDatasource: undefined | Datasource
-  if (datasource._id) {
-    existingDatasource = await sdk.datasources.get(datasource._id)
-  }
-  let enrichedDatasource = datasource
-  if (existingDatasource) {
-    enrichedDatasource = sdk.datasources.mergeConfigs(
-      datasource,
-      existingDatasource
-    )
-  }
-  const connector = await getConnector(enrichedDatasource)
-  if (!connector.testConnection) {
-    ctx.throw(400, "Connection information verification not supported")
-  }
-  const response = await connector.testConnection()
-
-  ctx.body = {
-    connected: response.connected,
-    error: response.error,
-  }
-}
-
 export async function buildSchemaFromDb(ctx: UserCtx) {
   const db = context.getAppDB()
   const datasource = await sdk.datasources.get(ctx.params.datasourceId)
@@ -398,3 +311,51 @@ export async function query(ctx: UserCtx) {
     ctx.throw(400, err)
   }
 }
+
+function getErrorTables(errors: any, errorType: string) {
+  return Object.entries(errors)
+    .filter(entry => entry[1] === errorType)
+    .map(([name]) => name)
+}
+
+function updateError(error: any, newError: any, tables: string[]) {
+  if (!error) {
+    error = ""
+  }
+  if (error.length > 0) {
+    error += "\n"
+  }
+  error += `${newError} ${tables.join(", ")}`
+  return error
+}
+
+async function buildSchemaHelper(datasource: Datasource) {
+  const Connector = await getIntegration(datasource.source)
+  datasource = await sdk.datasources.enrich(datasource)
+  // Connect to the DB and build the schema
+  const connector = new Connector(datasource.config)
+  await connector.buildSchema(datasource._id, datasource.entities)
+
+  const errors = connector.schemaErrors
+  let error = null
+  if (errors && Object.keys(errors).length > 0) {
+    const noKey = getErrorTables(errors, BuildSchemaErrors.NO_KEY)
+    const invalidCol = getErrorTables(errors, BuildSchemaErrors.INVALID_COLUMN)
+    if (noKey.length) {
+      error = updateError(
+        error,
+        "No primary key constraint found for the following:",
+        noKey
+      )
+    }
+    if (invalidCol.length) {
+      const invalidCols = Object.values(InvalidColumns).join(", ")
+      error = updateError(
+        error,
+        `Cannot use columns ${invalidCols} found in following:`,
+        invalidCol
+      )
+    }
+  }
+  return { tables: connector.tables, error }
+}

package/src/api/controllers/integration.ts

@@ -1,4 +1,4 @@
-import { getDefinition, getDefinitions } from "../../integrations"
+import { getDefinitions } from "../../integrations"
 import { BBContext } from "@budibase/types"
 
 export async function fetch(ctx: BBContext) {
@@ -7,7 +7,7 @@ export async function fetch(ctx: BBContext) {
 }
 
 export async function find(ctx: BBContext) {
-  const def = await getDefinition(ctx.params.type)
-  ctx.body = def
+  const defs = await getDefinitions()
   ctx.status = 200
+  ctx.body = defs[ctx.params.type]
 }

package/src/api/routes/datasource.ts

@@ -15,11 +15,6 @@ router
     authorized(permissions.BUILDER),
     datasourceController.fetch
   )
-  .post(
-    "/api/datasources/verify",
-    authorized(permissions.BUILDER),
-    datasourceController.verify
-  )
   .get(
     "/api/datasources/:datasourceId",
     authorized(

package/src/automations/steps/make.ts

@@ -26,10 +26,6 @@ export const definition: AutomationStepSchema = {
           type: AutomationIOType.STRING,
           title: "Webhook URL",
         },
-        body: {
-          type: AutomationIOType.JSON,
-          title: "Payload",
-        },
         value1: {
           type: AutomationIOType.STRING,
           title: "Input Value 1",
@@ -74,19 +70,7 @@ export const definition: AutomationStepSchema = {
 }
 
 export async function run({ inputs }: AutomationStepInput) {
-  //TODO - Remove deprecated values 1,2,3,4,5 after November 2023
-  const { url, value1, value2, value3, value4, value5, body } = inputs
-
-  let payload = {}
-  try {
-    payload = body?.value ? JSON.parse(body?.value) : {}
-  } catch (err) {
-    return {
-      httpStatus: 400,
-      response: "Invalid payload JSON",
-      success: false,
-    }
-  }
+  const { url, value1, value2, value3, value4, value5 } = inputs
 
   if (!url?.trim()?.length) {
     return {
@@ -105,7 +89,6 @@ export async function run({ inputs }: AutomationStepInput) {
         value3,
         value4,
         value5,
-        ...payload,
       }),
       headers: {
         "Content-Type": "application/json",

package/src/automations/steps/zapier.ts

@@ -24,10 +24,6 @@ export const definition: AutomationStepSchema = {
           type: AutomationIOType.STRING,
           title: "Webhook URL",
         },
-        body: {
-          type: AutomationIOType.JSON,
-          title: "Payload",
-        },
         value1: {
           type: AutomationIOType.STRING,
           title: "Payload Value 1",
@@ -67,19 +63,7 @@ export const definition: AutomationStepSchema = {
 }
 
 export async function run({ inputs }: AutomationStepInput) {
-  //TODO - Remove deprecated values 1,2,3,4,5 after November 2023
-  const { url, value1, value2, value3, value4, value5, body } = inputs
-
-  let payload = {}
-  try {
-    payload = body?.value ? JSON.parse(body?.value) : {}
-  } catch (err) {
-    return {
-      httpStatus: 400,
-      response: "Invalid payload JSON",
-      success: false,
-    }
-  }
+  const { url, value1, value2, value3, value4, value5 } = inputs
 
   if (!url?.trim()?.length) {
     return {
@@ -101,7 +85,6 @@ export async function run({ inputs }: AutomationStepInput) {
         value3,
         value4,
         value5,
-        ...payload,
       }),
       headers: {
         "Content-Type": "application/json",

package/src/automations/tests/zapier.spec.js

@@ -0,0 +1,27 @@
+const setup = require("./utilities")
+const fetch = require("node-fetch")
+
+jest.mock("node-fetch")
+
+describe("test the outgoing webhook action", () => {
+    let inputs
+    let config = setup.getConfig()
+
+    beforeAll(async () => {
+        await config.init()
+        inputs = {
+            value1: "test",
+            url: "http://www.test.com",
+        }
+    })
+
+    afterAll(setup.afterAll)
+
+    it("should be able to run the action", async () => {
+        const res = await setup.runStep(setup.actions.zapier.stepId, inputs)
+        expect(res.response.url).toEqual("http://www.test.com")
+        expect(res.response.method).toEqual("post")
+        expect(res.success).toEqual(true)
+    })
+
+})

package/src/automations/triggers.ts

@@ -9,7 +9,7 @@ import { checkTestFlag } from "../utilities/redis"
 import * as utils from "./utils"
 import env from "../environment"
 import { context, db as dbCore } from "@budibase/backend-core"
-import { Automation, Row } from "@budibase/types"
+import { Automation, Row, AutomationData, AutomationJob } from "@budibase/types"
 
 export const TRIGGER_DEFINITIONS = definitions
 const JOB_OPTS = {
@@ -109,14 +109,16 @@ export async function externalTrigger(
     }
     params.fields = coercedFields
   }
-  const data: Record<string, any> = { automation, event: params }
+
+  const data: AutomationData = { automation, event: params as any }
   if (getResponses) {
     data.event = {
       ...data.event,
       appId: context.getAppId(),
       automation,
     }
-    return utils.processEvent({ data })
+    const job = { data } as AutomationJob
+    return utils.processEvent(job)
   } else {
     return automationQueue.add(data, JOB_OPTS)
   }

package/src/automations/utils.ts

@@ -8,7 +8,7 @@ import { db as dbCore, context } from "@budibase/backend-core"
 import { getAutomationMetadataParams } from "../db/utils"
 import { cloneDeep } from "lodash/fp"
 import { quotas } from "@budibase/pro"
-import { Automation, WebhookActionType } from "@budibase/types"
+import { Automation, AutomationJob, WebhookActionType } from "@budibase/types"
 import sdk from "../sdk"
 
 const REBOOT_CRON = "@reboot"
@@ -16,27 +16,34 @@ const WH_STEP_ID = definitions.WEBHOOK.stepId
 const CRON_STEP_ID = definitions.CRON.stepId
 const Runner = new Thread(ThreadType.AUTOMATION)
 
-const jobMessage = (job: any, message: string) => {
-  return `app=${job.data.event.appId} automation=${job.data.automation._id} jobId=${job.id} trigger=${job.data.automation.definition.trigger.event} : ${message}`
+function loggingArgs(job: AutomationJob) {
+  return {
+    jobId: job.id,
+    trigger: job.data.automation.definition.trigger.event,
+  }
 }
 
-export async function processEvent(job: any) {
-  try {
-    const automationId = job.data.automation._id
-    console.log(jobMessage(job, "running"))
-    // need to actually await these so that an error can be captured properly
-    return await context.doInContext(job.data.event.appId, async () => {
+export async function processEvent(job: AutomationJob) {
+  const appId = job.data.event.appId!
+  const automationId = job.data.automation._id!
+  const task = async () => {
+    try {
+      // need to actually await these so that an error can be captured properly
+      console.log("automation running", loggingArgs(job))
+
       const runFn = () => Runner.run(job)
-      return quotas.addAutomation(runFn, {
+      const result = await quotas.addAutomation(runFn, {
         automationId,
       })
-    })
-  } catch (err) {
-    const errJson = JSON.stringify(err)
-    console.error(jobMessage(job, `was unable to run - ${errJson}`))
-    console.trace(err)
-    return { err }
+      console.log("automation completed", loggingArgs(job))
+      return result
+    } catch (err) {
+      console.error(`automation was unable to run`, err, loggingArgs(job))
+      return { err }
+    }
   }
+
+  return await context.doInAutomationContext({ appId, automationId, task })
 }
 
 export async function updateTestHistory(

package/src/db/dynamoClient.ts

@@ -140,7 +140,7 @@ export function init(endpoint: string) {
   docClient = new AWS.DynamoDB.DocumentClient(docClientParams)
 }
 
-if (!env.isProd() && !env.isJest()) {
+if (!env.isProd()) {
   env._set("AWS_ACCESS_KEY_ID", "KEY_ID")
   env._set("AWS_SECRET_ACCESS_KEY", "SECRET_KEY")
   init("http://localhost:8333")

package/src/definitions/automations.ts

@@ -1,4 +1,4 @@
-import { AutomationResults, AutomationStep, Document } from "@budibase/types"
+import { AutomationResults, AutomationStep } from "@budibase/types"
 
 export enum LoopStepType {
   ARRAY = "Array",
@@ -27,7 +27,3 @@ export interface AutomationContext extends AutomationResults {
   env?: Record<string, string>
   trigger: any
 }
-
-export interface AutomationMetadata extends Document {
-  errorCount?: number
-}

package/src/integration-test/postgres.spec.ts

@@ -19,6 +19,7 @@ import _ from "lodash"
 import { generator } from "@budibase/backend-core/tests"
 import { utils } from "@budibase/backend-core"
 import { GenericContainer } from "testcontainers"
+import { generateRowIdField } from "../integrations/utils"
 
 const config = setup.getConfig()!
 

package/src/integrations/airtable.ts

@@ -1,13 +1,11 @@
 import {
-  ConnectionInfo,
-  DatasourceFeature,
-  DatasourceFieldType,
   Integration,
-  IntegrationBase,
+  DatasourceFieldType,
   QueryType,
+  IntegrationBase,
 } from "@budibase/types"
 
-import Airtable from "airtable"
+const Airtable = require("airtable")
 
 interface AirtableConfig {
   apiKey: string
@@ -20,7 +18,6 @@ const SCHEMA: Integration = {
     "Airtable is a spreadsheet-database hybrid, with the features of a database but applied to a spreadsheet.",
   friendlyName: "Airtable",
   type: "Spreadsheet",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     apiKey: {
       type: DatasourceFieldType.PASSWORD,
@@ -84,37 +81,13 @@ const SCHEMA: Integration = {
 
 class AirtableIntegration implements IntegrationBase {
   private config: AirtableConfig
-  private client
+  private client: any
 
   constructor(config: AirtableConfig) {
     this.config = config
     this.client = new Airtable(config).base(config.base)
   }
 
-  async testConnection(): Promise<ConnectionInfo> {
-    const mockTable = Date.now().toString()
-    try {
-      await this.client.makeRequest({
-        path: `/${mockTable}`,
-      })
-
-      return { connected: true }
-    } catch (e: any) {
-      if (
-        e.message ===
-        `Could not find table ${mockTable} in application ${this.config.base}`
-      ) {
-        // The request managed to check the application, so the credentials are valid
-        return { connected: true }
-      }
-
-      return {
-        connected: false,
-        error: e.message as string,
-      }
-    }
-  }
-
   async create(query: { table: any; json: any }) {
     const { table, json } = query
 

package/src/integrations/arangodb.ts

@@ -3,11 +3,9 @@ import {
   DatasourceFieldType,
   QueryType,
   IntegrationBase,
-  DatasourceFeature,
-  ConnectionInfo,
 } from "@budibase/types"
 
-import { Database, aql } from "arangojs"
+const { Database, aql } = require("arangojs")
 
 interface ArangodbConfig {
   url: string
@@ -23,7 +21,6 @@ const SCHEMA: Integration = {
   type: "Non-relational",
   description:
     "ArangoDB is a scalable open-source multi-model database natively supporting graph, document and search. All supported data models & access patterns can be combined in queries allowing for maximal flexibility. ",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     url: {
       type: DatasourceFieldType.STRING,
@@ -61,7 +58,7 @@ const SCHEMA: Integration = {
 
 class ArangoDBIntegration implements IntegrationBase {
   private config: ArangodbConfig
-  private client
+  private client: any
 
   constructor(config: ArangodbConfig) {
     const newConfig = {
@@ -77,19 +74,6 @@ class ArangoDBIntegration implements IntegrationBase {
     this.client = new Database(newConfig)
   }
 
-  async testConnection() {
-    const response: ConnectionInfo = {
-      connected: false,
-    }
-    try {
-      await this.client.get()
-      response.connected = true
-    } catch (e: any) {
-      response.error = e.message as string
-    }
-    return response
-  }
-
   async read(query: { sql: any }) {
     try {
       const result = await this.client.query(query.sql)

package/src/integrations/couchdb.ts

@@ -1,6 +1,4 @@
 import {
-  ConnectionInfo,
-  DatasourceFeature,
   DatasourceFieldType,
   Document,
   Integration,
@@ -20,7 +18,6 @@ const SCHEMA: Integration = {
   type: "Non-relational",
   description:
     "Apache CouchDB is an open-source document-oriented NoSQL database, implemented in Erlang.",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     url: {
       type: DatasourceFieldType.STRING,
@@ -64,32 +61,21 @@ const SCHEMA: Integration = {
 }
 
 class CouchDBIntegration implements IntegrationBase {
-  private readonly client: dbCore.DatabaseImpl
+  private config: CouchDBConfig
+  private readonly client: any
 
   constructor(config: CouchDBConfig) {
+    this.config = config
     this.client = dbCore.DatabaseWithConnection(config.database, config.url)
   }
 
-  async testConnection() {
-    const response: ConnectionInfo = {
-      connected: false,
-    }
-    try {
-      const result = await this.query("exists", "validation error", {})
-      response.connected = result === true
-    } catch (e: any) {
-      response.error = e.message as string
-    }
-    return response
-  }
-
   async query(
     command: string,
     errorMsg: string,
     query: { json?: object; id?: string }
   ) {
     try {
-      return await (this.client as any)[command](query.id || query.json)
+      return await this.client[command](query.id || query.json)
     } catch (err) {
       console.error(errorMsg, err)
       throw err