@budibase/server 2.6.19-alpha.3 → 2.6.19-alpha.30

package/src/integrations/dynamodb.ts

@@ -25,7 +25,9 @@ const SCHEMA: Integration = {
     "Amazon DynamoDB is a key-value and document database that delivers single-digit millisecond performance at any scale.",
   friendlyName: "DynamoDB",
   type: "Non-relational",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     region: {
       type: DatasourceFieldType.STRING,

package/src/integrations/elasticsearch.ts

@@ -22,7 +22,9 @@ const SCHEMA: Integration = {
     "Elasticsearch is a search engine based on the Lucene library. It provides a distributed, multitenant-capable full-text search engine with an HTTP web interface and schema-free JSON documents.",
   friendlyName: "ElasticSearch",
   type: "Non-relational",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     url: {
       type: DatasourceFieldType.STRING,

package/src/integrations/firebase.ts

@@ -20,7 +20,9 @@ const SCHEMA: Integration = {
   type: "Non-relational",
   description:
     "Cloud Firestore is a flexible, scalable database for mobile, web, and server development from Firebase and Google Cloud.",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     email: {
       type: DatasourceFieldType.STRING,

package/src/integrations/googlesheets.ts

@@ -63,10 +63,13 @@ const SCHEMA: Integration = {
   relationships: false,
   docs: "https://developers.google.com/sheets/api/quickstart/nodejs",
   description:
-    "Create and collaborate on online spreadsheets in real-time and from any device. ",
+    "Create and collaborate on online spreadsheets in real-time and from any device.",
   friendlyName: "Google Sheets",
   type: "Spreadsheet",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+    [DatasourceFeature.FETCH_TABLE_NAMES]: true,
+  },
   datasource: {
     spreadsheetId: {
       display: "Google Sheet URL",
@@ -145,7 +148,6 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   async testConnection(): Promise<ConnectionInfo> {
     try {
       await this.connect()
-      await this.client.loadInfo()
       return { connected: true }
     } catch (e: any) {
       return {
@@ -240,6 +242,12 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     }
   }
 
+  async getTableNames(): Promise<string[]> {
+    await this.connect()
+    const sheets = this.client.sheetsByIndex
+    return sheets.map(s => s.title)
+  }
+
   getTableSchema(title: string, headerValues: string[], id?: string) {
     // base table
     const table: Table = {

package/src/integrations/microsoftSqlServer.ts

@@ -20,7 +20,6 @@ import {
 } from "./utils"
 import Sql from "./base/sql"
 import { MSSQLTablesResponse, MSSQLColumn } from "./base/types"
-
 const sqlServer = require("mssql")
 const DEFAULT_SCHEMA = "dbo"
 
@@ -41,7 +40,10 @@ const SCHEMA: Integration = {
     "Microsoft SQL Server is a relational database management system developed by Microsoft. ",
   friendlyName: "MS SQL Server",
   type: "Relational",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+    [DatasourceFeature.FETCH_TABLE_NAMES]: true,
+  },
   datasource: {
     user: {
       type: DatasourceFieldType.STRING,
@@ -284,6 +286,20 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
     this.schemaErrors = final.errors
   }
 
+  async queryTableNames() {
+    let tableInfo: MSSQLTablesResponse[] = await this.runSQL(this.TABLES_SQL)
+    const schema = this.config.schema || DEFAULT_SCHEMA
+    return tableInfo
+      .filter((record: any) => record.TABLE_SCHEMA === schema)
+      .map((record: any) => record.TABLE_NAME)
+      .filter((name: string) => this.MASTER_TABLES.indexOf(name) === -1)
+  }
+
+  async getTableNames() {
+    await this.connect()
+    return this.queryTableNames()
+  }
+
   async read(query: SqlQuery | string) {
     await this.connect()
     const response = await this.internalQuery(getSqlQuery(query))

package/src/integrations/mongodb.ts

@@ -40,7 +40,9 @@ const getSchema = () => {
     type: "Non-relational",
     description:
       "MongoDB is a general purpose, document-based, distributed database built for modern application developers and for the cloud era.",
-    features: [DatasourceFeature.CONNECTION_CHECKING],
+    features: {
+      [DatasourceFeature.CONNECTION_CHECKING]: true,
+    },
     datasource: {
       connectionString: {
         type: DatasourceFieldType.STRING,

package/src/integrations/mysql.ts

@@ -36,7 +36,10 @@ const SCHEMA: Integration = {
   type: "Relational",
   description:
     "MySQL Database Service is a fully managed database service to deploy cloud-native applications. ",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+    [DatasourceFeature.FETCH_TABLE_NAMES]: true,
+  },
   datasource: {
     host: {
       type: DatasourceFieldType.STRING,
@@ -214,20 +217,11 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
 
   async buildSchema(datasourceId: string, entities: Record<string, Table>) {
     const tables: { [key: string]: Table } = {}
-    const database = this.config.database
     await this.connect()
 
     try {
       // get the tables first
-      const tablesResp: Record<string, string>[] = await this.internalQuery(
-        { sql: "SHOW TABLES;" },
-        { connect: false }
-      )
-      const tableNames: string[] = tablesResp.map(
-        (obj: any) =>
-          obj[`Tables_in_${database}`] ||
-          obj[`Tables_in_${database.toLowerCase()}`]
-      )
+      const tableNames = await this.queryTableNames()
       for (let tableName of tableNames) {
         const primaryKeys = []
         const schema: TableSchema = {}
@@ -274,6 +268,28 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
     this.schemaErrors = final.errors
   }
 
+  async queryTableNames() {
+    const database = this.config.database
+    const tablesResp: Record<string, string>[] = await this.internalQuery(
+      { sql: "SHOW TABLES;" },
+      { connect: false }
+    )
+    return tablesResp.map(
+      (obj: any) =>
+        obj[`Tables_in_${database}`] ||
+        obj[`Tables_in_${database.toLowerCase()}`]
+    )
+  }
+
+  async getTableNames() {
+    await this.connect()
+    try {
+      return this.queryTableNames()
+    } finally {
+      await this.disconnect()
+    }
+  }
+
   async create(query: SqlQuery | string) {
     const results = await this.internalQuery(getSqlQuery(query))
     return results.length ? results : [{ created: true }]

package/src/integrations/oracle.ts

@@ -50,7 +50,10 @@ const SCHEMA: Integration = {
   type: "Relational",
   description:
     "Oracle Database is an object-relational database management system developed by Oracle Corporation",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+    [DatasourceFeature.FETCH_TABLE_NAMES]: true,
+  },
   datasource: {
     host: {
       type: DatasourceFieldType.STRING,
@@ -323,6 +326,13 @@ class OracleIntegration extends Sql implements DatasourcePlus {
     this.schemaErrors = final.errors
   }
 
+  async getTableNames() {
+    const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
+      sql: this.COLUMNS_SQL,
+    })
+    return (columnsResponse.rows || []).map(row => row.TABLE_NAME)
+  }
+
   async testConnection() {
     const response: ConnectionInfo = {
       connected: false,

package/src/integrations/postgres.ts

@@ -52,7 +52,10 @@ const SCHEMA: Integration = {
   type: "Relational",
   description:
     "PostgreSQL, also known as Postgres, is a free and open-source relational database management system emphasizing extensibility and SQL compliance.",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+    [DatasourceFeature.FETCH_TABLE_NAMES]: true,
+  },
   datasource: {
     host: {
       type: DatasourceFieldType.STRING,
@@ -314,6 +317,17 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
     }
   }
 
+  async getTableNames() {
+    try {
+      await this.openConnection()
+      const columnsResponse: { rows: PostgresColumn[] } =
+        await this.client.query(this.COLUMNS_SQL)
+      return columnsResponse.rows.map(row => row.table_name)
+    } finally {
+      await this.closeConnection()
+    }
+  }
+
   async create(query: SqlQuery | string) {
     const response = await this.internalQuery(getSqlQuery(query))
     return response.rows.length ? response.rows : [{ created: true }]

package/src/integrations/redis.ts

@@ -21,7 +21,9 @@ const SCHEMA: Integration = {
     "Redis is a caching tool, providing powerful key-value store capabilities.",
   friendlyName: "Redis",
   type: "Non-relational",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     host: {
       type: "string",

package/src/integrations/s3.ts

@@ -24,7 +24,9 @@ const SCHEMA: Integration = {
     "Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.",
   friendlyName: "Amazon S3",
   type: "Object store",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     region: {
       type: "string",

package/src/integrations/snowflake.ts

@@ -22,7 +22,9 @@ const SCHEMA: Integration = {
     "Snowflake is a solution for data warehousing, data lakes, data engineering, data science, data application development, and securely sharing and consuming shared data.",
   friendlyName: "Snowflake",
   type: "Relational",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     account: {
       type: "string",

package/src/integrations/tests/googlesheets.spec.ts

@@ -17,14 +17,15 @@ jest.mock("google-spreadsheet")
 const { GoogleSpreadsheet } = require("google-spreadsheet")
 
 const sheetsByTitle: { [title: string]: GoogleSpreadsheetWorksheet } = {}
+const sheetsByIndex: GoogleSpreadsheetWorksheet[] = []
+const mockGoogleIntegration = {
+  useOAuth2Client: jest.fn(),
+  loadInfo: jest.fn(),
+  sheetsByTitle,
+  sheetsByIndex,
+}
 
-GoogleSpreadsheet.mockImplementation(() => {
-  return {
-    useOAuth2Client: jest.fn(),
-    loadInfo: jest.fn(),
-    sheetsByTitle,
-  }
-})
+GoogleSpreadsheet.mockImplementation(() => mockGoogleIntegration)
 
 import { structures } from "@budibase/backend-core/tests"
 import TestConfiguration from "../../tests/utilities/TestConfiguration"
@@ -53,6 +54,8 @@ describe("Google Sheets Integration", () => {
       },
     })
     await config.init()
+
+    jest.clearAllMocks()
   })
 
   function createBasicTable(name: string, columns: string[]): Table {
@@ -88,7 +91,7 @@ describe("Google Sheets Integration", () => {
   }
 
   describe("update table", () => {
-    test("adding a new field will be adding a new header row", async () => {
+    it("adding a new field will be adding a new header row", async () => {
       await config.doInContext(structures.uuid(), async () => {
         const tableColumns = ["name", "description", "new field"]
         const table = createBasicTable(structures.uuid(), tableColumns)
@@ -103,7 +106,7 @@ describe("Google Sheets Integration", () => {
       })
     })
 
-    test("removing an existing field will remove the header from the google sheet", async () => {
+    it("removing an existing field will remove the header from the google sheet", async () => {
       const sheet = await config.doInContext(structures.uuid(), async () => {
         const tableColumns = ["name"]
         const table = createBasicTable(structures.uuid(), tableColumns)
@@ -123,4 +126,33 @@ describe("Google Sheets Integration", () => {
       expect((sheet.setHeaderRow as any).mock.calls[0][0]).toHaveLength(1)
     })
   })
+
+  describe("getTableNames", () => {
+    it("can fetch table names", async () => {
+      await config.doInContext(structures.uuid(), async () => {
+        const sheetNames: string[] = []
+        for (let i = 0; i < 5; i++) {
+          const sheet = createSheet({ headerValues: [] })
+          sheetsByIndex.push(sheet)
+          sheetNames.push(sheet.title)
+        }
+
+        const res = await integration.getTableNames()
+
+        expect(mockGoogleIntegration.loadInfo).toBeCalledTimes(1)
+        expect(res).toEqual(sheetNames)
+      })
+    })
+  })
+
+  describe("testConnection", () => {
+    it("can test successful connections", async () => {
+      await config.doInContext(structures.uuid(), async () => {
+        const res = await integration.testConnection()
+
+        expect(mockGoogleIntegration.loadInfo).toBeCalledTimes(1)
+        expect(res).toEqual({ connected: true })
+      })
+    })
+  })
 })

package/src/middleware/builder.ts

@@ -35,12 +35,11 @@ async function checkDevAppLocks(ctx: BBContext) {
   if (!appId || !appId.startsWith(APP_DEV_PREFIX)) {
     return
   }
-  if (!(await doesUserHaveLock(appId, ctx.user))) {
-    ctx.throw(400, "User does not hold app lock.")
-  }
 
-  // they do have lock, update it
-  await updateLock(appId, ctx.user)
+  // If this user already owns the lock, then update it
+  if (await doesUserHaveLock(appId, ctx.user)) {
+    await updateLock(appId, ctx.user)
+  }
 }
 
 async function updateAppUpdatedAt(ctx: BBContext) {

package/src/sdk/app/automations/index.ts

@@ -1,5 +1,7 @@
 import * as webhook from "./webhook"
+import * as utils from "./utils"
 
 export default {
   webhook,
+  utils,
 }

package/src/sdk/app/automations/utils.ts

@@ -0,0 +1,7 @@
+import { Automation, AutomationActionStepId } from "@budibase/types"
+
+export function checkForCollectStep(automation: Automation) {
+  return automation.definition.steps.some(
+    (step: any) => step.stepId === AutomationActionStepId.COLLECT
+  )
+}

package/src/startup.ts

@@ -16,6 +16,7 @@ import * as bullboard from "./automations/bullboard"
 import * as pro from "@budibase/pro"
 import * as api from "./api"
 import sdk from "./sdk"
+import { initialise as initialiseWebsockets } from "./websockets"
 
 let STARTUP_RAN = false
 
@@ -64,6 +65,7 @@ export async function startup(app?: any, server?: any) {
   fileSystem.init()
   await redis.init()
   eventInit()
+  initialiseWebsockets(app, server)
 
   // run migrations on startup if not done via http
   // not recommended in a clustered environment

package/src/tests/utilities/TestConfiguration.ts

@@ -373,7 +373,7 @@ class TestConfiguration {
 
   // HEADERS
 
-  defaultHeaders(extras = {}) {
+  defaultHeaders(extras = {}, prodApp = false) {
     const tenantId = this.getTenantId()
     const authObj: AuthToken = {
       userId: this.defaultUserValues.globalUserId,
@@ -390,7 +390,9 @@ class TestConfiguration {
       ...extras,
     }
 
-    if (this.appId) {
+    if (prodApp) {
+      headers[constants.Header.APP_ID] = this.prodAppId
+    } else if (this.appId) {
       headers[constants.Header.APP_ID] = this.appId
     }
     return headers

package/src/tests/utilities/structures.ts

@@ -199,6 +199,48 @@ export function loopAutomation(tableId: string, loopOpts?: any): Automation {
   return automation as Automation
 }
 
+export function collectAutomation(tableId?: string): Automation {
+  const automation: any = {
+    name: "looping",
+    type: "automation",
+    definition: {
+      steps: [
+        {
+          id: "b",
+          type: "ACTION",
+          internal: true,
+          stepId: AutomationActionStepId.EXECUTE_SCRIPT,
+          inputs: {
+            code: "return [1,2,3]",
+          },
+          schema: BUILTIN_ACTION_DEFINITIONS.EXECUTE_SCRIPT.schema,
+        },
+        {
+          id: "c",
+          type: "ACTION",
+          internal: true,
+          stepId: AutomationActionStepId.COLLECT,
+          inputs: {
+            collection: "{{ literal steps.1.value }}",
+          },
+          schema: BUILTIN_ACTION_DEFINITIONS.SERVER_LOG.schema,
+        },
+      ],
+      trigger: {
+        id: "a",
+        type: "TRIGGER",
+        event: "row:save",
+        stepId: AutomationTriggerStepId.ROW_SAVED,
+        inputs: {
+          tableId,
+        },
+        schema: TRIGGER_DEFINITIONS.ROW_SAVED.schema,
+      },
+    },
+  }
+  return automation as Automation
+}
+
 export function basicRow(tableId: string) {
   return {
     name: "Test Contact",

package/src/threads/automation.ts

@@ -68,6 +68,7 @@ class Orchestrator {
   constructor(job: AutomationJob) {
     let automation = job.data.automation
     let triggerOutput = job.data.event
+    let timeout = job.data.event.timeout
     const metadata = triggerOutput.metadata
     this._chainCount = metadata ? metadata.automationChainCount! : 0
     this._appId = triggerOutput.appId as string
@@ -240,7 +241,9 @@ class Orchestrator {
     let loopStepNumber: any = undefined
     let loopSteps: LoopStep[] | undefined = []
     let metadata
+    let timeoutFlag = false
     let wasLoopStep = false
+    let timeout = this._job.data.event.timeout
     // check if this is a recurring automation,
     if (isProdAppID(this._appId) && isRecurring(automation)) {
       metadata = await this.getMetadata()
@@ -251,6 +254,16 @@ class Orchestrator {
     }
 
     for (let step of automation.definition.steps) {
+      if (timeoutFlag) {
+        break
+      }
+
+      if (timeout) {
+        setTimeout(() => {
+          timeoutFlag = true
+        }, timeout || 12000)
+      }
+
       stepCount++
       let input: any,
         iterations = 1,
@@ -495,6 +508,32 @@ export function execute(job: Job, callback: WorkerCallback) {
   })
 }
 
+export function executeSynchronously(job: Job) {
+  const appId = job.data.event.appId
+  if (!appId) {
+    throw new Error("Unable to execute, event doesn't contain app ID.")
+  }
+
+  const timeoutPromise = new Promise((resolve, reject) => {
+    setTimeout(() => {
+      reject(new Error("Timeout exceeded"))
+    }, job.data.event.timeout || 12000)
+  })
+
+  return context.doInAppContext(appId, async () => {
+    const envVars = await sdkUtils.getEnvironmentVariables()
+    // put into automation thread for whole context
+    return context.doInEnvironmentContext(envVars, async () => {
+      const automationOrchestrator = new Orchestrator(job)
+      const response = await Promise.race([
+        automationOrchestrator.execute(),
+        timeoutPromise,
+      ])
+      return response
+    })
+  })
+}
+
 export const removeStalled = async (job: Job) => {
   const appId = job.data.event.appId
   if (!appId) {

package/src/utilities/fileSystem/app.ts

@@ -35,7 +35,7 @@ export const getComponentLibraryManifest = async (library: string) => {
   const filename = "manifest.json"
 
   if (env.isDev() || env.isTest()) {
-    const path = join(TOP_LEVEL_PATH, "../client", filename)
+    const path = join(TOP_LEVEL_PATH, "packages/client", filename)
     // always load from new so that updates are refreshed
     delete require.cache[require.resolve(path)]
     return require(path)

package/src/utilities/fileSystem/clientLibrary.ts

@@ -1,4 +1,4 @@
-import { join } from "path"
+import path, { join } from "path"
 import { ObjectStoreBuckets } from "../../constants"
 import fs from "fs"
 import { objectStore } from "@budibase/backend-core"
@@ -6,6 +6,10 @@ import { resolve } from "../centralPath"
 import env from "../../environment"
 import { TOP_LEVEL_PATH } from "./filesystem"
 
+export function devClientLibPath() {
+  return require.resolve("@budibase/client")
+}
+
 /**
  * Client library paths in the object store:
  * Previously, the entire client library package was downloaded from NPM
@@ -89,9 +93,10 @@ export async function updateClientLibrary(appId: string) {
   let manifest, client
 
   if (env.isDev()) {
+    const clientPath = devClientLibPath()
     // Load the symlinked version in dev which is always the newest
-    manifest = require.resolve("@budibase/client/manifest.json")
-    client = require.resolve("@budibase/client")
+    manifest = join(path.dirname(path.dirname(clientPath)), "manifest.json")
+    client = clientPath
   } else {
     // Load the bundled version in prod
     manifest = resolve(TOP_LEVEL_PATH, "client", "manifest.json")

package/src/utilities/fileSystem/filesystem.ts

@@ -4,9 +4,11 @@ import { budibaseTempDir } from "../budibaseDir"
 import { join } from "path"
 import env from "../../environment"
 import tar from "tar"
+import environment from "../../environment"
 const uuid = require("uuid/v4")
 
-export const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
+export const TOP_LEVEL_PATH =
+  environment.TOP_LEVEL_PATH || join(__dirname, "..", "..", "..")
 
 /**
  * Upon first startup of instance there may not be everything we need in tmp directory, set it up.

package/src/utilities/redis.ts

@@ -4,23 +4,33 @@ import { ContextUser } from "@budibase/types"
 
 const APP_DEV_LOCK_SECONDS = 600
 const AUTOMATION_TEST_FLAG_SECONDS = 60
-let devAppClient: any, debounceClient: any, flagClient: any
+let devAppClient: any, debounceClient: any, flagClient: any, socketClient: any
 
-// we init this as we want to keep the connection open all the time
+// We need to maintain a duplicate client for socket.io pub/sub
+let socketSubClient: any
+
+// We init this as we want to keep the connection open all the time
 // reduces the performance hit
 export async function init() {
   devAppClient = new redis.Client(redis.utils.Databases.DEV_LOCKS)
   debounceClient = new redis.Client(redis.utils.Databases.DEBOUNCE)
   flagClient = new redis.Client(redis.utils.Databases.FLAGS)
+  socketClient = new redis.Client(redis.utils.Databases.SOCKET_IO)
   await devAppClient.init()
   await debounceClient.init()
   await flagClient.init()
+  await socketClient.init()
+
+  // Duplicate the socket client for pub/sub
+  socketSubClient = socketClient.getClient().duplicate()
 }
 
 export async function shutdown() {
   if (devAppClient) await devAppClient.finish()
   if (debounceClient) await debounceClient.finish()
   if (flagClient) await flagClient.finish()
+  if (socketClient) await socketClient.finish()
+  if (socketSubClient) socketSubClient.disconnect()
   // shutdown core clients
   await redis.clients.shutdown()
   console.log("Redis shutdown")
@@ -86,3 +96,10 @@ export async function checkTestFlag(id: string) {
 export async function clearTestFlag(id: string) {
   await devAppClient.delete(id)
 }
+
+export function getSocketPubSubClients() {
+  return {
+    pub: socketClient.getClient(),
+    sub: socketSubClient,
+  }
+}