@budibase/server 2.6.22 → 2.6.24-alpha.0

package/src/environment.ts

@@ -71,6 +71,7 @@ const environment = {
   BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL,
   BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
   PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins",
+  OPENAI_API_KEY: process.env.OPENAI_API_KEY,
   // flags
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,
@@ -79,6 +80,7 @@ const environment = {
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   SELF_HOSTED: process.env.SELF_HOSTED,
   HTTP_MB_LIMIT: process.env.HTTP_MB_LIMIT,
+  FORKED_PROCESS_NAME: process.env.FORKED_PROCESS_NAME || "main",
   // old
   CLIENT_ID: process.env.CLIENT_ID,
   _set(key: string, value: any) {
@@ -96,6 +98,7 @@ const environment = {
   isInThread: () => {
     return process.env.FORKED_PROCESS
   },
+  TOP_LEVEL_PATH: process.env.TOP_LEVEL_PATH,
 }
 
 // threading can cause memory issues with node-ts in development

package/src/integration-test/postgres.spec.ts

@@ -19,15 +19,15 @@ import _ from "lodash"
 import { generator } from "@budibase/backend-core/tests"
 import { utils } from "@budibase/backend-core"
 import { GenericContainer } from "testcontainers"
-import { generateRowIdField } from "../integrations/utils"
 
 const config = setup.getConfig()!
 
 jest.setTimeout(30000)
 
 jest.unmock("pg")
+jest.mock("../websockets")
 
-describe("row api - postgres", () => {
+describe("postgres integrations", () => {
   let makeRequest: MakeRequestResponse,
     postgresDatasource: Datasource,
     primaryPostgresTable: Table,
@@ -53,8 +53,8 @@ describe("row api - postgres", () => {
     makeRequest = generateMakeRequest(apiKey, true)
   })
 
-  beforeEach(async () => {
-    postgresDatasource = await config.createDatasource({
+  function pgDatasourceConfig() {
+    return {
       datasource: {
         type: "datasource",
         source: SourceName.POSTGRES,
@@ -71,7 +71,11 @@ describe("row api - postgres", () => {
           ca: false,
         },
       },
-    })
+    }
+  }
+
+  beforeEach(async () => {
+    postgresDatasource = await config.createDatasource(pgDatasourceConfig())
 
     async function createAuxTable(prefix: string) {
       return await config.createTable({
@@ -1025,4 +1029,42 @@ describe("row api - postgres", () => {
       })
     })
   })
+
+  describe("POST /api/datasources/verify", () => {
+    it("should be able to verify the connection", async () => {
+      const config = pgDatasourceConfig()
+      const response = await makeRequest(
+        "post",
+        "/api/datasources/verify",
+        config
+      )
+      expect(response.status).toBe(200)
+      expect(response.body.connected).toBe(true)
+    })
+
+    it("should state an invalid datasource cannot connect", async () => {
+      const config = pgDatasourceConfig()
+      config.datasource.config.password = "wrongpassword"
+      const response = await makeRequest(
+        "post",
+        "/api/datasources/verify",
+        config
+      )
+      expect(response.status).toBe(200)
+      expect(response.body.connected).toBe(false)
+      expect(response.body.error).toBeDefined()
+    })
+  })
+
+  describe("POST /api/datasources/info", () => {
+    it("should fetch information about postgres datasource", async () => {
+      const primaryName = primaryPostgresTable.name
+      const response = await makeRequest("post", "/api/datasources/info", {
+        datasource: postgresDatasource,
+      })
+      expect(response.status).toBe(200)
+      expect(response.body.tableNames).toBeDefined()
+      expect(response.body.tableNames.indexOf(primaryName)).not.toBe(-1)
+    })
+  })
 })

package/src/integrations/airtable.ts

@@ -1,11 +1,13 @@
 import {
-  Integration,
+  ConnectionInfo,
+  DatasourceFeature,
   DatasourceFieldType,
-  QueryType,
+  Integration,
   IntegrationBase,
+  QueryType,
 } from "@budibase/types"
 
-const Airtable = require("airtable")
+import Airtable from "airtable"
 
 interface AirtableConfig {
   apiKey: string
@@ -18,6 +20,9 @@ const SCHEMA: Integration = {
     "Airtable is a spreadsheet-database hybrid, with the features of a database but applied to a spreadsheet.",
   friendlyName: "Airtable",
   type: "Spreadsheet",
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     apiKey: {
       type: DatasourceFieldType.PASSWORD,
@@ -81,13 +86,37 @@ const SCHEMA: Integration = {
 
 class AirtableIntegration implements IntegrationBase {
   private config: AirtableConfig
-  private client: any
+  private client
 
   constructor(config: AirtableConfig) {
     this.config = config
     this.client = new Airtable(config).base(config.base)
   }
 
+  async testConnection(): Promise<ConnectionInfo> {
+    const mockTable = Date.now().toString()
+    try {
+      await this.client.makeRequest({
+        path: `/${mockTable}`,
+      })
+
+      return { connected: true }
+    } catch (e: any) {
+      if (
+        e.message ===
+        `Could not find table ${mockTable} in application ${this.config.base}`
+      ) {
+        // The request managed to check the application, so the credentials are valid
+        return { connected: true }
+      }
+
+      return {
+        connected: false,
+        error: e.message as string,
+      }
+    }
+  }
+
   async create(query: { table: any; json: any }) {
     const { table, json } = query
 

package/src/integrations/arangodb.ts

@@ -3,9 +3,11 @@ import {
   DatasourceFieldType,
   QueryType,
   IntegrationBase,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"
 
-const { Database, aql } = require("arangojs")
+import { Database, aql } from "arangojs"
 
 interface ArangodbConfig {
   url: string
@@ -21,6 +23,9 @@ const SCHEMA: Integration = {
   type: "Non-relational",
   description:
     "ArangoDB is a scalable open-source multi-model database natively supporting graph, document and search. All supported data models & access patterns can be combined in queries allowing for maximal flexibility. ",
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     url: {
       type: DatasourceFieldType.STRING,
@@ -58,7 +63,7 @@ const SCHEMA: Integration = {
 
 class ArangoDBIntegration implements IntegrationBase {
   private config: ArangodbConfig
-  private client: any
+  private client
 
   constructor(config: ArangodbConfig) {
     const newConfig = {
@@ -74,6 +79,19 @@ class ArangoDBIntegration implements IntegrationBase {
     this.client = new Database(newConfig)
   }
 
+  async testConnection() {
+    const response: ConnectionInfo = {
+      connected: false,
+    }
+    try {
+      await this.client.get()
+      response.connected = true
+    } catch (e: any) {
+      response.error = e.message as string
+    }
+    return response
+  }
+
   async read(query: { sql: any }) {
     try {
       const result = await this.client.query(query.sql)

package/src/integrations/base/sqlTable.ts

@@ -206,4 +206,3 @@ class SqlTableQueryBuilder {
 }
 
 export default SqlTableQueryBuilder
-module.exports = SqlTableQueryBuilder

package/src/integrations/couchdb.ts

@@ -1,4 +1,6 @@
 import {
+  ConnectionInfo,
+  DatasourceFeature,
   DatasourceFieldType,
   Document,
   Integration,
@@ -18,6 +20,9 @@ const SCHEMA: Integration = {
   type: "Non-relational",
   description:
     "Apache CouchDB is an open-source document-oriented NoSQL database, implemented in Erlang.",
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     url: {
       type: DatasourceFieldType.STRING,
@@ -61,21 +66,32 @@ const SCHEMA: Integration = {
 }
 
 class CouchDBIntegration implements IntegrationBase {
-  private config: CouchDBConfig
-  private readonly client: any
+  private readonly client: dbCore.DatabaseImpl
 
   constructor(config: CouchDBConfig) {
-    this.config = config
     this.client = dbCore.DatabaseWithConnection(config.database, config.url)
   }
 
+  async testConnection() {
+    const response: ConnectionInfo = {
+      connected: false,
+    }
+    try {
+      const result = await this.query("exists", "validation error", {})
+      response.connected = result === true
+    } catch (e: any) {
+      response.error = e.message as string
+    }
+    return response
+  }
+
   async query(
     command: string,
     errorMsg: string,
     query: { json?: object; id?: string }
   ) {
     try {
-      return await this.client[command](query.id || query.json)
+      return await (this.client as any)[command](query.id || query.json)
     } catch (err) {
       console.error(errorMsg, err)
       throw err

package/src/integrations/dynamodb.ts

@@ -3,10 +3,13 @@ import {
   DatasourceFieldType,
   QueryType,
   IntegrationBase,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"
 
 import AWS from "aws-sdk"
 import { AWS_REGION } from "../db/dynamoClient"
+import { DocumentClient } from "aws-sdk/clients/dynamodb"
 
 interface DynamoDBConfig {
   region: string
@@ -22,6 +25,9 @@ const SCHEMA: Integration = {
     "Amazon DynamoDB is a key-value and document database that delivers single-digit millisecond performance at any scale.",
   friendlyName: "DynamoDB",
   type: "Non-relational",
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     region: {
       type: DatasourceFieldType.STRING,
@@ -128,7 +134,7 @@ const SCHEMA: Integration = {
 
 class DynamoDBIntegration implements IntegrationBase {
   private config: DynamoDBConfig
-  private client: any
+  private client
 
   constructor(config: DynamoDBConfig) {
     this.config = config
@@ -148,7 +154,23 @@ class DynamoDBIntegration implements IntegrationBase {
     this.client = new AWS.DynamoDB.DocumentClient(this.config)
   }
 
-  async create(query: { table: string; json: object }) {
+  async testConnection() {
+    const response: ConnectionInfo = {
+      connected: false,
+    }
+    try {
+      const scanRes = await new AWS.DynamoDB(this.config).listTables().promise()
+      response.connected = !!scanRes.$response
+    } catch (e: any) {
+      response.error = e.message as string
+    }
+    return response
+  }
+
+  async create(query: {
+    table: string
+    json: Omit<DocumentClient.PutItemInput, "TableName">
+  }) {
     const params = {
       TableName: query.table,
       ...query.json,
@@ -189,7 +211,10 @@ class DynamoDBIntegration implements IntegrationBase {
     return new AWS.DynamoDB(this.config).describeTable(params).promise()
   }
 
-  async get(query: { table: string; json: object }) {
+  async get(query: {
+    table: string
+    json: Omit<DocumentClient.GetItemInput, "TableName">
+  }) {
     const params = {
       TableName: query.table,
       ...query.json,
@@ -197,7 +222,10 @@ class DynamoDBIntegration implements IntegrationBase {
     return this.client.get(params).promise()
   }
 
-  async update(query: { table: string; json: object }) {
+  async update(query: {
+    table: string
+    json: Omit<DocumentClient.UpdateItemInput, "TableName">
+  }) {
     const params = {
       TableName: query.table,
       ...query.json,
@@ -205,7 +233,10 @@ class DynamoDBIntegration implements IntegrationBase {
     return this.client.update(params).promise()
   }
 
-  async delete(query: { table: string; json: object }) {
+  async delete(query: {
+    table: string
+    json: Omit<DocumentClient.DeleteItemInput, "TableName">
+  }) {
     const params = {
       TableName: query.table,
       ...query.json,

package/src/integrations/elasticsearch.ts

@@ -3,6 +3,8 @@ import {
   DatasourceFieldType,
   QueryType,
   IntegrationBase,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"
 
 import { Client, ClientOptions } from "@elastic/elasticsearch"
@@ -20,6 +22,9 @@ const SCHEMA: Integration = {
     "Elasticsearch is a search engine based on the Lucene library. It provides a distributed, multitenant-capable full-text search engine with an HTTP web interface and schema-free JSON documents.",
   friendlyName: "ElasticSearch",
   type: "Non-relational",
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     url: {
       type: DatasourceFieldType.STRING,
@@ -95,7 +100,7 @@ const SCHEMA: Integration = {
 
 class ElasticSearchIntegration implements IntegrationBase {
   private config: ElasticsearchConfig
-  private client: any
+  private client
 
   constructor(config: ElasticsearchConfig) {
     this.config = config
@@ -114,6 +119,18 @@ class ElasticSearchIntegration implements IntegrationBase {
     this.client = new Client(clientConfig)
   }
 
+  async testConnection(): Promise<ConnectionInfo> {
+    try {
+      await this.client.info()
+      return { connected: true }
+    } catch (e: any) {
+      return {
+        connected: false,
+        error: e.message as string,
+      }
+    }
+  }
+
   async create(query: { index: string; json: object }) {
     const { index, json } = query
 

package/src/integrations/firebase.ts

@@ -3,6 +3,8 @@ import {
   Integration,
   QueryType,
   IntegrationBase,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"
 import { Firestore, WhereFilterOp } from "@google-cloud/firestore"
 
@@ -18,6 +20,9 @@ const SCHEMA: Integration = {
   type: "Non-relational",
   description:
     "Cloud Firestore is a flexible, scalable database for mobile, web, and server development from Firebase and Google Cloud.",
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+  },
   datasource: {
     email: {
       type: DatasourceFieldType.STRING,
@@ -99,6 +104,18 @@ class FirebaseIntegration implements IntegrationBase {
     })
   }
 
+  async testConnection(): Promise<ConnectionInfo> {
+    try {
+      await this.client.listCollections()
+      return { connected: true }
+    } catch (e: any) {
+      return {
+        connected: false,
+        error: e.message as string,
+      }
+    }
+  }
+
   async create(query: { json: object; extra: { [key: string]: string } }) {
     try {
       const documentReference = this.client

package/src/integrations/googlesheets.ts

@@ -1,4 +1,6 @@
 import {
+  ConnectionInfo,
+  DatasourceFeature,
   DatasourceFieldType,
   DatasourcePlus,
   FieldType,
@@ -15,7 +17,7 @@ import {
 } from "@budibase/types"
 import { OAuth2Client } from "google-auth-library"
 import { buildExternalTableId, finaliseExternalTables } from "./utils"
-import { GoogleSpreadsheet } from "google-spreadsheet"
+import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
 import fetch from "node-fetch"
 import { configs, HTTPError } from "@budibase/backend-core"
 import { dataFilters } from "@budibase/shared-core"
@@ -61,9 +63,13 @@ const SCHEMA: Integration = {
   relationships: false,
   docs: "https://developers.google.com/sheets/api/quickstart/nodejs",
   description:
-    "Create and collaborate on online spreadsheets in real-time and from any device. ",
+    "Create and collaborate on online spreadsheets in real-time and from any device.",
   friendlyName: "Google Sheets",
   type: "Spreadsheet",
+  features: {
+    [DatasourceFeature.CONNECTION_CHECKING]: true,
+    [DatasourceFeature.FETCH_TABLE_NAMES]: true,
+  },
   datasource: {
     spreadsheetId: {
       display: "Google Sheet URL",
@@ -139,6 +145,18 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     this.client = new GoogleSpreadsheet(spreadsheetId)
   }
 
+  async testConnection(): Promise<ConnectionInfo> {
+    try {
+      await this.connect()
+      return { connected: true }
+    } catch (e: any) {
+      return {
+        connected: false,
+        error: e.message as string,
+      }
+    }
+  }
+
   getBindingIdentifier() {
     return ""
   }
@@ -224,6 +242,12 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     }
   }
 
+  async getTableNames(): Promise<string[]> {
+    await this.connect()
+    const sheets = this.client.sheetsByIndex
+    return sheets.map(s => s.title)
+  }
+
   getTableSchema(title: string, headerValues: string[], id?: string) {
     // base table
     const table: Table = {
@@ -433,9 +457,41 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   }) {
     try {
       await this.connect()
+      const hasFilters = dataFilters.hasFilters(query.filters)
+      const limit = query.paginate?.limit || 100
+      const page: number =
+        typeof query.paginate?.page === "number"
+          ? query.paginate.page
+          : parseInt(query.paginate?.page || "1")
+      const offset = (page - 1) * limit
       const sheet = this.client.sheetsByTitle[query.sheet]
-      const rows = await sheet.getRows()
-      const filtered = dataFilters.runLuceneQuery(rows, query.filters)
+      let rows: GoogleSpreadsheetRow[] = []
+      if (query.paginate && !hasFilters) {
+        rows = await sheet.getRows({
+          limit,
+          offset,
+        })
+      } else {
+        rows = await sheet.getRows()
+      }
+      // this is a special case - need to handle the _id, it doesn't exist
+      // we cannot edit the returned structure from google, it does not have
+      // setter functions and is immutable, easier to update the filters
+      // to look for the _rowNumber property rather than rowNumber
+      if (query.filters?.equal) {
+        const idFilterKeys = Object.keys(query.filters.equal).filter(filter =>
+          filter.includes(GOOGLE_SHEETS_PRIMARY_KEY)
+        )
+        for (let idFilterKey of idFilterKeys) {
+          const id = query.filters.equal[idFilterKey]
+          delete query.filters.equal[idFilterKey]
+          query.filters.equal[`_${GOOGLE_SHEETS_PRIMARY_KEY}`] = id
+        }
+      }
+      let filtered = dataFilters.runLuceneQuery(rows, query.filters)
+      if (hasFilters && query.paginate) {
+        filtered = filtered.slice(offset, offset + limit)
+      }
       const headerValues = sheet.headerValues
       let response = []
       for (let row of filtered) {
@@ -498,7 +554,12 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     const row = rows[query.rowIndex]
     if (row) {
       await row.delete()
-      return [{ deleted: query.rowIndex }]
+      return [
+        {
+          deleted: query.rowIndex,
+          [GOOGLE_SHEETS_PRIMARY_KEY]: query.rowIndex,
+        },
+      ]
     } else {
       throw new Error("Row does not exist.")
     }

package/src/integrations/index.ts

@@ -20,7 +20,7 @@ import env from "../environment"
 import { cloneDeep } from "lodash"
 import sdk from "../sdk"
 
-const DEFINITIONS: { [key: string]: Integration } = {
+const DEFINITIONS: Record<SourceName, Integration | undefined> = {
   [SourceName.POSTGRES]: postgres.schema,
   [SourceName.DYNAMODB]: dynamodb.schema,
   [SourceName.MONGODB]: mongodb.schema,
@@ -36,9 +36,10 @@ const DEFINITIONS: { [key: string]: Integration } = {
   [SourceName.GOOGLE_SHEETS]: googlesheets.schema,
   [SourceName.REDIS]: redis.schema,
   [SourceName.SNOWFLAKE]: snowflake.schema,
+  [SourceName.ORACLE]: undefined,
 }
 
-const INTEGRATIONS: { [key: string]: any } = {
+const INTEGRATIONS: Record<SourceName, any> = {
   [SourceName.POSTGRES]: postgres.integration,
   [SourceName.DYNAMODB]: dynamodb.integration,
   [SourceName.MONGODB]: mongodb.integration,
@@ -55,6 +56,7 @@ const INTEGRATIONS: { [key: string]: any } = {
   [SourceName.REDIS]: redis.integration,
   [SourceName.FIRESTORE]: firebase.integration,
   [SourceName.SNOWFLAKE]: snowflake.integration,
+  [SourceName.ORACLE]: undefined,
 }
 
 // optionally add oracle integration if the oracle binary can be installed
@@ -67,10 +69,13 @@ if (
   INTEGRATIONS[SourceName.ORACLE] = oracle.integration
 }
 
-export async function getDefinition(source: SourceName): Promise<Integration> {
+export async function getDefinition(
+  source: SourceName
+): Promise<Integration | undefined> {
   // check if its integrated, faster
-  if (DEFINITIONS[source]) {
-    return DEFINITIONS[source]
+  const definition = DEFINITIONS[source]
+  if (definition) {
+    return definition
   }
   const allDefinitions = await getDefinitions()
   return allDefinitions[source]
@@ -98,7 +103,7 @@ export async function getDefinitions() {
   }
 }
 
-export async function getIntegration(integration: string) {
+export async function getIntegration(integration: SourceName) {
   if (INTEGRATIONS[integration]) {
     return INTEGRATIONS[integration]
   }
@@ -107,7 +112,7 @@ export async function getIntegration(integration: string) {
     for (let plugin of plugins) {
       if (plugin.name === integration) {
         // need to use commonJS require due to its dynamic runtime nature
-        const retrieved: any = await getDatasourcePlugin(plugin)
+        const retrieved = await getDatasourcePlugin(plugin)
         if (retrieved.integration) {
           return retrieved.integration
         } else {