@budibase/server 2.6.18 → 2.6.19-alpha.0

package/src/integrations/dynamodb.ts

@@ -3,10 +3,13 @@ import {
   DatasourceFieldType,
   QueryType,
   IntegrationBase,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"
 
 import AWS from "aws-sdk"
 import { AWS_REGION } from "../db/dynamoClient"
+import { DocumentClient } from "aws-sdk/clients/dynamodb"
 
 interface DynamoDBConfig {
   region: string
@@ -22,6 +25,7 @@ const SCHEMA: Integration = {
     "Amazon DynamoDB is a key-value and document database that delivers single-digit millisecond performance at any scale.",
   friendlyName: "DynamoDB",
   type: "Non-relational",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     region: {
       type: DatasourceFieldType.STRING,
@@ -128,7 +132,7 @@ const SCHEMA: Integration = {
 
 class DynamoDBIntegration implements IntegrationBase {
   private config: DynamoDBConfig
-  private client: any
+  private client
 
   constructor(config: DynamoDBConfig) {
     this.config = config
@@ -148,7 +152,23 @@ class DynamoDBIntegration implements IntegrationBase {
     this.client = new AWS.DynamoDB.DocumentClient(this.config)
   }
 
-  async create(query: { table: string; json: object }) {
+  async testConnection() {
+    const response: ConnectionInfo = {
+      connected: false,
+    }
+    try {
+      const scanRes = await new AWS.DynamoDB(this.config).listTables().promise()
+      response.connected = !!scanRes.$response
+    } catch (e: any) {
+      response.error = e.message as string
+    }
+    return response
+  }
+
+  async create(query: {
+    table: string
+    json: Omit<DocumentClient.PutItemInput, "TableName">
+  }) {
     const params = {
       TableName: query.table,
       ...query.json,
@@ -189,7 +209,10 @@ class DynamoDBIntegration implements IntegrationBase {
     return new AWS.DynamoDB(this.config).describeTable(params).promise()
   }
 
-  async get(query: { table: string; json: object }) {
+  async get(query: {
+    table: string
+    json: Omit<DocumentClient.GetItemInput, "TableName">
+  }) {
     const params = {
       TableName: query.table,
       ...query.json,
@@ -197,7 +220,10 @@ class DynamoDBIntegration implements IntegrationBase {
     return this.client.get(params).promise()
   }
 
-  async update(query: { table: string; json: object }) {
+  async update(query: {
+    table: string
+    json: Omit<DocumentClient.UpdateItemInput, "TableName">
+  }) {
     const params = {
       TableName: query.table,
       ...query.json,
@@ -205,7 +231,10 @@ class DynamoDBIntegration implements IntegrationBase {
     return this.client.update(params).promise()
   }
 
-  async delete(query: { table: string; json: object }) {
+  async delete(query: {
+    table: string
+    json: Omit<DocumentClient.DeleteItemInput, "TableName">
+  }) {
     const params = {
       TableName: query.table,
       ...query.json,

package/src/integrations/elasticsearch.ts

@@ -3,6 +3,8 @@ import {
   DatasourceFieldType,
   QueryType,
   IntegrationBase,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"
 
 import { Client, ClientOptions } from "@elastic/elasticsearch"
@@ -20,6 +22,7 @@ const SCHEMA: Integration = {
     "Elasticsearch is a search engine based on the Lucene library. It provides a distributed, multitenant-capable full-text search engine with an HTTP web interface and schema-free JSON documents.",
   friendlyName: "ElasticSearch",
   type: "Non-relational",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     url: {
       type: DatasourceFieldType.STRING,
@@ -95,7 +98,7 @@ const SCHEMA: Integration = {
 
 class ElasticSearchIntegration implements IntegrationBase {
   private config: ElasticsearchConfig
-  private client: any
+  private client
 
   constructor(config: ElasticsearchConfig) {
     this.config = config
@@ -114,6 +117,18 @@ class ElasticSearchIntegration implements IntegrationBase {
     this.client = new Client(clientConfig)
   }
 
+  async testConnection(): Promise<ConnectionInfo> {
+    try {
+      await this.client.info()
+      return { connected: true }
+    } catch (e: any) {
+      return {
+        connected: false,
+        error: e.message as string,
+      }
+    }
+  }
+
   async create(query: { index: string; json: object }) {
     const { index, json } = query
 

package/src/integrations/firebase.ts

@@ -3,6 +3,8 @@ import {
   Integration,
   QueryType,
   IntegrationBase,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"
 import { Firestore, WhereFilterOp } from "@google-cloud/firestore"
 
@@ -18,6 +20,7 @@ const SCHEMA: Integration = {
   type: "Non-relational",
   description:
     "Cloud Firestore is a flexible, scalable database for mobile, web, and server development from Firebase and Google Cloud.",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     email: {
       type: DatasourceFieldType.STRING,
@@ -99,6 +102,18 @@ class FirebaseIntegration implements IntegrationBase {
     })
   }
 
+  async testConnection(): Promise<ConnectionInfo> {
+    try {
+      await this.client.listCollections()
+      return { connected: true }
+    } catch (e: any) {
+      return {
+        connected: false,
+        error: e.message as string,
+      }
+    }
+  }
+
   async create(query: { json: object; extra: { [key: string]: string } }) {
     try {
       const documentReference = this.client

package/src/integrations/googlesheets.ts

@@ -1,4 +1,6 @@
 import {
+  ConnectionInfo,
+  DatasourceFeature,
   DatasourceFieldType,
   DatasourcePlus,
   FieldType,
@@ -15,7 +17,7 @@ import {
 } from "@budibase/types"
 import { OAuth2Client } from "google-auth-library"
 import { buildExternalTableId, finaliseExternalTables } from "./utils"
-import { GoogleSpreadsheet } from "google-spreadsheet"
+import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
 import fetch from "node-fetch"
 import { configs, HTTPError } from "@budibase/backend-core"
 import { dataFilters } from "@budibase/shared-core"
@@ -64,6 +66,7 @@ const SCHEMA: Integration = {
     "Create and collaborate on online spreadsheets in real-time and from any device. ",
   friendlyName: "Google Sheets",
   type: "Spreadsheet",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     spreadsheetId: {
       display: "Google Sheet URL",
@@ -139,6 +142,19 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     this.client = new GoogleSpreadsheet(spreadsheetId)
   }
 
+  async testConnection(): Promise<ConnectionInfo> {
+    try {
+      await this.connect()
+      await this.client.loadInfo()
+      return { connected: true }
+    } catch (e: any) {
+      return {
+        connected: false,
+        error: e.message as string,
+      }
+    }
+  }
+
   getBindingIdentifier() {
     return ""
   }
@@ -434,7 +450,20 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     try {
       await this.connect()
       const sheet = this.client.sheetsByTitle[query.sheet]
-      const rows = await sheet.getRows()
+      let rows: GoogleSpreadsheetRow[] = []
+      if (query.paginate) {
+        const limit = query.paginate.limit || 100
+        let page: number =
+          typeof query.paginate.page === "number"
+            ? query.paginate.page
+            : parseInt(query.paginate.page || "1")
+        rows = await sheet.getRows({
+          limit,
+          offset: (page - 1) * limit,
+        })
+      } else {
+        rows = await sheet.getRows()
+      }
       const filtered = dataFilters.runLuceneQuery(rows, query.filters)
       const headerValues = sheet.headerValues
       let response = []

package/src/integrations/index.ts

@@ -20,7 +20,7 @@ import env from "../environment"
 import { cloneDeep } from "lodash"
 import sdk from "../sdk"
 
-const DEFINITIONS: { [key: string]: Integration } = {
+const DEFINITIONS: Record<SourceName, Integration | undefined> = {
   [SourceName.POSTGRES]: postgres.schema,
   [SourceName.DYNAMODB]: dynamodb.schema,
   [SourceName.MONGODB]: mongodb.schema,
@@ -36,9 +36,10 @@ const DEFINITIONS: { [key: string]: Integration } = {
   [SourceName.GOOGLE_SHEETS]: googlesheets.schema,
   [SourceName.REDIS]: redis.schema,
   [SourceName.SNOWFLAKE]: snowflake.schema,
+  [SourceName.ORACLE]: undefined,
 }
 
-const INTEGRATIONS: { [key: string]: any } = {
+const INTEGRATIONS: Record<SourceName, any> = {
   [SourceName.POSTGRES]: postgres.integration,
   [SourceName.DYNAMODB]: dynamodb.integration,
   [SourceName.MONGODB]: mongodb.integration,
@@ -55,6 +56,7 @@ const INTEGRATIONS: { [key: string]: any } = {
   [SourceName.REDIS]: redis.integration,
   [SourceName.FIRESTORE]: firebase.integration,
   [SourceName.SNOWFLAKE]: snowflake.integration,
+  [SourceName.ORACLE]: undefined,
 }
 
 // optionally add oracle integration if the oracle binary can be installed
@@ -67,10 +69,13 @@ if (
   INTEGRATIONS[SourceName.ORACLE] = oracle.integration
 }
 
-export async function getDefinition(source: SourceName): Promise<Integration> {
+export async function getDefinition(
+  source: SourceName
+): Promise<Integration | undefined> {
   // check if its integrated, faster
-  if (DEFINITIONS[source]) {
-    return DEFINITIONS[source]
+  const definition = DEFINITIONS[source]
+  if (definition) {
+    return definition
   }
   const allDefinitions = await getDefinitions()
   return allDefinitions[source]
@@ -98,7 +103,7 @@ export async function getDefinitions() {
   }
 }
 
-export async function getIntegration(integration: string) {
+export async function getIntegration(integration: SourceName) {
   if (INTEGRATIONS[integration]) {
     return INTEGRATIONS[integration]
   }
@@ -107,7 +112,7 @@ export async function getIntegration(integration: string) {
     for (let plugin of plugins) {
       if (plugin.name === integration) {
         // need to use commonJS require due to its dynamic runtime nature
-        const retrieved: any = await getDatasourcePlugin(plugin)
+        const retrieved = await getDatasourcePlugin(plugin)
         if (retrieved.integration) {
           return retrieved.integration
         } else {

package/src/integrations/microsoftSqlServer.ts

@@ -8,6 +8,8 @@ import {
   QueryType,
   SqlQuery,
   DatasourcePlus,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -39,6 +41,7 @@ const SCHEMA: Integration = {
     "Microsoft SQL Server is a relational database management system developed by Microsoft. ",
   friendlyName: "MS SQL Server",
   type: "Relational",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     user: {
       type: DatasourceFieldType.STRING,
@@ -121,6 +124,19 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
     }
   }
 
+  async testConnection() {
+    const response: ConnectionInfo = {
+      connected: false,
+    }
+    try {
+      await this.connect()
+      response.connected = true
+    } catch (e: any) {
+      response.error = e.message as string
+    }
+    return response
+  }
+
   getBindingIdentifier(): string {
     return `@p${this.index++}`
   }

package/src/integrations/mongodb.ts

@@ -3,6 +3,8 @@ import {
   DatasourceFieldType,
   QueryType,
   IntegrationBase,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"
 import {
   MongoClient,
@@ -38,6 +40,7 @@ const getSchema = () => {
     type: "Non-relational",
     description:
       "MongoDB is a general purpose, document-based, distributed database built for modern application developers and for the cloud era.",
+    features: [DatasourceFeature.CONNECTION_CHECKING],
     datasource: {
       connectionString: {
         type: DatasourceFieldType.STRING,
@@ -358,6 +361,19 @@ class MongoIntegration implements IntegrationBase {
     this.client = new MongoClient(config.connectionString, options)
   }
 
+  async testConnection() {
+    const response: ConnectionInfo = {
+      connected: false,
+    }
+    try {
+      await this.connect()
+      response.connected = true
+    } catch (e: any) {
+      response.error = e.message as string
+    }
+    return response
+  }
+
   async connect() {
     return this.client.connect()
   }

package/src/integrations/mysql.ts

@@ -7,6 +7,8 @@ import {
   Table,
   TableSchema,
   DatasourcePlus,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -20,18 +22,11 @@ import { NUMBER_REGEX } from "../utilities"
 import Sql from "./base/sql"
 import { MySQLColumn } from "./base/types"
 
-const mysql = require("mysql2/promise")
+import mysql from "mysql2/promise"
 
-interface MySQLConfig {
-  host: string
-  port: number
-  user: string
-  password: string
+interface MySQLConfig extends mysql.ConnectionOptions {
   database: string
-  ssl?: { [key: string]: any }
   rejectUnauthorized: boolean
-  typeCast: Function
-  multipleStatements: boolean
 }
 
 const SCHEMA: Integration = {
@@ -41,6 +36,7 @@ const SCHEMA: Integration = {
   type: "Relational",
   description:
     "MySQL Database Service is a fully managed database service to deploy cloud-native applications. ",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     host: {
       type: DatasourceFieldType.STRING,
@@ -92,8 +88,6 @@ const SCHEMA: Integration = {
   },
 }
 
-const TimezoneAwareDateTypes = ["timestamp"]
-
 function bindingTypeCoerce(bindings: any[]) {
   for (let i = 0; i < bindings.length; i++) {
     const binding = bindings[i]
@@ -120,7 +114,7 @@ function bindingTypeCoerce(bindings: any[]) {
 
 class MySQLIntegration extends Sql implements DatasourcePlus {
   private config: MySQLConfig
-  private client: any
+  private client?: mysql.Connection
   public tables: Record<string, Table> = {}
   public schemaErrors: Record<string, string> = {}
 
@@ -134,7 +128,8 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
     if (
       config.rejectUnauthorized != null &&
       !config.rejectUnauthorized &&
-      config.ssl
+      config.ssl &&
+      typeof config.ssl !== "string"
     ) {
       config.ssl.rejectUnauthorized = config.rejectUnauthorized
     }
@@ -160,6 +155,22 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
     }
   }
 
+  async testConnection() {
+    const response: ConnectionInfo = {
+      connected: false,
+    }
+    try {
+      const [result] = await this.internalQuery(
+        { sql: "SELECT 1+1 AS checkRes" },
+        { connect: true }
+      )
+      response.connected = result?.checkRes == 2
+    } catch (e: any) {
+      response.error = e.message as string
+    }
+    return response
+  }
+
   getBindingIdentifier(): string {
     return "?"
   }
@@ -173,7 +184,7 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
   }
 
   async disconnect() {
-    await this.client.end()
+    await this.client!.end()
   }
 
   async internalQuery(
@@ -192,10 +203,10 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
         ? baseBindings
         : bindingTypeCoerce(baseBindings)
       // Node MySQL is callback based, so we must wrap our call in a promise
-      const response = await this.client.query(query.sql, bindings)
+      const response = await this.client!.query(query.sql, bindings)
       return response[0]
     } finally {
-      if (opts?.connect) {
+      if (opts?.connect && this.client) {
         await this.disconnect()
       }
     }

package/src/integrations/oracle.ts

@@ -7,6 +7,8 @@ import {
   SqlQuery,
   Table,
   DatasourcePlus,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"
 import {
   buildExternalTableId,
@@ -24,12 +26,7 @@ import {
   ExecuteOptions,
   Result,
 } from "oracledb"
-import {
-  OracleTable,
-  OracleColumn,
-  OracleColumnsResponse,
-  OracleConstraint,
-} from "./base/types"
+import { OracleTable, OracleColumn, OracleColumnsResponse } from "./base/types"
 let oracledb: any
 try {
   oracledb = require("oracledb")
@@ -53,6 +50,7 @@ const SCHEMA: Integration = {
   type: "Relational",
   description:
     "Oracle Database is an object-relational database management system developed by Oracle Corporation",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     host: {
       type: DatasourceFieldType.STRING,
@@ -325,6 +323,30 @@ class OracleIntegration extends Sql implements DatasourcePlus {
     this.schemaErrors = final.errors
   }
 
+  async testConnection() {
+    const response: ConnectionInfo = {
+      connected: false,
+    }
+    let connection
+    try {
+      connection = await this.getConnection()
+      response.connected = true
+    } catch (err: any) {
+      response.connected = false
+      response.error = err.message
+    } finally {
+      if (connection) {
+        try {
+          await connection.close()
+        } catch (err: any) {
+          response.connected = false
+          response.error = err.message
+        }
+      }
+    }
+    return response
+  }
+
   private async internalQuery<T>(query: SqlQuery): Promise<Result<T>> {
     let connection
     try {

package/src/integrations/postgres.ts

@@ -6,6 +6,8 @@ import {
   SqlQuery,
   Table,
   DatasourcePlus,
+  DatasourceFeature,
+  ConnectionInfo,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -18,7 +20,7 @@ import Sql from "./base/sql"
 import { PostgresColumn } from "./base/types"
 import { escapeDangerousCharacters } from "../utilities"
 
-const { Client, types } = require("pg")
+import { Client, types } from "pg"
 
 // Return "date" and "timestamp" types as plain strings.
 // This lets us reference the original stored timezone.
@@ -50,6 +52,7 @@ const SCHEMA: Integration = {
   type: "Relational",
   description:
     "PostgreSQL, also known as Postgres, is a free and open-source relational database management system emphasizing extensibility and SQL compliance.",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     host: {
       type: DatasourceFieldType.STRING,
@@ -114,7 +117,7 @@ const SCHEMA: Integration = {
 }
 
 class PostgresIntegration extends Sql implements DatasourcePlus {
-  private readonly client: any
+  private readonly client: Client
   private readonly config: PostgresConfig
   private index: number = 1
   private open: boolean
@@ -150,6 +153,21 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
     this.open = false
   }
 
+  async testConnection() {
+    const response: ConnectionInfo = {
+      connected: false,
+    }
+    try {
+      await this.openConnection()
+      response.connected = true
+    } catch (e: any) {
+      response.error = e.message as string
+    } finally {
+      await this.closeConnection()
+    }
+    return response
+  }
+
   getBindingIdentifier(): string {
     return `$${this.index++}`
   }
@@ -163,7 +181,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
     if (!this.config.schema) {
       this.config.schema = "public"
     }
-    this.client.query(`SET search_path TO ${this.config.schema}`)
+    await this.client.query(`SET search_path TO ${this.config.schema}`)
     this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = '${this.config.schema}'`
     this.open = true
   }

package/src/integrations/redis.ts

@@ -1,4 +1,10 @@
-import { DatasourceFieldType, Integration, QueryType } from "@budibase/types"
+import {
+  ConnectionInfo,
+  DatasourceFeature,
+  DatasourceFieldType,
+  Integration,
+  QueryType,
+} from "@budibase/types"
 import Redis from "ioredis"
 
 interface RedisConfig {
@@ -11,9 +17,11 @@ interface RedisConfig {
 
 const SCHEMA: Integration = {
   docs: "https://redis.io/docs/",
-  description: "",
+  description:
+    "Redis is a caching tool, providing powerful key-value store capabilities.",
   friendlyName: "Redis",
   type: "Non-relational",
+  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     host: {
       type: "string",
@@ -86,7 +94,7 @@ const SCHEMA: Integration = {
 
 class RedisIntegration {
   private readonly config: RedisConfig
-  private client: any
+  private client
 
   constructor(config: RedisConfig) {
     this.config = config
@@ -99,6 +107,21 @@ class RedisIntegration {
     })
   }
 
+  async testConnection() {
+    const response: ConnectionInfo = {
+      connected: false,
+    }
+    try {
+      await this.client.ping()
+      response.connected = true
+    } catch (e: any) {
+      response.error = e.message as string
+    } finally {
+      await this.disconnect()
+    }
+    return response
+  }
+
   async disconnect() {
     return this.client.quit()
   }