@budibase/server 2.6.19-alpha.0 → 2.6.20

package/src/integrations/elasticsearch.ts

@@ -3,8 +3,6 @@ import {
   DatasourceFieldType,
   QueryType,
   IntegrationBase,
-  DatasourceFeature,
-  ConnectionInfo,
 } from "@budibase/types"
 
 import { Client, ClientOptions } from "@elastic/elasticsearch"
@@ -22,7 +20,6 @@ const SCHEMA: Integration = {
     "Elasticsearch is a search engine based on the Lucene library. It provides a distributed, multitenant-capable full-text search engine with an HTTP web interface and schema-free JSON documents.",
   friendlyName: "ElasticSearch",
   type: "Non-relational",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     url: {
       type: DatasourceFieldType.STRING,
@@ -98,7 +95,7 @@ const SCHEMA: Integration = {
 
 class ElasticSearchIntegration implements IntegrationBase {
   private config: ElasticsearchConfig
-  private client
+  private client: any
 
   constructor(config: ElasticsearchConfig) {
     this.config = config
@@ -117,18 +114,6 @@ class ElasticSearchIntegration implements IntegrationBase {
     this.client = new Client(clientConfig)
   }
 
-  async testConnection(): Promise<ConnectionInfo> {
-    try {
-      await this.client.info()
-      return { connected: true }
-    } catch (e: any) {
-      return {
-        connected: false,
-        error: e.message as string,
-      }
-    }
-  }
-
   async create(query: { index: string; json: object }) {
     const { index, json } = query
 

package/src/integrations/firebase.ts

@@ -3,8 +3,6 @@ import {
   Integration,
   QueryType,
   IntegrationBase,
-  DatasourceFeature,
-  ConnectionInfo,
 } from "@budibase/types"
 import { Firestore, WhereFilterOp } from "@google-cloud/firestore"
 
@@ -20,7 +18,6 @@ const SCHEMA: Integration = {
   type: "Non-relational",
   description:
     "Cloud Firestore is a flexible, scalable database for mobile, web, and server development from Firebase and Google Cloud.",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     email: {
       type: DatasourceFieldType.STRING,
@@ -102,18 +99,6 @@ class FirebaseIntegration implements IntegrationBase {
     })
   }
 
-  async testConnection(): Promise<ConnectionInfo> {
-    try {
-      await this.client.listCollections()
-      return { connected: true }
-    } catch (e: any) {
-      return {
-        connected: false,
-        error: e.message as string,
-      }
-    }
-  }
-
   async create(query: { json: object; extra: { [key: string]: string } }) {
     try {
       const documentReference = this.client

package/src/integrations/googlesheets.ts

@@ -1,6 +1,4 @@
 import {
-  ConnectionInfo,
-  DatasourceFeature,
   DatasourceFieldType,
   DatasourcePlus,
   FieldType,
@@ -17,7 +15,7 @@ import {
 } from "@budibase/types"
 import { OAuth2Client } from "google-auth-library"
 import { buildExternalTableId, finaliseExternalTables } from "./utils"
-import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
+import { GoogleSpreadsheet } from "google-spreadsheet"
 import fetch from "node-fetch"
 import { configs, HTTPError } from "@budibase/backend-core"
 import { dataFilters } from "@budibase/shared-core"
@@ -66,7 +64,6 @@ const SCHEMA: Integration = {
     "Create and collaborate on online spreadsheets in real-time and from any device. ",
   friendlyName: "Google Sheets",
   type: "Spreadsheet",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     spreadsheetId: {
       display: "Google Sheet URL",
@@ -142,19 +139,6 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     this.client = new GoogleSpreadsheet(spreadsheetId)
   }
 
-  async testConnection(): Promise<ConnectionInfo> {
-    try {
-      await this.connect()
-      await this.client.loadInfo()
-      return { connected: true }
-    } catch (e: any) {
-      return {
-        connected: false,
-        error: e.message as string,
-      }
-    }
-  }
-
   getBindingIdentifier() {
     return ""
   }
@@ -450,20 +434,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     try {
       await this.connect()
       const sheet = this.client.sheetsByTitle[query.sheet]
-      let rows: GoogleSpreadsheetRow[] = []
-      if (query.paginate) {
-        const limit = query.paginate.limit || 100
-        let page: number =
-          typeof query.paginate.page === "number"
-            ? query.paginate.page
-            : parseInt(query.paginate.page || "1")
-        rows = await sheet.getRows({
-          limit,
-          offset: (page - 1) * limit,
-        })
-      } else {
-        rows = await sheet.getRows()
-      }
+      const rows = await sheet.getRows()
       const filtered = dataFilters.runLuceneQuery(rows, query.filters)
       const headerValues = sheet.headerValues
       let response = []

package/src/integrations/index.ts

@@ -20,7 +20,7 @@ import env from "../environment"
 import { cloneDeep } from "lodash"
 import sdk from "../sdk"
 
-const DEFINITIONS: Record<SourceName, Integration | undefined> = {
+const DEFINITIONS: { [key: string]: Integration } = {
   [SourceName.POSTGRES]: postgres.schema,
   [SourceName.DYNAMODB]: dynamodb.schema,
   [SourceName.MONGODB]: mongodb.schema,
@@ -36,10 +36,9 @@ const DEFINITIONS: Record<SourceName, Integration | undefined> = {
   [SourceName.GOOGLE_SHEETS]: googlesheets.schema,
   [SourceName.REDIS]: redis.schema,
   [SourceName.SNOWFLAKE]: snowflake.schema,
-  [SourceName.ORACLE]: undefined,
 }
 
-const INTEGRATIONS: Record<SourceName, any> = {
+const INTEGRATIONS: { [key: string]: any } = {
   [SourceName.POSTGRES]: postgres.integration,
   [SourceName.DYNAMODB]: dynamodb.integration,
   [SourceName.MONGODB]: mongodb.integration,
@@ -56,7 +55,6 @@ const INTEGRATIONS: Record<SourceName, any> = {
   [SourceName.REDIS]: redis.integration,
   [SourceName.FIRESTORE]: firebase.integration,
   [SourceName.SNOWFLAKE]: snowflake.integration,
-  [SourceName.ORACLE]: undefined,
 }
 
 // optionally add oracle integration if the oracle binary can be installed
@@ -69,13 +67,10 @@ if (
   INTEGRATIONS[SourceName.ORACLE] = oracle.integration
 }
 
-export async function getDefinition(
-  source: SourceName
-): Promise<Integration | undefined> {
+export async function getDefinition(source: SourceName): Promise<Integration> {
   // check if its integrated, faster
-  const definition = DEFINITIONS[source]
-  if (definition) {
-    return definition
+  if (DEFINITIONS[source]) {
+    return DEFINITIONS[source]
   }
   const allDefinitions = await getDefinitions()
   return allDefinitions[source]
@@ -103,7 +98,7 @@ export async function getDefinitions() {
   }
 }
 
-export async function getIntegration(integration: SourceName) {
+export async function getIntegration(integration: string) {
   if (INTEGRATIONS[integration]) {
     return INTEGRATIONS[integration]
   }
@@ -112,7 +107,7 @@ export async function getIntegration(integration: SourceName) {
     for (let plugin of plugins) {
       if (plugin.name === integration) {
         // need to use commonJS require due to its dynamic runtime nature
-        const retrieved = await getDatasourcePlugin(plugin)
+        const retrieved: any = await getDatasourcePlugin(plugin)
         if (retrieved.integration) {
           return retrieved.integration
         } else {

package/src/integrations/microsoftSqlServer.ts

@@ -8,8 +8,6 @@ import {
   QueryType,
   SqlQuery,
   DatasourcePlus,
-  DatasourceFeature,
-  ConnectionInfo,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -41,7 +39,6 @@ const SCHEMA: Integration = {
     "Microsoft SQL Server is a relational database management system developed by Microsoft. ",
   friendlyName: "MS SQL Server",
   type: "Relational",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     user: {
       type: DatasourceFieldType.STRING,
@@ -124,19 +121,6 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
     }
   }
 
-  async testConnection() {
-    const response: ConnectionInfo = {
-      connected: false,
-    }
-    try {
-      await this.connect()
-      response.connected = true
-    } catch (e: any) {
-      response.error = e.message as string
-    }
-    return response
-  }
-
   getBindingIdentifier(): string {
     return `@p${this.index++}`
   }

package/src/integrations/mongodb.ts

@@ -3,8 +3,6 @@ import {
   DatasourceFieldType,
   QueryType,
   IntegrationBase,
-  DatasourceFeature,
-  ConnectionInfo,
 } from "@budibase/types"
 import {
   MongoClient,
@@ -40,7 +38,6 @@ const getSchema = () => {
     type: "Non-relational",
     description:
       "MongoDB is a general purpose, document-based, distributed database built for modern application developers and for the cloud era.",
-    features: [DatasourceFeature.CONNECTION_CHECKING],
     datasource: {
       connectionString: {
         type: DatasourceFieldType.STRING,
@@ -361,19 +358,6 @@ class MongoIntegration implements IntegrationBase {
     this.client = new MongoClient(config.connectionString, options)
   }
 
-  async testConnection() {
-    const response: ConnectionInfo = {
-      connected: false,
-    }
-    try {
-      await this.connect()
-      response.connected = true
-    } catch (e: any) {
-      response.error = e.message as string
-    }
-    return response
-  }
-
   async connect() {
     return this.client.connect()
   }

package/src/integrations/mysql.ts

@@ -7,8 +7,6 @@ import {
   Table,
   TableSchema,
   DatasourcePlus,
-  DatasourceFeature,
-  ConnectionInfo,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -22,11 +20,18 @@ import { NUMBER_REGEX } from "../utilities"
 import Sql from "./base/sql"
 import { MySQLColumn } from "./base/types"
 
-import mysql from "mysql2/promise"
+const mysql = require("mysql2/promise")
 
-interface MySQLConfig extends mysql.ConnectionOptions {
+interface MySQLConfig {
+  host: string
+  port: number
+  user: string
+  password: string
   database: string
+  ssl?: { [key: string]: any }
   rejectUnauthorized: boolean
+  typeCast: Function
+  multipleStatements: boolean
 }
 
 const SCHEMA: Integration = {
@@ -36,7 +41,6 @@ const SCHEMA: Integration = {
   type: "Relational",
   description:
     "MySQL Database Service is a fully managed database service to deploy cloud-native applications. ",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     host: {
       type: DatasourceFieldType.STRING,
@@ -88,6 +92,8 @@ const SCHEMA: Integration = {
   },
 }
 
+const TimezoneAwareDateTypes = ["timestamp"]
+
 function bindingTypeCoerce(bindings: any[]) {
   for (let i = 0; i < bindings.length; i++) {
     const binding = bindings[i]
@@ -114,7 +120,7 @@ function bindingTypeCoerce(bindings: any[]) {
 
 class MySQLIntegration extends Sql implements DatasourcePlus {
   private config: MySQLConfig
-  private client?: mysql.Connection
+  private client: any
   public tables: Record<string, Table> = {}
   public schemaErrors: Record<string, string> = {}
 
@@ -128,8 +134,7 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
     if (
       config.rejectUnauthorized != null &&
       !config.rejectUnauthorized &&
-      config.ssl &&
-      typeof config.ssl !== "string"
+      config.ssl
     ) {
       config.ssl.rejectUnauthorized = config.rejectUnauthorized
     }
@@ -155,22 +160,6 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
     }
   }
 
-  async testConnection() {
-    const response: ConnectionInfo = {
-      connected: false,
-    }
-    try {
-      const [result] = await this.internalQuery(
-        { sql: "SELECT 1+1 AS checkRes" },
-        { connect: true }
-      )
-      response.connected = result?.checkRes == 2
-    } catch (e: any) {
-      response.error = e.message as string
-    }
-    return response
-  }
-
   getBindingIdentifier(): string {
     return "?"
   }
@@ -184,7 +173,7 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
   }
 
   async disconnect() {
-    await this.client!.end()
+    await this.client.end()
   }
 
   async internalQuery(
@@ -203,10 +192,10 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
         ? baseBindings
         : bindingTypeCoerce(baseBindings)
       // Node MySQL is callback based, so we must wrap our call in a promise
-      const response = await this.client!.query(query.sql, bindings)
+      const response = await this.client.query(query.sql, bindings)
       return response[0]
     } finally {
-      if (opts?.connect && this.client) {
+      if (opts?.connect) {
         await this.disconnect()
       }
     }

package/src/integrations/oracle.ts

@@ -7,8 +7,6 @@ import {
   SqlQuery,
   Table,
   DatasourcePlus,
-  DatasourceFeature,
-  ConnectionInfo,
 } from "@budibase/types"
 import {
   buildExternalTableId,
@@ -26,7 +24,12 @@ import {
   ExecuteOptions,
   Result,
 } from "oracledb"
-import { OracleTable, OracleColumn, OracleColumnsResponse } from "./base/types"
+import {
+  OracleTable,
+  OracleColumn,
+  OracleColumnsResponse,
+  OracleConstraint,
+} from "./base/types"
 let oracledb: any
 try {
   oracledb = require("oracledb")
@@ -50,7 +53,6 @@ const SCHEMA: Integration = {
   type: "Relational",
   description:
     "Oracle Database is an object-relational database management system developed by Oracle Corporation",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     host: {
       type: DatasourceFieldType.STRING,
@@ -323,30 +325,6 @@ class OracleIntegration extends Sql implements DatasourcePlus {
     this.schemaErrors = final.errors
   }
 
-  async testConnection() {
-    const response: ConnectionInfo = {
-      connected: false,
-    }
-    let connection
-    try {
-      connection = await this.getConnection()
-      response.connected = true
-    } catch (err: any) {
-      response.connected = false
-      response.error = err.message
-    } finally {
-      if (connection) {
-        try {
-          await connection.close()
-        } catch (err: any) {
-          response.connected = false
-          response.error = err.message
-        }
-      }
-    }
-    return response
-  }
-
   private async internalQuery<T>(query: SqlQuery): Promise<Result<T>> {
     let connection
     try {

package/src/integrations/postgres.ts

@@ -6,8 +6,6 @@ import {
   SqlQuery,
   Table,
   DatasourcePlus,
-  DatasourceFeature,
-  ConnectionInfo,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -20,7 +18,7 @@ import Sql from "./base/sql"
 import { PostgresColumn } from "./base/types"
 import { escapeDangerousCharacters } from "../utilities"
 
-import { Client, types } from "pg"
+const { Client, types } = require("pg")
 
 // Return "date" and "timestamp" types as plain strings.
 // This lets us reference the original stored timezone.
@@ -52,7 +50,6 @@ const SCHEMA: Integration = {
   type: "Relational",
   description:
     "PostgreSQL, also known as Postgres, is a free and open-source relational database management system emphasizing extensibility and SQL compliance.",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     host: {
       type: DatasourceFieldType.STRING,
@@ -117,7 +114,7 @@ const SCHEMA: Integration = {
 }
 
 class PostgresIntegration extends Sql implements DatasourcePlus {
-  private readonly client: Client
+  private readonly client: any
   private readonly config: PostgresConfig
   private index: number = 1
   private open: boolean
@@ -153,21 +150,6 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
     this.open = false
   }
 
-  async testConnection() {
-    const response: ConnectionInfo = {
-      connected: false,
-    }
-    try {
-      await this.openConnection()
-      response.connected = true
-    } catch (e: any) {
-      response.error = e.message as string
-    } finally {
-      await this.closeConnection()
-    }
-    return response
-  }
-
   getBindingIdentifier(): string {
     return `$${this.index++}`
   }
@@ -181,7 +163,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
     if (!this.config.schema) {
       this.config.schema = "public"
     }
-    await this.client.query(`SET search_path TO ${this.config.schema}`)
+    this.client.query(`SET search_path TO ${this.config.schema}`)
     this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = '${this.config.schema}'`
     this.open = true
   }

package/src/integrations/redis.ts

@@ -1,10 +1,4 @@
-import {
-  ConnectionInfo,
-  DatasourceFeature,
-  DatasourceFieldType,
-  Integration,
-  QueryType,
-} from "@budibase/types"
+import { DatasourceFieldType, Integration, QueryType } from "@budibase/types"
 import Redis from "ioredis"
 
 interface RedisConfig {
@@ -17,11 +11,9 @@ interface RedisConfig {
 
 const SCHEMA: Integration = {
   docs: "https://redis.io/docs/",
-  description:
-    "Redis is a caching tool, providing powerful key-value store capabilities.",
+  description: "",
   friendlyName: "Redis",
   type: "Non-relational",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     host: {
       type: "string",
@@ -94,7 +86,7 @@ const SCHEMA: Integration = {
 
 class RedisIntegration {
   private readonly config: RedisConfig
-  private client
+  private client: any
 
   constructor(config: RedisConfig) {
     this.config = config
@@ -107,21 +99,6 @@ class RedisIntegration {
     })
   }
 
-  async testConnection() {
-    const response: ConnectionInfo = {
-      connected: false,
-    }
-    try {
-      await this.client.ping()
-      response.connected = true
-    } catch (e: any) {
-      response.error = e.message as string
-    } finally {
-      await this.disconnect()
-    }
-    return response
-  }
-
   async disconnect() {
     return this.client.quit()
   }

package/src/integrations/s3.ts

@@ -3,12 +3,10 @@ import {
   QueryType,
   IntegrationBase,
   DatasourceFieldType,
-  DatasourceFeature,
-  ConnectionInfo,
 } from "@budibase/types"
 
-import AWS from "aws-sdk"
-import csv from "csvtojson"
+const AWS = require("aws-sdk")
+const csv = require("csvtojson")
 
 interface S3Config {
   region: string
@@ -24,7 +22,6 @@ const SCHEMA: Integration = {
     "Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.",
   friendlyName: "Amazon S3",
   type: "Object store",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     region: {
       type: "string",
@@ -155,7 +152,7 @@ const SCHEMA: Integration = {
 
 class S3Integration implements IntegrationBase {
   private readonly config: S3Config
-  private client
+  private client: any
 
   constructor(config: S3Config) {
     this.config = config
@@ -168,19 +165,6 @@ class S3Integration implements IntegrationBase {
     this.client = new AWS.S3(this.config)
   }
 
-  async testConnection() {
-    const response: ConnectionInfo = {
-      connected: false,
-    }
-    try {
-      await this.client.listBuckets().promise()
-      response.connected = true
-    } catch (e: any) {
-      response.error = e.message as string
-    }
-    return response
-  }
-
   async create(query: {
     bucket: string
     location: string

package/src/integrations/snowflake.ts

@@ -1,10 +1,4 @@
-import {
-  ConnectionInfo,
-  DatasourceFeature,
-  Integration,
-  QueryType,
-  SqlQuery,
-} from "@budibase/types"
+import { Integration, QueryType, SqlQuery } from "@budibase/types"
 import { Snowflake } from "snowflake-promise"
 
 interface SnowflakeConfig {
@@ -22,7 +16,6 @@ const SCHEMA: Integration = {
     "Snowflake is a solution for data warehousing, data lakes, data engineering, data science, data application development, and securely sharing and consuming shared data.",
   friendlyName: "Snowflake",
   type: "Relational",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
   datasource: {
     account: {
       type: "string",
@@ -72,18 +65,6 @@ class SnowflakeIntegration {
     this.client = new Snowflake(config)
   }
 
-  async testConnection(): Promise<ConnectionInfo> {
-    try {
-      await this.client.connect()
-      return { connected: true }
-    } catch (e: any) {
-      return {
-        connected: false,
-        error: e.message as string,
-      }
-    }
-  }
-
   async internalQuery(query: SqlQuery) {
     await this.client.connect()
     try {

package/src/migrations/functions/backfill/app/queries.ts

@@ -33,7 +33,7 @@ export const backfill = async (appDb: any, timestamp: string | number) => {
         datasource = {
           type: "unknown",
           _id: query.datasourceId,
-          source: "unknown" as SourceName,
+          source: SourceName.UNKNOWN,
         }
       } else {
         throw e