@budibase/server 0.9.188 → 0.9.190-alpha.2

package/src/integrations/oracle.ts

@@ -0,0 +1,458 @@
+import {
+  Integration,
+  DatasourceFieldTypes,
+  QueryTypes,
+  SqlQuery,
+  QueryJson,
+  Operation,
+} from "../definitions/datasource"
+import {
+  finaliseExternalTables,
+  getSqlQuery,
+  buildExternalTableId,
+  convertSqlType,
+  SqlClients,
+} from "./utils"
+import oracledb, {
+  ExecuteOptions,
+  Result,
+  Connection,
+  ConnectionAttributes,
+  BindParameters,
+} from "oracledb"
+import Sql from "./base/sql"
+import { Table } from "../definitions/common"
+import { DatasourcePlus } from "./base/datasourcePlus"
+import { FieldTypes } from "../constants"
+
+module OracleModule {
+  oracledb.outFormat = oracledb.OUT_FORMAT_OBJECT
+
+  interface OracleConfig {
+    host: string
+    port: number
+    database: string
+    user: string
+    password: string
+  }
+
+  const SCHEMA: Integration = {
+    docs: "https://github.com/oracle/node-oracledb",
+    plus: true,
+    friendlyName: "Oracle",
+    description:
+      "Oracle Database is an object-relational database management system developed by Oracle Corporation",
+    datasource: {
+      host: {
+        type: DatasourceFieldTypes.STRING,
+        default: "localhost",
+        required: true,
+      },
+      port: {
+        type: DatasourceFieldTypes.NUMBER,
+        required: true,
+        default: 1521,
+      },
+      database: {
+        type: DatasourceFieldTypes.STRING,
+        required: true,
+      },
+      user: {
+        type: DatasourceFieldTypes.STRING,
+        required: true,
+      },
+      password: {
+        type: DatasourceFieldTypes.PASSWORD,
+        required: true,
+      },
+    },
+    query: {
+      create: {
+        type: QueryTypes.SQL,
+      },
+      read: {
+        type: QueryTypes.SQL,
+      },
+      update: {
+        type: QueryTypes.SQL,
+      },
+      delete: {
+        type: QueryTypes.SQL,
+      },
+    },
+  }
+
+  const UNSUPPORTED_TYPES = ["BLOB", "CLOB", "NCLOB"]
+
+  /**
+   * Raw query response
+   */
+  interface ColumnsResponse {
+    TABLE_NAME: string
+    COLUMN_NAME: string
+    DATA_TYPE: string
+    DATA_DEFAULT: string | null
+    COLUMN_ID: number
+    CONSTRAINT_NAME: string | null
+    CONSTRAINT_TYPE: string | null
+    R_CONSTRAINT_NAME: string | null
+    SEARCH_CONDITION: string | null
+  }
+
+  /**
+   * An oracle constraint
+   */
+  interface OracleConstraint {
+    name: string
+    type: string
+    relatedConstraintName: string | null
+    searchCondition: string | null
+  }
+
+  /**
+   * An oracle column and it's related constraints
+   */
+  interface OracleColumn {
+    name: string
+    type: string
+    default: string | null
+    id: number
+    constraints: { [key: string]: OracleConstraint }
+  }
+
+  /**
+   * An oracle table and it's related columns
+   */
+  interface OracleTable {
+    name: string
+    columns: { [key: string]: OracleColumn }
+  }
+
+  const OracleContraintTypes = {
+    PRIMARY: "P",
+    NOT_NULL_OR_CHECK: "C",
+    FOREIGN_KEY: "R",
+    UNIQUE: "U",
+  }
+
+  class OracleIntegration extends Sql implements DatasourcePlus {
+    private readonly config: OracleConfig
+
+    public tables: Record<string, Table> = {}
+    public schemaErrors: Record<string, string> = {}
+
+    private readonly COLUMNS_SQL = `
+      SELECT
+        tabs.table_name,
+        cols.column_name,
+        cols.data_type,
+        cols.data_default,
+        cols.column_id,
+        cons.constraint_name,
+        cons.constraint_type,
+        cons.r_constraint_name,
+        cons.search_condition
+      FROM
+        user_tables tabs
+      JOIN 
+        user_tab_columns cols
+        ON tabs.table_name = cols.table_name 
+      LEFT JOIN 
+        user_cons_columns col_cons
+        ON cols.column_name = col_cons.column_name
+        AND cols.table_name = col_cons.table_name
+      LEFT JOIN 
+        user_constraints cons
+        ON col_cons.constraint_name = cons.constraint_name
+        AND cons.table_name = cols.table_name
+      WHERE
+        (cons.status = 'ENABLED'
+          OR cons.status IS NULL)
+    `
+    constructor(config: OracleConfig) {
+      super(SqlClients.ORACLE)
+      this.config = config
+    }
+
+    /**
+     * Map the flat tabular columns and constraints data into a nested object
+     */
+    private mapColumns(result: Result<ColumnsResponse>): {
+      [key: string]: OracleTable
+    } {
+      const oracleTables: { [key: string]: OracleTable } = {}
+
+      if (result.rows) {
+        result.rows.forEach(row => {
+          const tableName = row.TABLE_NAME
+          const columnName = row.COLUMN_NAME
+          const dataType = row.DATA_TYPE
+          const dataDefault = row.DATA_DEFAULT
+          const columnId = row.COLUMN_ID
+          const constraintName = row.CONSTRAINT_NAME
+          const constraintType = row.CONSTRAINT_TYPE
+          const relatedConstraintName = row.R_CONSTRAINT_NAME
+          const searchCondition = row.SEARCH_CONDITION
+
+          let table = oracleTables[tableName]
+          if (!table) {
+            table = {
+              name: tableName,
+              columns: {},
+            }
+            oracleTables[tableName] = table
+          }
+
+          let column = table.columns[columnName]
+          if (!column) {
+            column = {
+              name: columnName,
+              type: dataType,
+              default: dataDefault,
+              id: columnId,
+              constraints: {},
+            }
+            table.columns[columnName] = column
+          }
+
+          if (constraintName && constraintType) {
+            let constraint = column.constraints[constraintName]
+            if (!constraint) {
+              constraint = {
+                name: constraintName,
+                type: constraintType,
+                relatedConstraintName: relatedConstraintName,
+                searchCondition: searchCondition,
+              }
+            }
+            column.constraints[constraintName] = constraint
+          }
+        })
+      }
+
+      return oracleTables
+    }
+
+    private isSupportedColumn(column: OracleColumn) {
+      if (UNSUPPORTED_TYPES.includes(column.type)) {
+        return false
+      }
+
+      return true
+    }
+
+    private isAutoColumn(column: OracleColumn) {
+      if (column.default && column.default.toLowerCase().includes("nextval")) {
+        return true
+      }
+
+      return false
+    }
+
+    /**
+     * No native boolean in oracle. Best we can do is to check if a manual 1 or 0 number constraint has been set up
+     * This matches the default behaviour for generating DDL used in knex.
+     */
+    private isBooleanType(column: OracleColumn): boolean {
+      if (
+        column.type.toLowerCase() === "number" &&
+        Object.values(column.constraints).filter(c => {
+          if (
+            c.type === OracleContraintTypes.NOT_NULL_OR_CHECK &&
+            c.searchCondition
+          ) {
+            const condition = c.searchCondition
+              .replace(/\s/g, "") // remove spaces
+              .replace(/[']+/g, "") // remove quotes
+            if (
+              condition.includes("in(0,1)") ||
+              condition.includes("in(1,0)")
+            ) {
+              return true
+            }
+          }
+          return false
+        }).length > 0
+      ) {
+        return true
+      }
+
+      return false
+    }
+
+    private internalConvertType(column: OracleColumn): string {
+      if (this.isBooleanType(column)) {
+        return FieldTypes.BOOLEAN
+      }
+
+      return convertSqlType(column.type)
+    }
+
+    /**
+     * Fetches the tables from the oracle table and assigns them to the datasource.
+     * @param {*} datasourceId - datasourceId to fetch
+     * @param entities - the tables that are to be built
+     */
+    async buildSchema(datasourceId: string, entities: Record<string, Table>) {
+      const columnsResponse = await this.internalQuery<ColumnsResponse>({
+        sql: this.COLUMNS_SQL,
+      })
+      const oracleTables = this.mapColumns(columnsResponse)
+
+      const tables: { [key: string]: Table } = {}
+
+      // iterate each table
+      Object.values(oracleTables).forEach(oracleTable => {
+        let table = tables[oracleTable.name]
+        if (!table) {
+          table = {
+            _id: buildExternalTableId(datasourceId, oracleTable.name),
+            primary: [],
+            name: oracleTable.name,
+            schema: {},
+          }
+          tables[oracleTable.name] = table
+        }
+
+        // iterate each column on the table
+        Object.values(oracleTable.columns)
+          // remove columns that we can't read / save
+          .filter(oracleColumn => this.isSupportedColumn(oracleColumn))
+          // match the order of the columns in the db
+          .sort((c1, c2) => c1.id - c2.id)
+          .forEach(oracleColumn => {
+            const columnName = oracleColumn.name
+            let fieldSchema = table.schema[columnName]
+            if (!fieldSchema) {
+              fieldSchema = {
+                autocolumn: this.isAutoColumn(oracleColumn),
+                name: columnName,
+                type: this.internalConvertType(oracleColumn),
+              }
+              table.schema[columnName] = fieldSchema
+            }
+
+            // iterate each constraint on the column
+            Object.values(oracleColumn.constraints).forEach(
+              oracleConstraint => {
+                if (oracleConstraint.type === OracleContraintTypes.PRIMARY) {
+                  table.primary!.push(columnName)
+                }
+              }
+            )
+          })
+      })
+
+      const final = finaliseExternalTables(tables, entities)
+      this.tables = final.tables
+      this.schemaErrors = final.errors
+    }
+
+    /**
+     * Knex default returning behaviour does not work with oracle
+     * Manually add the behaviour for the return column
+     */
+    private addReturning(
+      query: SqlQuery,
+      bindings: BindParameters,
+      returnColumn: string
+    ) {
+      if (bindings instanceof Array) {
+        bindings.push({ dir: oracledb.BIND_OUT })
+        query.sql =
+          query.sql + ` returning \"${returnColumn}\" into :${bindings.length}`
+      }
+    }
+
+    private async internalQuery<T>(
+      query: SqlQuery,
+      returnColum?: string,
+      operation?: string
+    ): Promise<Result<T>> {
+      let connection
+      try {
+        connection = await this.getConnection()
+
+        const options: ExecuteOptions = { autoCommit: true }
+        const bindings: BindParameters = query.bindings || []
+
+        if (
+          returnColum &&
+          (operation === Operation.CREATE || operation === Operation.UPDATE)
+        ) {
+          this.addReturning(query, bindings, returnColum)
+        }
+
+        const result: Result<T> = await connection.execute<T>(
+          query.sql,
+          bindings,
+          options
+        )
+
+        return result
+      } finally {
+        if (connection) {
+          try {
+            await connection.close()
+          } catch (err) {
+            console.error(err)
+          }
+        }
+      }
+    }
+
+    private getConnection = async (): Promise<Connection> => {
+      //connectString : "(DESCRIPTION =(ADDRESS = (PROTOCOL = TCP)(HOST = localhost)(PORT = 1521))(CONNECT_DATA =(SID= ORCL)))"
+      const connectString = `${this.config.host}:${this.config.port || 1521}/${
+        this.config.database
+      }`
+      const attributes: ConnectionAttributes = {
+        user: this.config.user,
+        password: this.config.user,
+        connectString,
+      }
+      return oracledb.getConnection(attributes)
+    }
+
+    async create(query: SqlQuery | string): Promise<any[]> {
+      const response = await this.internalQuery<any>(getSqlQuery(query))
+      return response.rows && response.rows.length
+        ? response.rows
+        : [{ created: true }]
+    }
+
+    async read(query: SqlQuery | string): Promise<any[]> {
+      const response = await this.internalQuery<any>(getSqlQuery(query))
+      return response.rows ? response.rows : []
+    }
+
+    async update(query: SqlQuery | string): Promise<any[]> {
+      const response = await this.internalQuery(getSqlQuery(query))
+      return response.rows && response.rows.length
+        ? response.rows
+        : [{ updated: true }]
+    }
+
+    async delete(query: SqlQuery | string): Promise<any[]> {
+      const response = await this.internalQuery(getSqlQuery(query))
+      return response.rows && response.rows.length
+        ? response.rows
+        : [{ deleted: true }]
+    }
+
+    async query(json: QueryJson) {
+      const primaryKeys = json.meta!.table!.primary
+      const primaryKey = primaryKeys ? primaryKeys[0] : undefined
+      const queryFn = (query: any, operation: string) =>
+        this.internalQuery(query, primaryKey, operation)
+      const processFn = (response: any) => (response.rows ? response.rows : [])
+      const output = await this.queryWithReturning(json, queryFn, processFn)
+      return output
+    }
+  }
+
+  module.exports = {
+    schema: SCHEMA,
+    integration: OracleIntegration,
+  }
+}

package/src/integrations/postgres.ts

@@ -6,17 +6,18 @@ import {
   SqlQuery,
 } from "../definitions/datasource"
 import { Table } from "../definitions/common"
-import { getSqlQuery } from "./utils"
+import {
+  getSqlQuery,
+  buildExternalTableId,
+  convertSqlType,
+  finaliseExternalTables,
+  SqlClients,
+} from "./utils"
+import { DatasourcePlus } from "./base/datasourcePlus"
 
 module PostgresModule {
   const { Pool } = require("pg")
   const Sql = require("./base/sql")
-  const { FieldTypes } = require("../constants")
-  const {
-    buildExternalTableId,
-    convertType,
-    copyExistingPropsOver,
-  } = require("./utils")
   const { escapeDangerousCharacters } = require("../utilities")
 
   const JSON_REGEX = /'{.*}'::json/s
@@ -27,6 +28,7 @@ module PostgresModule {
     database: string
     user: string
     password: string
+    schema: string
     ssl?: boolean
     ca?: string
     rejectUnauthorized?: boolean
@@ -64,6 +66,11 @@ module PostgresModule {
         default: "root",
         required: true,
       },
+      schema: {
+        type: DatasourceFieldTypes.STRING,
+        default: "public",
+        required: true,
+      },
       ssl: {
         type: DatasourceFieldTypes.BOOLEAN,
         default: false,
@@ -96,22 +103,6 @@ module PostgresModule {
     },
   }
 
-  const TYPE_MAP = {
-    text: FieldTypes.LONGFORM,
-    varchar: FieldTypes.STRING,
-    integer: FieldTypes.NUMBER,
-    bigint: FieldTypes.NUMBER,
-    decimal: FieldTypes.NUMBER,
-    smallint: FieldTypes.NUMBER,
-    real: FieldTypes.NUMBER,
-    "double precision": FieldTypes.NUMBER,
-    timestamp: FieldTypes.DATETIME,
-    time: FieldTypes.DATETIME,
-    boolean: FieldTypes.BOOLEAN,
-    json: FieldTypes.JSON,
-    date: FieldTypes.DATETIME,
-  }
-
   async function internalQuery(client: any, query: SqlQuery) {
     // need to handle a specific issue with json data types in postgres,
     // new lines inside the JSON data will break it
@@ -132,13 +123,14 @@ module PostgresModule {
     }
   }
 
-  class PostgresIntegration extends Sql {
+  class PostgresIntegration extends Sql implements DatasourcePlus {
     static pool: any
     private readonly client: any
     private readonly config: PostgresConfig
+    public tables: Record<string, Table> = {}
+    public schemaErrors: Record<string, string> = {}
 
-    COLUMNS_SQL =
-      "select * from information_schema.columns where not table_schema = 'information_schema' and not table_schema = 'pg_catalog'"
+    COLUMNS_SQL!: string
 
     PRIMARY_KEYS_SQL = `
     select tc.table_schema, tc.table_name, kc.column_name as primary_key 
@@ -151,7 +143,7 @@ module PostgresModule {
     `
 
     constructor(config: PostgresConfig) {
-      super("pg")
+      super(SqlClients.POSTGRES)
       this.config = config
 
       let newConfig = {
@@ -168,6 +160,17 @@ module PostgresModule {
       }
 
       this.client = this.pool
+      this.setSchema()
+    }
+
+    setSchema() {
+      if (!this.config.schema) {
+        this.config.schema = "public"
+      }
+      this.client.on("connect", (client: any) => {
+        client.query(`SET search_path TO ${this.config.schema}`)
+      })
+      this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = '${this.config.schema}'`
     }
 
     /**
@@ -207,13 +210,13 @@ module PostgresModule {
         if (!tables[tableName] || !tables[tableName].schema) {
           tables[tableName] = {
             _id: buildExternalTableId(datasourceId, tableName),
-            primary: tableKeys[tableName] || ["id"],
+            primary: tableKeys[tableName] || [],
             name: tableName,
             schema: {},
           }
         }
 
-        const type: string = convertType(column.data_type, TYPE_MAP)
+        const type: string = convertSqlType(column.data_type)
         const identity = !!(
           column.identity_generation ||
           column.identity_start ||
@@ -232,10 +235,9 @@ module PostgresModule {
         }
       }
 
-      for (let tableName of Object.keys(tables)) {
-        copyExistingPropsOver(tableName, tables, entities)
-      }
-      this.tables = tables
+      const final = finaliseExternalTables(tables, entities)
+      this.tables = final.tables
+      this.schemaErrors = final.errors
     }
 
     async create(query: SqlQuery | string) {
@@ -261,8 +263,16 @@ module PostgresModule {
     async query(json: QueryJson) {
       const operation = this._operation(json).toLowerCase()
       const input = this._query(json)
-      const response = await internalQuery(this.client, input)
-      return response.rows.length ? response.rows : [{ [operation]: true }]
+      if (Array.isArray(input)) {
+        const responses = []
+        for (let query of input) {
+          responses.push(await internalQuery(this.client, query))
+        }
+        return responses
+      } else {
+        const response = await internalQuery(this.client, input)
+        return response.rows.length ? response.rows : [{ [operation]: true }]
+      }
     }
   }
 

package/src/integrations/rest.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module RestModule {
   const fetch = require("node-fetch")
@@ -131,7 +132,7 @@ module RestModule {
     },
   }
 
-  class RestIntegration {
+  class RestIntegration implements IntegrationBase {
     private config: RestConfig
     private headers: {
       [key: string]: string

package/src/integrations/s3.ts

@@ -1,4 +1,5 @@
 import { Integration, QueryTypes } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module S3Module {
   const AWS = require("aws-sdk")
@@ -42,7 +43,7 @@ module S3Module {
     },
   }
 
-  class S3Integration {
+  class S3Integration implements IntegrationBase {
     private readonly config: S3Config
     private client: any
     private connectionPromise: Promise<any>

package/src/integrations/tests/microsoftSqlServer.spec.js

@@ -9,32 +9,39 @@ class TestConfiguration {
 }
 
 describe("MS SQL Server Integration", () => {
-  let config 
+  let config
 
-  beforeEach(() => {
+  beforeEach(async () => {
     config = new TestConfiguration()
   })
 
-  it("calls the create method with the correct params", async () => {
-    const sql = "insert into users (name, age) values ('Joe', 123);"
-    const response = await config.integration.create({ 
-      sql
+  describe("check sql used", () => {
+    beforeEach(async () => {
+      await config.integration.connect()
+    })
+
+    it("calls the create method with the correct params", async () => {
+      const sql = "insert into users (name, age) values ('Joe', 123);"
+      const response = await config.integration.create({
+        sql
+      })
+      expect(config.integration.client.request).toHaveBeenCalledWith()
+      expect(response[0]).toEqual(sql)
     })
-    expect(config.integration.client.query).toHaveBeenCalledWith(sql, {})
-  })
 
-  it("calls the read method with the correct params", async () => {
-    const sql = "select * from users;"
-    const response = await config.integration.read({ 
-      sql
+    it("calls the read method with the correct params", async () => {
+      const sql = "select * from users;"
+      const response = await config.integration.read({
+        sql
+      })
+      expect(config.integration.client.request).toHaveBeenCalledWith()
+      expect(response[0]).toEqual(sql)
     })
-    expect(config.integration.client.query).toHaveBeenCalledWith(sql, {})
   })
 
   describe("no rows returned", () => {
     beforeEach(async () => {
       await config.integration.connect()
-      config.integration.client.query.mockImplementation(() => ({ rows: [] }))
     })
 
     it("returns the correct response when the create response has no rows", async () => {
@@ -42,7 +49,7 @@ describe("MS SQL Server Integration", () => {
       const response = await config.integration.create({ 
         sql
       })
-      expect(response).toEqual([{ created: true }])
+      expect(response[0]).toEqual(sql)
     })
   })
 })