@budibase/server 0.9.185-alpha.14 → 0.9.185-alpha.18

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.185-alpha.14",
+  "version": "0.9.185-alpha.18",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -68,9 +68,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.185-alpha.14",
-    "@budibase/client": "^0.9.185-alpha.14",
-    "@budibase/string-templates": "^0.9.185-alpha.14",
+    "@budibase/auth": "^0.9.185-alpha.18",
+    "@budibase/client": "^0.9.185-alpha.18",
+    "@budibase/string-templates": "^0.9.185-alpha.18",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -135,6 +135,7 @@
     "@types/koa": "^2.13.3",
     "@types/koa-router": "^7.4.2",
     "@types/node": "^15.12.4",
+    "@types/oracledb": "^5.2.1",
     "@typescript-eslint/parser": "4.28.0",
     "babel-jest": "^27.0.2",
     "copyfiles": "^2.4.1",
@@ -150,5 +151,8 @@
     "typescript": "^4.3.5",
     "update-dotenv": "^1.1.1"
   },
-  "gitHead": "4e76951a03d3ea6793c1820d6c5590a36de6496e"
+  "optionalDependencies": {
+    "oracledb": "^5.3.0"
+  },
+  "gitHead": "aefc0a46953807c460bfb6ab11c5e541ef17a741"
 }

package/scripts/integrations/oracle/docker-compose.yml

@@ -0,0 +1,19 @@
+# For more information see:
+# https://container-registry.oracle.com/
+# - Database > Express
+version: "3.8"
+services:
+  db:
+    container_name: oracle-xe
+    platform: linux/x86_64
+    image: container-registry.oracle.com/database/express:18.4.0-xe
+    environment:
+      ORACLE_PWD: oracle
+    ports:
+      - 1521:1521
+      - 5500:5500
+    volumes:
+      - oracle_data:/opt/oracle/oradata
+
+volumes:
+ oracle_data:

package/scripts/integrations/oracle/instantclient/linux/x86-64/install.sh

@@ -0,0 +1,23 @@
+#!/bin/bash 
+
+# Must be root to continue
+if [[ $(id -u) -ne 0 ]] ; then echo "Please run as root" ; exit 1 ; fi
+
+# Allow for re-runs
+rm -rf /opt/oracle
+
+echo "Installing oracle instant client"
+
+# copy and unzip package
+mkdir -p /opt/oracle
+cp scripts/integrations/oracle/instantclient/linux/x86-64/basiclite-21.4.zip /opt/oracle
+cd /opt/oracle
+unzip -qq basiclite-21.4.zip -d .
+rm *.zip
+mv instantclient* instantclient
+
+# update runtime link path
+sh -c "echo /opt/oracle/instantclient > /etc/ld.so.conf.d/oracle-instantclient.conf"
+ldconfig /etc/ld.so.conf.d
+
+echo "Installation complete"

package/scripts/integrations/oracle/oracle.md

@@ -0,0 +1,90 @@
+# Installation
+
+## Database
+
+**Important**
+- Oracle database is supported only on **x86-64 architecture**
+- Oracle database is **not supported on Mac ARM architecture** (either via docker or linux virtualization)
+
+To install oracle express edition simply run `docker-compose up`
+
+- A single instance pluggable database (PDB) will be created named `xepdb`
+- The default password is configured in the compose file as `oracle`
+  - The `system`, `sys` and `pdbadmin` users all share this password
+
+## Instant Client 
+
+Before oracle can be connected to from nodejs, the oracle client must be installed.
+For more information see https://www.oracle.com/database/technologies/instant-client/downloads.html
+
+**Important**
+- Oracle client is supported only on **x86-64 architecture**
+- Oracle client is **not supported on Mac ARM architecture** 
+
+### Linux
+Run the provided install script for linux from the `server` root path:
+
+```bash
+sudo /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh
+```
+For more information see: https://www.oracle.com/database/technologies/instant-client/linux-x86-64-downloads.html#ic_x64_inst
+
+### Mac 
+**This has not yet been tested**
+
+See: https://www.oracle.com/database/technologies/instant-client/macos-intel-x86-downloads.html#ic_osx_inst
+
+# Management
+To connect to oracle sql command line:
+
+```bash
+docker exec -it oracle-xe sqlplus -l system/oracle@localhost/xepdb1
+```
+
+To create a new schema (where a user is the same as a schema in oracle) named `sales`:
+
+```sql
+define USERNAME = sales
+
+create user &USERNAME;
+
+alter user &USERNAME
+    default tablespace users
+    temporary tablespace temp
+    quota unlimited on users;
+
+grant create session,
+    create view,
+    create sequence,
+    create procedure,
+    create table,
+    create trigger,
+    create type,
+    create materialized view
+    to &USERNAME;
+```
+
+To set the password for the sales schema use: 
+
+```sql
+define USERNAME = sales
+define PASSWORD = sales
+
+alter user &USERNAME identified by &PASSWORD;
+```
+
+As before the database schema can now be connected to using:
+```bash
+docker exec -it oracle-xe sqlplus -l sales/sales@localhost:1521/xepdb1
+```
+
+## HR Schema
+
+The `HR` schema is populated with dummy data by default in oracle for testing purposes. 
+To connect to the HR schema first update the user password and unlock the account by performing
+```sql
+ALTER USER hr ACCOUNT UNLOCK;
+ALTER USER hr IDENTIFIED BY hr
+```
+You should now be able to connect to the hr schema using the credentials hr/hr
+

package/scripts/integrations/postgres/docker-compose.yml

@@ -11,7 +11,7 @@ services:
     ports:
       - "5432:5432"
     volumes:
-      #- pg_data:/var/lib/postgresql/data/
+      - pg_data:/var/lib/postgresql/data/
       - ./init.sql:/docker-entrypoint-initdb.d/init.sql
 
   pgadmin:
@@ -24,5 +24,5 @@ services:
     ports:
       - "5050:80"
 
-#volumes:
-#  pg_data:
+volumes:
+ pg_data:

package/src/definitions/datasource.ts

@@ -46,6 +46,7 @@ export enum SourceNames {
   MYSQL = "MYSQL",
   ARANGODB = "ARANGODB",
   REST = "REST",
+  ORACLE = "ORACLE",
 }
 
 export enum IncludeRelationships {

package/src/integrations/base/IntegrationBase.ts

@@ -1,6 +1,6 @@
 export interface IntegrationBase {
-  create?(query: any): Promise<[any]>
-  read?(query: any): Promise<[any]>
-  update?(query: any): Promise<[any]>
-  delete?(query: any): Promise<[any]>
+  create?(query: any): Promise<any[]>
+  read?(query: any): Promise<any[]>
+  update?(query: any): Promise<any[]>
+  delete?(query: any): Promise<any[]>
 }

package/src/integrations/base/sql.ts

@@ -416,6 +416,8 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
         id = results?.[0].id
       } else if (sqlClient === SqlClients.MY_SQL) {
         id = results?.insertId
+      } else if (sqlClient === SqlClients.ORACLE) {
+        id = response.outBinds[0][0]
       }
       row = processFn(
         await this.getReturningRow(queryFn, this.checkLookupKeys(id, json))
@@ -428,4 +430,5 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
   }
 }
 
+export default SqlQueryBuilder
 module.exports = SqlQueryBuilder

package/src/integrations/index.ts

@@ -39,6 +39,13 @@ const INTEGRATIONS = {
   [SourceNames.REST]: rest.integration,
 }
 
+// optionally add oracle integration if the oracle binary can be installed
+if (!(process.arch === 'arm64' && process.platform === 'darwin')) {
+  const oracle = require("./oracle")
+  DEFINITIONS[SourceNames.ORACLE] =  oracle.schema
+  INTEGRATIONS[SourceNames.ORACLE] = oracle.integration
+}
+
 module.exports = {
   definitions: DEFINITIONS,
   integrations: INTEGRATIONS,

package/src/integrations/oracle.ts

@@ -0,0 +1,445 @@
+import {
+  Integration,
+  DatasourceFieldTypes,
+  QueryTypes,
+  SqlQuery,
+  QueryJson,
+  Operation,
+} from "../definitions/datasource"
+import {
+  finaliseExternalTables,
+  getSqlQuery,
+  buildExternalTableId,
+  convertSqlType,
+  SqlClients,
+} from "./utils"
+import oracledb, {
+  ExecuteOptions,
+  Result,
+  Connection,
+  ConnectionAttributes,
+  BindParameters,
+} from "oracledb"
+import Sql from "./base/sql"
+import { Table } from "../definitions/common"
+import { DatasourcePlus } from "./base/datasourcePlus"
+import { FieldTypes } from "../constants"
+
+module OracleModule {
+  oracledb.outFormat = oracledb.OUT_FORMAT_OBJECT
+
+  interface OracleConfig {
+    host: string
+    port: number
+    database: string
+    user: string
+    password: string
+  }
+
+  const SCHEMA: Integration = {
+    docs: "https://github.com/oracle/node-oracledb",
+    plus: true,
+    friendlyName: "Oracle",
+    description:
+      "Oracle Database is an object-relational database management system developed by Oracle Corporation",
+    datasource: {
+      host: {
+        type: DatasourceFieldTypes.STRING,
+        default: "localhost",
+        required: true,
+      },
+      port: {
+        type: DatasourceFieldTypes.NUMBER,
+        required: true,
+        default: 1521,
+      },
+      database: {
+        type: DatasourceFieldTypes.STRING,
+        required: true,
+      },
+      user: {
+        type: DatasourceFieldTypes.STRING,
+        required: true,
+      },
+      password: {
+        type: DatasourceFieldTypes.PASSWORD,
+        required: true,
+      },
+    },
+    query: {
+      create: {
+        type: QueryTypes.SQL,
+      },
+      read: {
+        type: QueryTypes.SQL,
+      },
+      update: {
+        type: QueryTypes.SQL,
+      },
+      delete: {
+        type: QueryTypes.SQL,
+      },
+    },
+  }
+
+  const UNSUPPORTED_TYPES = ["BLOB", "CLOB", "NCLOB"]
+
+  /**
+   * Raw query response
+   */
+  interface ColumnsResponse {
+    TABLE_NAME: string
+    COLUMN_NAME: string
+    DATA_TYPE: string
+    DATA_DEFAULT: string | null
+    COLUMN_ID: number
+    CONSTRAINT_NAME: string | null
+    CONSTRAINT_TYPE: string | null
+    R_CONSTRAINT_NAME: string | null
+    SEARCH_CONDITION: string | null
+  }
+
+  /**
+   * An oracle constraint
+   */
+  interface OracleConstraint {
+    name: string
+    type: string
+    relatedConstraintName: string | null
+    searchCondition: string | null
+  }
+
+  /**
+   * An oracle column and it's related constraints
+   */
+  interface OracleColumn {
+    name: string
+    type: string
+    default: string | null
+    id: number
+    constraints: { [key: string]: OracleConstraint }
+  }
+
+  /**
+   * An oracle table and it's related columns
+   */
+  interface OracleTable {
+    name: string
+    columns: { [key: string]: OracleColumn }
+  }
+
+  const OracleContraintTypes = {
+    PRIMARY: "P",
+    NOT_NULL_OR_CHECK: "C",
+    FOREIGN_KEY: "R",
+    UNIQUE: "U",
+  }
+
+  class OracleIntegration extends Sql implements DatasourcePlus {
+    private readonly config: OracleConfig
+
+    public tables: Record<string, Table> = {}
+    public schemaErrors: Record<string, string> = {}
+
+    private readonly COLUMNS_SQL = `
+      SELECT
+        tabs.table_name,
+        cols.column_name,
+        cols.data_type,
+        cols.data_default,
+        cols.column_id,
+        cons.constraint_name,
+        cons.constraint_type,
+        cons.r_constraint_name,
+        cons.search_condition
+      FROM
+        user_tables tabs
+      JOIN 
+        user_tab_columns cols
+        ON tabs.table_name = cols.table_name 
+      LEFT JOIN 
+        user_cons_columns col_cons
+        ON cols.column_name = col_cons.column_name
+        AND cols.table_name = col_cons.table_name
+      LEFT JOIN 
+        user_constraints cons
+        ON col_cons.constraint_name = cons.constraint_name
+        AND cons.table_name = cols.table_name
+      WHERE
+        (cons.status = 'ENABLED'
+          OR cons.status IS NULL)
+    `
+    constructor(config: OracleConfig) {
+      super(SqlClients.ORACLE)
+      this.config = config
+    }
+
+    /**
+     * Map the flat tabular columns and constraints data into a nested object
+     */
+    private mapColumns(result: Result<ColumnsResponse>): {
+      [key: string]: OracleTable
+    } {
+      const oracleTables: { [key: string]: OracleTable } = {}
+
+      if (result.rows) {
+        result.rows.forEach(row => {
+          const tableName = row.TABLE_NAME
+          const columnName = row.COLUMN_NAME
+          const dataType = row.DATA_TYPE
+          const dataDefault = row.DATA_DEFAULT
+          const columnId = row.COLUMN_ID
+          const constraintName = row.CONSTRAINT_NAME
+          const constraintType = row.CONSTRAINT_TYPE
+          const relatedConstraintName = row.R_CONSTRAINT_NAME
+          const searchCondition = row.SEARCH_CONDITION
+
+          let table = oracleTables[tableName]
+          if (!table) {
+            table = {
+              name: tableName,
+              columns: {},
+            }
+            oracleTables[tableName] = table
+          }
+
+          let column = table.columns[columnName]
+          if (!column) {
+            column = {
+              name: columnName,
+              type: dataType,
+              default: dataDefault,
+              id: columnId,
+              constraints: {},
+            }
+            table.columns[columnName] = column
+          }
+
+          if (constraintName && constraintType) {
+            let constraint = column.constraints[constraintName]
+            if (!constraint) {
+              constraint = {
+                name: constraintName,
+                type: constraintType,
+                relatedConstraintName: relatedConstraintName,
+                searchCondition: searchCondition,
+              }
+            }
+            column.constraints[constraintName] = constraint
+          }
+        })
+      }
+
+      return oracleTables
+    }
+
+    private isSupportedColumn(column: OracleColumn) {
+      if (UNSUPPORTED_TYPES.includes(column.type)) {
+        return false
+      }
+
+      return true
+    }
+
+    private isAutoColumn(column: OracleColumn) {
+      if (column.default && column.default.toLowerCase().includes("nextval")) {
+        return true
+      }
+
+      return false
+    }
+
+    /**
+     * No native boolean in oracle. Best we can do is to check if a manual 1 or 0 number constraint has been set up
+     * This matches the default behaviour for generating DDL used in knex.
+     */
+    private isBooleanType(column: OracleColumn): boolean {
+      if (
+        column.type.toLowerCase() === "number" &&
+        Object.values(column.constraints).filter(c => {
+          if (
+            c.type === OracleContraintTypes.NOT_NULL_OR_CHECK &&
+            c.searchCondition
+          ) {
+            const condition = c.searchCondition
+              .replace(/\s/g, "") // remove spaces
+              .replace(/[']+/g, "") // remove quotes
+            if (
+              condition.includes("in(0,1)") ||
+              condition.includes("in(1,0)")
+            ) {
+              return true
+            }
+          }
+          return false
+        }).length > 0
+      ) {
+        return true
+      }
+
+      return false
+    }
+
+    private internalConvertType(column: OracleColumn): string {
+      if (this.isBooleanType(column)) {
+        return FieldTypes.BOOLEAN
+      }
+
+      return convertSqlType(column.type)
+    }
+
+    /**
+     * Fetches the tables from the oracle table and assigns them to the datasource.
+     * @param {*} datasourceId - datasourceId to fetch
+     * @param entities - the tables that are to be built
+     */
+    async buildSchema(datasourceId: string, entities: Record<string, Table>) {
+      const columnsResponse = await this.internalQuery<ColumnsResponse>({
+        sql: this.COLUMNS_SQL,
+      })
+      const oracleTables = this.mapColumns(columnsResponse)
+
+      const tables: { [key: string]: Table } = {}
+
+      // iterate each table
+      Object.values(oracleTables).forEach(oracleTable => {
+        let table = tables[oracleTable.name]
+        if (!table) {
+          table = {
+            _id: buildExternalTableId(datasourceId, oracleTable.name),
+            primary: [],
+            name: oracleTable.name,
+            schema: {},
+          }
+          tables[oracleTable.name] = table
+        }
+
+        // iterate each column on the table
+        Object.values(oracleTable.columns)
+          // remove columns that we can't read / save
+          .filter(oracleColumn => this.isSupportedColumn(oracleColumn))
+          // match the order of the columns in the db
+          .sort((c1, c2) => c1.id - c2.id)
+          .forEach(oracleColumn => {
+            const columnName = oracleColumn.name
+            let fieldSchema = table.schema[columnName]
+            if (!fieldSchema) {
+              fieldSchema = {
+                autocolumn: this.isAutoColumn(oracleColumn),
+                name: columnName,
+                type: this.internalConvertType(oracleColumn),
+              }
+              table.schema[columnName] = fieldSchema
+            }
+
+            // iterate each constraint on the column
+            Object.values(oracleColumn.constraints).forEach(
+              oracleConstraint => {
+                if (oracleConstraint.type === OracleContraintTypes.PRIMARY) {
+                  table.primary!.push(columnName)
+                }
+              }
+            )
+          })
+      })
+
+      const final = finaliseExternalTables(tables, entities)
+      this.tables = final.tables
+      this.schemaErrors = final.errors
+    }
+
+    /**
+     * Knex default returning behaviour does not work with oracle
+     * Manually add the behaviour for the return column
+     */
+    private addReturning(query: SqlQuery, bindings: BindParameters, returnColumn: string) {
+      if (bindings instanceof Array) {
+        bindings.push({ dir: oracledb.BIND_OUT })
+        query.sql = query.sql + ` returning \"${returnColumn}\" into :${bindings.length}`
+      }
+    }
+
+    private async internalQuery<T>(query: SqlQuery, returnColum?: string, operation?: string): Promise<Result<T>> {
+      let connection
+      try {
+        connection = await this.getConnection()
+
+        const options: ExecuteOptions = { autoCommit: true }
+        const bindings: BindParameters = query.bindings || []
+
+        if (returnColum && (operation === Operation.CREATE || operation === Operation.UPDATE)) {
+          this.addReturning(query, bindings, returnColum)
+        }
+
+        const result: Result<T> = await connection.execute<T>(
+          query.sql,
+          bindings,
+          options
+        )
+
+        return result
+      } finally {
+        if (connection) {
+          try {
+            await connection.close()
+          } catch (err) {
+            console.error(err)
+          }
+        }
+      }
+    }
+
+    private getConnection = async (): Promise<Connection> => {
+      //connectString : "(DESCRIPTION =(ADDRESS = (PROTOCOL = TCP)(HOST = localhost)(PORT = 1521))(CONNECT_DATA =(SID= ORCL)))"
+      const connectString = `${this.config.host}:${this.config.port || 1521}/${
+        this.config.database
+      }`
+      const attributes: ConnectionAttributes = {
+        user: this.config.user,
+        password: this.config.user,
+        connectString,
+      }
+      return oracledb.getConnection(attributes)
+    }
+
+    async create(query: SqlQuery | string): Promise<any[]> {
+      const response = await this.internalQuery<any>(getSqlQuery(query))
+      return response.rows && response.rows.length
+        ? response.rows
+        : [{ created: true }]
+    }
+
+    async read(query: SqlQuery | string): Promise<any[]> {
+      const response = await this.internalQuery<any>(getSqlQuery(query))
+      return response.rows ? response.rows : []
+    }
+
+    async update(query: SqlQuery | string): Promise<any[]>  {
+      const response = await this.internalQuery(getSqlQuery(query))
+      return response.rows && response.rows.length
+        ? response.rows
+        : [{ updated: true }]
+    }
+
+    async delete(query: SqlQuery | string): Promise<any[]>  {
+      const response = await this.internalQuery(getSqlQuery(query))
+      return response.rows && response.rows.length
+        ? response.rows
+        : [{ deleted: true }]
+    }
+
+    async query(json: QueryJson) {
+      const primaryKeys = json.meta!.table!.primary
+      const primaryKey = primaryKeys ? primaryKeys[0] : undefined
+      const queryFn = (query: any, operation: string) => this.internalQuery(query, primaryKey, operation)
+      const processFn = (response: any) => response.rows ? response.rows : []
+      const output = await this.queryWithReturning(json, queryFn, processFn)
+      return output
+    }
+  }
+
+  module.exports = {
+    schema: SCHEMA,
+    integration: OracleIntegration,
+  }
+}