@budibase/server 0.9.185-alpha.13 → 0.9.185-alpha.17

package/src/integrations/oracle.ts

@@ -0,0 +1,445 @@
+import {
+  Integration,
+  DatasourceFieldTypes,
+  QueryTypes,
+  SqlQuery,
+  QueryJson,
+  Operation,
+} from "../definitions/datasource"
+import {
+  finaliseExternalTables,
+  getSqlQuery,
+  buildExternalTableId,
+  convertSqlType,
+  SqlClients,
+} from "./utils"
+import oracledb, {
+  ExecuteOptions,
+  Result,
+  Connection,
+  ConnectionAttributes,
+  BindParameters,
+} from "oracledb"
+import Sql from "./base/sql"
+import { Table } from "../definitions/common"
+import { DatasourcePlus } from "./base/datasourcePlus"
+import { FieldTypes } from "../constants"
+
+module OracleModule {
+  oracledb.outFormat = oracledb.OUT_FORMAT_OBJECT
+
+  interface OracleConfig {
+    host: string
+    port: number
+    database: string
+    user: string
+    password: string
+  }
+
+  const SCHEMA: Integration = {
+    docs: "https://github.com/oracle/node-oracledb",
+    plus: true,
+    friendlyName: "Oracle",
+    description:
+      "Oracle Database is an object-relational database management system developed by Oracle Corporation",
+    datasource: {
+      host: {
+        type: DatasourceFieldTypes.STRING,
+        default: "localhost",
+        required: true,
+      },
+      port: {
+        type: DatasourceFieldTypes.NUMBER,
+        required: true,
+        default: 1521,
+      },
+      database: {
+        type: DatasourceFieldTypes.STRING,
+        required: true,
+      },
+      user: {
+        type: DatasourceFieldTypes.STRING,
+        required: true,
+      },
+      password: {
+        type: DatasourceFieldTypes.PASSWORD,
+        required: true,
+      },
+    },
+    query: {
+      create: {
+        type: QueryTypes.SQL,
+      },
+      read: {
+        type: QueryTypes.SQL,
+      },
+      update: {
+        type: QueryTypes.SQL,
+      },
+      delete: {
+        type: QueryTypes.SQL,
+      },
+    },
+  }
+
+  const UNSUPPORTED_TYPES = ["BLOB", "CLOB", "NCLOB"]
+
+  /**
+   * Raw query response
+   */
+  interface ColumnsResponse {
+    TABLE_NAME: string
+    COLUMN_NAME: string
+    DATA_TYPE: string
+    DATA_DEFAULT: string | null
+    COLUMN_ID: number
+    CONSTRAINT_NAME: string | null
+    CONSTRAINT_TYPE: string | null
+    R_CONSTRAINT_NAME: string | null
+    SEARCH_CONDITION: string | null
+  }
+
+  /**
+   * An oracle constraint
+   */
+  interface OracleConstraint {
+    name: string
+    type: string
+    relatedConstraintName: string | null
+    searchCondition: string | null
+  }
+
+  /**
+   * An oracle column and it's related constraints
+   */
+  interface OracleColumn {
+    name: string
+    type: string
+    default: string | null
+    id: number
+    constraints: { [key: string]: OracleConstraint }
+  }
+
+  /**
+   * An oracle table and it's related columns
+   */
+  interface OracleTable {
+    name: string
+    columns: { [key: string]: OracleColumn }
+  }
+
+  const OracleContraintTypes = {
+    PRIMARY: "P",
+    NOT_NULL_OR_CHECK: "C",
+    FOREIGN_KEY: "R",
+    UNIQUE: "U",
+  }
+
+  class OracleIntegration extends Sql implements DatasourcePlus {
+    private readonly config: OracleConfig
+
+    public tables: Record<string, Table> = {}
+    public schemaErrors: Record<string, string> = {}
+
+    private readonly COLUMNS_SQL = `
+      SELECT
+        tabs.table_name,
+        cols.column_name,
+        cols.data_type,
+        cols.data_default,
+        cols.column_id,
+        cons.constraint_name,
+        cons.constraint_type,
+        cons.r_constraint_name,
+        cons.search_condition
+      FROM
+        user_tables tabs
+      JOIN 
+        user_tab_columns cols
+        ON tabs.table_name = cols.table_name 
+      LEFT JOIN 
+        user_cons_columns col_cons
+        ON cols.column_name = col_cons.column_name
+        AND cols.table_name = col_cons.table_name
+      LEFT JOIN 
+        user_constraints cons
+        ON col_cons.constraint_name = cons.constraint_name
+        AND cons.table_name = cols.table_name
+      WHERE
+        (cons.status = 'ENABLED'
+          OR cons.status IS NULL)
+    `
+    constructor(config: OracleConfig) {
+      super(SqlClients.ORACLE)
+      this.config = config
+    }
+
+    /**
+     * Map the flat tabular columns and constraints data into a nested object
+     */
+    private mapColumns(result: Result<ColumnsResponse>): {
+      [key: string]: OracleTable
+    } {
+      const oracleTables: { [key: string]: OracleTable } = {}
+
+      if (result.rows) {
+        result.rows.forEach(row => {
+          const tableName = row.TABLE_NAME
+          const columnName = row.COLUMN_NAME
+          const dataType = row.DATA_TYPE
+          const dataDefault = row.DATA_DEFAULT
+          const columnId = row.COLUMN_ID
+          const constraintName = row.CONSTRAINT_NAME
+          const constraintType = row.CONSTRAINT_TYPE
+          const relatedConstraintName = row.R_CONSTRAINT_NAME
+          const searchCondition = row.SEARCH_CONDITION
+
+          let table = oracleTables[tableName]
+          if (!table) {
+            table = {
+              name: tableName,
+              columns: {},
+            }
+            oracleTables[tableName] = table
+          }
+
+          let column = table.columns[columnName]
+          if (!column) {
+            column = {
+              name: columnName,
+              type: dataType,
+              default: dataDefault,
+              id: columnId,
+              constraints: {},
+            }
+            table.columns[columnName] = column
+          }
+
+          if (constraintName && constraintType) {
+            let constraint = column.constraints[constraintName]
+            if (!constraint) {
+              constraint = {
+                name: constraintName,
+                type: constraintType,
+                relatedConstraintName: relatedConstraintName,
+                searchCondition: searchCondition,
+              }
+            }
+            column.constraints[constraintName] = constraint
+          }
+        })
+      }
+
+      return oracleTables
+    }
+
+    private isSupportedColumn(column: OracleColumn) {
+      if (UNSUPPORTED_TYPES.includes(column.type)) {
+        return false
+      }
+
+      return true
+    }
+
+    private isAutoColumn(column: OracleColumn) {
+      if (column.default && column.default.toLowerCase().includes("nextval")) {
+        return true
+      }
+
+      return false
+    }
+
+    /**
+     * No native boolean in oracle. Best we can do is to check if a manual 1 or 0 number constraint has been set up
+     * This matches the default behaviour for generating DDL used in knex.
+     */
+    private isBooleanType(column: OracleColumn): boolean {
+      if (
+        column.type.toLowerCase() === "number" &&
+        Object.values(column.constraints).filter(c => {
+          if (
+            c.type === OracleContraintTypes.NOT_NULL_OR_CHECK &&
+            c.searchCondition
+          ) {
+            const condition = c.searchCondition
+              .replace(/\s/g, "") // remove spaces
+              .replace(/[']+/g, "") // remove quotes
+            if (
+              condition.includes("in(0,1)") ||
+              condition.includes("in(1,0)")
+            ) {
+              return true
+            }
+          }
+          return false
+        }).length > 0
+      ) {
+        return true
+      }
+
+      return false
+    }
+
+    private internalConvertType(column: OracleColumn): string {
+      if (this.isBooleanType(column)) {
+        return FieldTypes.BOOLEAN
+      }
+
+      return convertSqlType(column.type)
+    }
+
+    /**
+     * Fetches the tables from the oracle table and assigns them to the datasource.
+     * @param {*} datasourceId - datasourceId to fetch
+     * @param entities - the tables that are to be built
+     */
+    async buildSchema(datasourceId: string, entities: Record<string, Table>) {
+      const columnsResponse = await this.internalQuery<ColumnsResponse>({
+        sql: this.COLUMNS_SQL,
+      })
+      const oracleTables = this.mapColumns(columnsResponse)
+
+      const tables: { [key: string]: Table } = {}
+
+      // iterate each table
+      Object.values(oracleTables).forEach(oracleTable => {
+        let table = tables[oracleTable.name]
+        if (!table) {
+          table = {
+            _id: buildExternalTableId(datasourceId, oracleTable.name),
+            primary: [],
+            name: oracleTable.name,
+            schema: {},
+          }
+          tables[oracleTable.name] = table
+        }
+
+        // iterate each column on the table
+        Object.values(oracleTable.columns)
+          // remove columns that we can't read / save
+          .filter(oracleColumn => this.isSupportedColumn(oracleColumn))
+          // match the order of the columns in the db
+          .sort((c1, c2) => c1.id - c2.id)
+          .forEach(oracleColumn => {
+            const columnName = oracleColumn.name
+            let fieldSchema = table.schema[columnName]
+            if (!fieldSchema) {
+              fieldSchema = {
+                autocolumn: this.isAutoColumn(oracleColumn),
+                name: columnName,
+                type: this.internalConvertType(oracleColumn),
+              }
+              table.schema[columnName] = fieldSchema
+            }
+
+            // iterate each constraint on the column
+            Object.values(oracleColumn.constraints).forEach(
+              oracleConstraint => {
+                if (oracleConstraint.type === OracleContraintTypes.PRIMARY) {
+                  table.primary!.push(columnName)
+                }
+              }
+            )
+          })
+      })
+
+      const final = finaliseExternalTables(tables, entities)
+      this.tables = final.tables
+      this.schemaErrors = final.errors
+    }
+
+    /**
+     * Knex default returning behaviour does not work with oracle
+     * Manually add the behaviour for the return column
+     */
+    private addReturning(query: SqlQuery, bindings: BindParameters, returnColumn: string) {
+      if (bindings instanceof Array) {
+        bindings.push({ dir: oracledb.BIND_OUT })
+        query.sql = query.sql + ` returning \"${returnColumn}\" into :${bindings.length}`
+      }
+    }
+
+    private async internalQuery<T>(query: SqlQuery, returnColum?: string, operation?: string): Promise<Result<T>> {
+      let connection
+      try {
+        connection = await this.getConnection()
+
+        const options: ExecuteOptions = { autoCommit: true }
+        const bindings: BindParameters = query.bindings || []
+
+        if (returnColum && (operation === Operation.CREATE || operation === Operation.UPDATE)) {
+          this.addReturning(query, bindings, returnColum)
+        }
+
+        const result: Result<T> = await connection.execute<T>(
+          query.sql,
+          bindings,
+          options
+        )
+
+        return result
+      } finally {
+        if (connection) {
+          try {
+            await connection.close()
+          } catch (err) {
+            console.error(err)
+          }
+        }
+      }
+    }
+
+    private getConnection = async (): Promise<Connection> => {
+      //connectString : "(DESCRIPTION =(ADDRESS = (PROTOCOL = TCP)(HOST = localhost)(PORT = 1521))(CONNECT_DATA =(SID= ORCL)))"
+      const connectString = `${this.config.host}:${this.config.port || 1521}/${
+        this.config.database
+      }`
+      const attributes: ConnectionAttributes = {
+        user: this.config.user,
+        password: this.config.user,
+        connectString,
+      }
+      return oracledb.getConnection(attributes)
+    }
+
+    async create(query: SqlQuery | string): Promise<any[]> {
+      const response = await this.internalQuery<any>(getSqlQuery(query))
+      return response.rows && response.rows.length
+        ? response.rows
+        : [{ created: true }]
+    }
+
+    async read(query: SqlQuery | string): Promise<any[]> {
+      const response = await this.internalQuery<any>(getSqlQuery(query))
+      return response.rows ? response.rows : []
+    }
+
+    async update(query: SqlQuery | string): Promise<any[]>  {
+      const response = await this.internalQuery(getSqlQuery(query))
+      return response.rows && response.rows.length
+        ? response.rows
+        : [{ updated: true }]
+    }
+
+    async delete(query: SqlQuery | string): Promise<any[]>  {
+      const response = await this.internalQuery(getSqlQuery(query))
+      return response.rows && response.rows.length
+        ? response.rows
+        : [{ deleted: true }]
+    }
+
+    async query(json: QueryJson) {
+      const primaryKeys = json.meta!.table!.primary
+      const primaryKey = primaryKeys ? primaryKeys[0] : undefined
+      const queryFn = (query: any, operation: string) => this.internalQuery(query, primaryKey, operation)
+      const processFn = (response: any) => response.rows ? response.rows : []
+      const output = await this.queryWithReturning(json, queryFn, processFn)
+      return output
+    }
+  }
+
+  module.exports = {
+    schema: SCHEMA,
+    integration: OracleIntegration,
+  }
+}

package/src/integrations/tests/oracle.spec.js

@@ -0,0 +1,94 @@
+const oracledb = require("oracledb")
+const OracleIntegration = require("../oracle")
+jest.mock("oracledb")
+
+class TestConfiguration {
+  constructor(config = {}) {
+    this.integration = new OracleIntegration.integration(config) 
+  }
+}
+
+const options = { autoCommit: true }
+
+describe("Oracle Integration", () => {
+  let config 
+
+  beforeEach(() => {
+    jest.clearAllMocks()
+    config = new TestConfiguration()
+  })
+
+  afterEach(() => {
+    expect(oracledb.closeMock).toHaveBeenCalled()
+    expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
+  })
+
+  it("calls the create method with the correct params", async () => {
+    const sql = "insert into users (name, age) values ('Joe', 123);"
+    await config.integration.create({
+      sql
+    })
+    expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
+    expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+  })
+
+  it("calls the read method with the correct params", async () => {
+    const sql = "select * from users;"
+    await config.integration.read({
+      sql
+    })
+    expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
+    expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+  })
+
+  it("calls the update method with the correct params", async () => {
+    const sql = "update table users set name = 'test';"
+    const response = await config.integration.update({ 
+      sql
+    })
+    expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
+    expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+  })
+
+  it("calls the delete method with the correct params", async () => {
+    const sql = "delete from users where name = 'todelete';"
+    await config.integration.delete({
+      sql
+    })
+    expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
+    expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+  })
+
+  describe("no rows returned", () => {
+    beforeEach(() => {
+      oracledb.executeMock.mockImplementation(() => ({ rows: [] }))
+    })
+
+    it("returns the correct response when the create response has no rows", async () => {
+    const sql = "insert into users (name, age) values ('Joe', 123);"
+      const response = await config.integration.create({ 
+        sql
+      })
+      expect(response).toEqual([{ created: true }])
+      expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+    })
+
+    it("returns the correct response when the update response has no rows", async () => {
+      const sql = "update table users set name = 'test';"
+      const response = await config.integration.update({ 
+        sql
+      })
+      expect(response).toEqual([{ updated: true }])
+      expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+    })
+
+    it("returns the correct response when the delete response has no rows", async () => {
+      const sql = "delete from users where name = 'todelete';"
+      const response = await config.integration.delete({ 
+        sql
+      })
+      expect(response).toEqual([{ deleted: true }])
+      expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
+    })
+  })
+})

package/src/integrations/utils.ts

@@ -32,12 +32,17 @@ const SQL_TYPE_MAP = {
   fixed: FieldTypes.NUMBER,
   datetime: FieldTypes.DATETIME,
   tinyint: FieldTypes.BOOLEAN,
+  long: FieldTypes.LONGFORM,
+  number: FieldTypes.NUMBER,
+  binary_float: FieldTypes.NUMBER,
+  binary_double: FieldTypes.NUMBER,
 }
 
 export enum SqlClients {
   MS_SQL = "mssql",
   POSTGRES = "pg",
   MY_SQL = "mysql",
+  ORACLE = "oracledb"
 }
 
 export function isExternalTable(tableId: string) {
@@ -162,7 +167,7 @@ export function finaliseExternalTables(
   entities: { [key: string]: any }
 ) {
   const invalidColumns = Object.values(InvalidColumns)
-  const finalTables: { [key: string]: any } = {}
+  let finalTables: { [key: string]: any } = {}
   const errors: { [key: string]: string } = {}
   for (let [name, table] of Object.entries(tables)) {
     const schemaFields = Object.keys(table.schema)
@@ -177,5 +182,9 @@ export function finaliseExternalTables(
     // make sure all previous props have been added back
     finalTables[name] = copyExistingPropsOver(name, table, entities)
   }
+  // sort the tables by name
+  finalTables = Object.entries(finalTables)
+    .sort(([a], [b]) => a.localeCompare(b))
+    .reduce((r, [k, v]) => ({ ...r, [k]: v }), {})
   return { tables: finalTables, errors }
 }

package/src/threads/automation.js

@@ -1,3 +1,6 @@
+// when thread starts, make sure it is recorded
+const env = require("../environment")
+env.setInThread()
 const actions = require("../automations/actions")
 const automationUtils = require("../automations/automationUtils")
 const AutomationEmitter = require("../events/AutomationEmitter")
@@ -6,7 +9,6 @@ const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants
 const CouchDB = require("../db")
 const { DocumentTypes, isDevAppID } = require("../db/utils")
 const { doInTenant } = require("@budibase/auth/tenancy")
-const env = require("../environment")
 const usage = require("../utilities/usageQuota")
 const { definitions: triggerDefs } = require("../automations/triggerInfo")
 

package/src/threads/index.js

@@ -24,10 +24,16 @@ function typeToFile(type) {
 class Thread {
   constructor(type, opts = { timeoutMs: null, count: 1 }) {
     this.type = type
-    if (!env.isTest()) {
+    this.count = opts.count ? opts.count : 1
+    this.disableThreading =
+      env.isTest() ||
+      env.DISABLE_THREADING ||
+      this.count === 0 ||
+      env.isInThread()
+    if (!this.disableThreading) {
       const workerOpts = {
         autoStart: true,
-        maxConcurrentWorkers: opts.count ? opts.count : 1,
+        maxConcurrentWorkers: this.count,
       }
       if (opts.timeoutMs) {
         workerOpts.maxCallTime = opts.timeoutMs
@@ -40,7 +46,7 @@ class Thread {
     return new Promise((resolve, reject) => {
       let fncToCall
       // if in test then don't use threading
-      if (env.isTest()) {
+      if (this.disableThreading) {
         fncToCall = require(typeToFile(this.type))
       } else {
         fncToCall = this.workers

package/src/threads/query.js

@@ -1,3 +1,6 @@
+// when thread starts, make sure it is recorded
+const env = require("../environment")
+env.setInThread()
 const ScriptRunner = require("../utilities/scriptRunner")
 const { integrations } = require("../integrations")