@budibase/server 1.0.79 → 1.0.80-alpha.2

package/src/environment.js

@@ -45,6 +45,7 @@ module.exports = {
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS,
+  API_REQ_LIMIT_PER_SEC: process.env.API_REQ_LIMIT_PER_SEC,
   // environment
   NODE_ENV: process.env.NODE_ENV,
   JEST_WORKER_ID: process.env.JEST_WORKER_ID,

package/src/integrations/base/datasourcePlus.ts

@@ -5,5 +5,8 @@ export interface DatasourcePlus extends IntegrationBase {
   tables: Record<string, Table>
   schemaErrors: Record<string, string>
 
+  // if the datasource supports the use of bindings directly (to protect against SQL injection)
+  // this returns the format of the identifier
+  getBindingIdentifier(): string
   buildSchema(datasourceId: string, entities: Record<string, Table>): any
 }

package/src/integrations/base/sql.ts

@@ -249,6 +249,9 @@ class InternalBuilder {
   create(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, body } = json
     let query: KnexQuery = knex(endpoint.entityId)
+    if (endpoint.schema) {
+      query = query.withSchema(endpoint.schema)
+    }
     const parsedBody = parseBody(body)
     // make sure no null values in body for creation
     for (let [key, value] of Object.entries(parsedBody)) {
@@ -267,6 +270,9 @@ class InternalBuilder {
   bulkCreate(knex: Knex, json: QueryJson): KnexQuery {
     const { endpoint, body } = json
     let query: KnexQuery = knex(endpoint.entityId)
+    if (endpoint.schema) {
+      query = query.withSchema(endpoint.schema)
+    }
     if (!Array.isArray(body)) {
       return query
     }
@@ -275,7 +281,7 @@ class InternalBuilder {
   }
 
   read(knex: Knex, json: QueryJson, limit: number): KnexQuery {
-    let { endpoint, resource, filters, sort, paginate, relationships } = json
+    let { endpoint, resource, filters, paginate, relationships } = json
     const tableName = endpoint.entityId
     // select all if not specified
     if (!resource) {
@@ -302,6 +308,9 @@ class InternalBuilder {
     }
     // start building the query
     let query: KnexQuery = knex(tableName).limit(foundLimit)
+    if (endpoint.schema) {
+      query = query.withSchema(endpoint.schema)
+    }
     if (foundOffset) {
       query = query.offset(foundOffset)
     }
@@ -331,6 +340,9 @@ class InternalBuilder {
   update(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, body, filters } = json
     let query: KnexQuery = knex(endpoint.entityId)
+    if (endpoint.schema) {
+      query = query.withSchema(endpoint.schema)
+    }
     const parsedBody = parseBody(body)
     query = this.addFilters(query, filters, { tableName: endpoint.entityId })
     // mysql can't use returning
@@ -344,6 +356,9 @@ class InternalBuilder {
   delete(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, filters } = json
     let query: KnexQuery = knex(endpoint.entityId)
+    if (endpoint.schema) {
+      query = query.withSchema(endpoint.schema)
+    }
     query = this.addFilters(query, filters, { tableName: endpoint.entityId })
     // mysql can't use returning
     if (opts.disableReturning) {

package/src/integrations/base/sqlTable.ts

@@ -101,28 +101,28 @@ function generateSchema(
 }
 
 function buildCreateTable(
-  knex: Knex,
+  knex: SchemaBuilder,
   table: Table,
   tables: Record<string, Table>
 ): SchemaBuilder {
-  return knex.schema.createTable(table.name, schema => {
+  return knex.createTable(table.name, schema => {
     generateSchema(schema, table, tables)
   })
 }
 
 function buildUpdateTable(
-  knex: Knex,
+  knex: SchemaBuilder,
   table: Table,
   tables: Record<string, Table>,
   oldTable: Table
 ): SchemaBuilder {
-  return knex.schema.alterTable(table.name, schema => {
+  return knex.alterTable(table.name, schema => {
     generateSchema(schema, table, tables, oldTable)
   })
 }
 
-function buildDeleteTable(knex: Knex, table: Table): SchemaBuilder {
-  return knex.schema.dropTable(table.name)
+function buildDeleteTable(knex: SchemaBuilder, table: Table): SchemaBuilder {
+  return knex.dropTable(table.name)
 }
 
 class SqlTableQueryBuilder {
@@ -146,7 +146,11 @@ class SqlTableQueryBuilder {
   }
 
   _tableQuery(json: QueryJson): any {
-    const client = knex({ client: this.sqlClient })
+    let client = knex({ client: this.sqlClient }).schema
+    if (json?.endpoint?.schema) {
+      client = client.withSchema(json.endpoint.schema)
+    }
+
     let query
     if (!json.table || !json.meta || !json.meta.tables) {
       throw "Cannot execute without table being specified"

package/src/integrations/googlesheets.ts

@@ -6,11 +6,10 @@ import {
 } from "../definitions/datasource"
 import { OAuth2Client } from "google-auth-library"
 import { DatasourcePlus } from "./base/datasourcePlus"
-import { Row, Table, TableSchema } from "../definitions/common"
+import { Table, TableSchema } from "../definitions/common"
 import { buildExternalTableId } from "./utils"
 import { DataSourceOperation, FieldTypes } from "../constants"
 import { GoogleSpreadsheet } from "google-spreadsheet"
-import { table } from "console"
 
 module GoogleSheetsModule {
   const { getGlobalDB } = require("@budibase/backend-core/tenancy")
@@ -112,6 +111,10 @@ module GoogleSheetsModule {
       this.client = new GoogleSpreadsheet(spreadsheetId)
     }
 
+    getBindingIdentifier() {
+      return ""
+    }
+
     /**
      * Pull the spreadsheet ID out from a valid google sheets URL
      * @param spreadsheetId - the URL or standard spreadsheetId of the google sheet

package/src/integrations/microsoftSqlServer.ts

@@ -19,6 +19,7 @@ import { Table, TableSchema } from "../definitions/common"
 module MSSQLModule {
   const sqlServer = require("mssql")
   const Sql = require("./base/sql")
+  const DEFAULT_SCHEMA = "dbo"
 
   interface MSSQLConfig {
     user: string
@@ -26,9 +27,17 @@ module MSSQLModule {
     server: string
     port: number
     database: string
+    schema: string
     encrypt?: boolean
   }
 
+  interface TablesResponse {
+    TABLE_CATALOG: string
+    TABLE_SCHEMA: string
+    TABLE_NAME: string
+    TABLE_TYPE: string
+  }
+
   const SCHEMA: Integration = {
     docs: "https://github.com/tediousjs/node-mssql",
     plus: true,
@@ -58,6 +67,10 @@ module MSSQLModule {
         type: DatasourceFieldTypes.STRING,
         default: "root",
       },
+      schema: {
+        type: DatasourceFieldTypes.STRING,
+        default: DEFAULT_SCHEMA,
+      },
       encrypt: {
         type: DatasourceFieldTypes.BOOLEAN,
         default: true,
@@ -79,34 +92,9 @@ module MSSQLModule {
     },
   }
 
-  async function internalQuery(
-    client: any,
-    query: SqlQuery,
-    operation: string | undefined = undefined
-  ) {
-    const request = client.request()
-    try {
-      if (Array.isArray(query.bindings)) {
-        let count = 0
-        for (let binding of query.bindings) {
-          request.input(`p${count++}`, binding)
-        }
-      }
-      // this is a hack to get the inserted ID back,
-      //  no way to do this with Knex nicely
-      const sql =
-        operation === Operation.CREATE
-          ? `${query.sql}; SELECT SCOPE_IDENTITY() AS id;`
-          : query.sql
-      return await request.query(sql)
-    } catch (err) {
-      // @ts-ignore
-      throw new Error(err)
-    }
-  }
-
   class SqlServerIntegration extends Sql implements DatasourcePlus {
     private readonly config: MSSQLConfig
+    private index: number = 0
     static pool: any
     public tables: Record<string, Table> = {}
     public schemaErrors: Record<string, string> = {}
@@ -121,6 +109,62 @@ module MSSQLModule {
     TABLES_SQL =
       "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='BASE TABLE'"
 
+    constructor(config: MSSQLConfig) {
+      super(SqlClients.MS_SQL)
+      this.config = config
+      const clientCfg = {
+        ...this.config,
+        options: {
+          encrypt: this.config.encrypt,
+          enableArithAbort: true,
+        },
+      }
+      delete clientCfg.encrypt
+      if (!this.pool) {
+        this.pool = new sqlServer.ConnectionPool(clientCfg)
+      }
+    }
+
+    getBindingIdentifier(): string {
+      return `(@p${this.index++})`
+    }
+
+    async connect() {
+      try {
+        this.client = await this.pool.connect()
+      } catch (err) {
+        // @ts-ignore
+        throw new Error(err)
+      }
+    }
+
+    async internalQuery(
+      query: SqlQuery,
+      operation: string | undefined = undefined
+    ) {
+      const client = this.client
+      const request = client.request()
+      this.index = 0
+      try {
+        if (Array.isArray(query.bindings)) {
+          let count = 0
+          for (let binding of query.bindings) {
+            request.input(`p${count++}`, binding)
+          }
+        }
+        // this is a hack to get the inserted ID back,
+        //  no way to do this with Knex nicely
+        const sql =
+          operation === Operation.CREATE
+            ? `${query.sql}; SELECT SCOPE_IDENTITY() AS id;`
+            : query.sql
+        return await request.query(sql)
+      } catch (err) {
+        // @ts-ignore
+        throw new Error(err)
+      }
+    }
+
     getDefinitionSQL(tableName: string) {
       return `select *
               from INFORMATION_SCHEMA.COLUMNS
@@ -149,33 +193,8 @@ module MSSQLModule {
               WHERE TABLE_NAME='${tableName}'`
     }
 
-    constructor(config: MSSQLConfig) {
-      super(SqlClients.MS_SQL)
-      this.config = config
-      const clientCfg = {
-        ...this.config,
-        options: {
-          encrypt: this.config.encrypt,
-          enableArithAbort: true,
-        },
-      }
-      delete clientCfg.encrypt
-      if (!this.pool) {
-        this.pool = new sqlServer.ConnectionPool(clientCfg)
-      }
-    }
-
-    async connect() {
-      try {
-        this.client = await this.pool.connect()
-      } catch (err) {
-        // @ts-ignore
-        throw new Error(err)
-      }
-    }
-
     async runSQL(sql: string) {
-      return (await internalQuery(this.client, getSqlQuery(sql))).recordset
+      return (await this.internalQuery(getSqlQuery(sql))).recordset
     }
 
     /**
@@ -185,11 +204,14 @@ module MSSQLModule {
      */
     async buildSchema(datasourceId: string, entities: Record<string, Table>) {
       await this.connect()
-      let tableNames = await this.runSQL(this.TABLES_SQL)
-      if (tableNames == null || !Array.isArray(tableNames)) {
+      let tableInfo: TablesResponse[] = await this.runSQL(this.TABLES_SQL)
+      if (tableInfo == null || !Array.isArray(tableInfo)) {
         throw "Unable to get list of tables in database"
       }
-      tableNames = tableNames
+
+      const schema = this.config.schema || DEFAULT_SCHEMA
+      const tableNames = tableInfo
+        .filter((record: any) => record.TABLE_SCHEMA === schema)
         .map((record: any) => record.TABLE_NAME)
         .filter((name: string) => this.MASTER_TABLES.indexOf(name) === -1)
 
@@ -238,33 +260,36 @@ module MSSQLModule {
 
     async read(query: SqlQuery | string) {
       await this.connect()
-      const response = await internalQuery(this.client, getSqlQuery(query))
+      const response = await this.internalQuery(getSqlQuery(query))
       return response.recordset
     }
 
     async create(query: SqlQuery | string) {
       await this.connect()
-      const response = await internalQuery(this.client, getSqlQuery(query))
+      const response = await this.internalQuery(getSqlQuery(query))
       return response.recordset || [{ created: true }]
     }
 
     async update(query: SqlQuery | string) {
       await this.connect()
-      const response = await internalQuery(this.client, getSqlQuery(query))
+      const response = await this.internalQuery(getSqlQuery(query))
       return response.recordset || [{ updated: true }]
     }
 
     async delete(query: SqlQuery | string) {
       await this.connect()
-      const response = await internalQuery(this.client, getSqlQuery(query))
+      const response = await this.internalQuery(getSqlQuery(query))
       return response.recordset || [{ deleted: true }]
     }
 
     async query(json: QueryJson) {
+      const schema = this.config.schema
       await this.connect()
+      if (schema && schema !== DEFAULT_SCHEMA && json?.endpoint) {
+        json.endpoint.schema = schema
+      }
       const operation = this._operation(json)
-      const queryFn = (query: any, op: string) =>
-        internalQuery(this.client, query, op)
+      const queryFn = (query: any, op: string) => this.internalQuery(query, op)
       const processFn = (result: any) =>
         result.recordset ? result.recordset : [{ [operation]: true }]
       return this.queryWithReturning(json, queryFn, processFn)

package/src/integrations/mysql.ts

@@ -95,6 +95,10 @@ module MySQLModule {
       this.config = config
     }
 
+    getBindingIdentifier(): string {
+      return "?"
+    }
+
     async connect() {
       this.client = await mysql.createConnection(this.config)
     }

package/src/integrations/oracle.ts

@@ -1,24 +1,24 @@
 import {
-  Integration,
   DatasourceFieldTypes,
+  Integration,
+  Operation,
+  QueryJson,
   QueryTypes,
   SqlQuery,
-  QueryJson,
-  Operation,
 } from "../definitions/datasource"
 import {
-  finaliseExternalTables,
-  getSqlQuery,
   buildExternalTableId,
   convertSqlType,
+  finaliseExternalTables,
+  getSqlQuery,
   SqlClients,
 } from "./utils"
 import oracledb, {
-  ExecuteOptions,
-  Result,
+  BindParameters,
   Connection,
   ConnectionAttributes,
-  BindParameters,
+  ExecuteOptions,
+  Result,
 } from "oracledb"
 import Sql from "./base/sql"
 import { Table } from "../definitions/common"
@@ -137,6 +137,7 @@ module OracleModule {
 
   class OracleIntegration extends Sql implements DatasourcePlus {
     private readonly config: OracleConfig
+    private index: number = 1
 
     public tables: Record<string, Table> = {}
     public schemaErrors: Record<string, string> = {}
@@ -174,6 +175,10 @@ module OracleModule {
       this.config = config
     }
 
+    getBindingIdentifier(): string {
+      return `:${this.index++}`
+    }
+
     /**
      * Map the flat tabular columns and constraints data into a nested object
      */
@@ -233,20 +238,14 @@ module OracleModule {
       return oracleTables
     }
 
-    private isSupportedColumn(column: OracleColumn) {
-      if (UNSUPPORTED_TYPES.includes(column.type)) {
-        return false
-      }
-
-      return true
+    private static isSupportedColumn(column: OracleColumn) {
+      return !UNSUPPORTED_TYPES.includes(column.type)
     }
 
-    private isAutoColumn(column: OracleColumn) {
-      if (column.default && column.default.toLowerCase().includes("nextval")) {
-        return true
-      }
-
-      return false
+    private static isAutoColumn(column: OracleColumn) {
+      return !!(
+        column.default && column.default.toLowerCase().includes("nextval")
+      )
     }
 
     /**
@@ -254,7 +253,7 @@ module OracleModule {
      * This matches the default behaviour for generating DDL used in knex.
      */
     private isBooleanType(column: OracleColumn): boolean {
-      if (
+      return (
         column.type.toLowerCase() === "number" &&
         Object.values(column.constraints).filter(c => {
           if (
@@ -273,11 +272,7 @@ module OracleModule {
           }
           return false
         }).length > 0
-      ) {
-        return true
-      }
-
-      return false
+      )
     }
 
     private internalConvertType(column: OracleColumn): string {
@@ -317,7 +312,9 @@ module OracleModule {
         // iterate each column on the table
         Object.values(oracleTable.columns)
           // remove columns that we can't read / save
-          .filter(oracleColumn => this.isSupportedColumn(oracleColumn))
+          .filter(oracleColumn =>
+            OracleIntegration.isSupportedColumn(oracleColumn)
+          )
           // match the order of the columns in the db
           .sort((c1, c2) => c1.id - c2.id)
           .forEach(oracleColumn => {
@@ -325,7 +322,7 @@ module OracleModule {
             let fieldSchema = table.schema[columnName]
             if (!fieldSchema) {
               fieldSchema = {
-                autocolumn: this.isAutoColumn(oracleColumn),
+                autocolumn: OracleIntegration.isAutoColumn(oracleColumn),
                 name: columnName,
                 type: this.internalConvertType(oracleColumn),
               }
@@ -351,18 +348,13 @@ module OracleModule {
     private async internalQuery<T>(query: SqlQuery): Promise<Result<T>> {
       let connection
       try {
+        this.index = 1
         connection = await this.getConnection()
 
         const options: ExecuteOptions = { autoCommit: true }
         const bindings: BindParameters = query.bindings || []
 
-        const result: Result<T> = await connection.execute<T>(
-          query.sql,
-          bindings,
-          options
-        )
-
-        return result
+        return await connection.execute<T>(query.sql, bindings, options)
       } finally {
         if (connection) {
           try {

package/src/integrations/postgres.ts

@@ -103,30 +103,11 @@ module PostgresModule {
     },
   }
 
-  async function internalQuery(client: any, query: SqlQuery) {
-    // need to handle a specific issue with json data types in postgres,
-    // new lines inside the JSON data will break it
-    if (query && query.sql) {
-      const matches = query.sql.match(JSON_REGEX)
-      if (matches && matches.length > 0) {
-        for (let match of matches) {
-          const escaped = escapeDangerousCharacters(match)
-          query.sql = query.sql.replace(match, escaped)
-        }
-      }
-    }
-    try {
-      return await client.query(query.sql, query.bindings || [])
-    } catch (err) {
-      // @ts-ignore
-      throw new Error(err)
-    }
-  }
-
   class PostgresIntegration extends Sql implements DatasourcePlus {
     static pool: any
     private readonly client: any
     private readonly config: PostgresConfig
+    private index: number = 1
     public tables: Record<string, Table> = {}
     public schemaErrors: Record<string, string> = {}
 
@@ -163,6 +144,32 @@ module PostgresModule {
       this.setSchema()
     }
 
+    getBindingIdentifier(): string {
+      return `$${this.index++}`
+    }
+
+    async internalQuery(query: SqlQuery) {
+      const client = this.client
+      this.index = 1
+      // need to handle a specific issue with json data types in postgres,
+      // new lines inside the JSON data will break it
+      if (query && query.sql) {
+        const matches = query.sql.match(JSON_REGEX)
+        if (matches && matches.length > 0) {
+          for (let match of matches) {
+            const escaped = escapeDangerousCharacters(match)
+            query.sql = query.sql.replace(match, escaped)
+          }
+        }
+      }
+      try {
+        return await client.query(query.sql, query.bindings || [])
+      } catch (err) {
+        // @ts-ignore
+        throw new Error(err)
+      }
+    }
+
     setSchema() {
       if (!this.config.schema) {
         this.config.schema = "public"
@@ -241,22 +248,22 @@ module PostgresModule {
     }
 
     async create(query: SqlQuery | string) {
-      const response = await internalQuery(this.client, getSqlQuery(query))
+      const response = await this.internalQuery(getSqlQuery(query))
       return response.rows.length ? response.rows : [{ created: true }]
     }
 
     async read(query: SqlQuery | string) {
-      const response = await internalQuery(this.client, getSqlQuery(query))
+      const response = await this.internalQuery(getSqlQuery(query))
       return response.rows
     }
 
     async update(query: SqlQuery | string) {
-      const response = await internalQuery(this.client, getSqlQuery(query))
+      const response = await this.internalQuery(getSqlQuery(query))
       return response.rows.length ? response.rows : [{ updated: true }]
     }
 
     async delete(query: SqlQuery | string) {
-      const response = await internalQuery(this.client, getSqlQuery(query))
+      const response = await this.internalQuery(getSqlQuery(query))
       return response.rows.length ? response.rows : [{ deleted: true }]
     }
 
@@ -266,11 +273,11 @@ module PostgresModule {
       if (Array.isArray(input)) {
         const responses = []
         for (let query of input) {
-          responses.push(await internalQuery(this.client, query))
+          responses.push(await this.internalQuery(query))
         }
         return responses
       } else {
-        const response = await internalQuery(this.client, input)
+        const response = await this.internalQuery(input)
         return response.rows.length ? response.rows : [{ [operation]: true }]
       }
     }