@budibase/server 0.9.184 → 0.9.185-alpha.11

package/src/integrations/base/sqlTable.ts

@@ -0,0 +1,180 @@
+import { Knex, knex } from "knex"
+import { Table } from "../../definitions/common"
+import { Operation, QueryJson } from "../../definitions/datasource"
+import { breakExternalTableId } from "../utils"
+import SchemaBuilder = Knex.SchemaBuilder
+import CreateTableBuilder = Knex.CreateTableBuilder
+const { FieldTypes, RelationshipTypes } = require("../../constants")
+
+function generateSchema(
+  schema: CreateTableBuilder,
+  table: Table,
+  tables: Record<string, Table>,
+  oldTable: null | Table = null
+) {
+  let primaryKey = table && table.primary ? table.primary[0] : null
+  const columns = Object.values(table.schema)
+  // all columns in a junction table will be meta
+  let metaCols = columns.filter(col => col.meta)
+  let isJunction = metaCols.length === columns.length
+  // can't change primary once its set for now
+  if (primaryKey && !oldTable && !isJunction) {
+    schema.increments(primaryKey).primary()
+  } else if (!oldTable && isJunction) {
+    schema.primary(metaCols.map(col => col.name))
+  }
+
+  // check if any columns need added
+  const foreignKeys = Object.values(table.schema).map(col => col.foreignKey)
+  for (let [key, column] of Object.entries(table.schema)) {
+    // skip things that are already correct
+    const oldColumn = oldTable ? oldTable.schema[key] : null
+    if ((oldColumn && oldColumn.type) || (primaryKey === key && !isJunction)) {
+      continue
+    }
+    switch (column.type) {
+      case FieldTypes.STRING:
+      case FieldTypes.OPTIONS:
+      case FieldTypes.LONGFORM:
+        schema.string(key)
+        break
+      case FieldTypes.NUMBER:
+        // if meta is specified then this is a junction table entry
+        if (column.meta && column.meta.toKey && column.meta.toTable) {
+          const { toKey, toTable } = column.meta
+          schema.integer(key).unsigned()
+          schema.foreign(key).references(`${toTable}.${toKey}`)
+        } else if (foreignKeys.indexOf(key) === -1) {
+          schema.float(key)
+        }
+        break
+      case FieldTypes.BOOLEAN:
+        schema.boolean(key)
+        break
+      case FieldTypes.DATETIME:
+        schema.datetime(key)
+        break
+      case FieldTypes.ARRAY:
+        schema.json(key)
+        break
+      case FieldTypes.LINK:
+        // this side of the relationship doesn't need any SQL work
+        if (
+          column.relationshipType !== RelationshipTypes.MANY_TO_ONE &&
+          column.relationshipType !== RelationshipTypes.MANY_TO_MANY
+        ) {
+          if (!column.foreignKey || !column.tableId) {
+            throw "Invalid relationship schema"
+          }
+          const { tableName } = breakExternalTableId(column.tableId)
+          // @ts-ignore
+          const relatedTable = tables[tableName]
+          if (!relatedTable) {
+            throw "Referenced table doesn't exist"
+          }
+          schema.integer(column.foreignKey).unsigned()
+          schema
+            .foreign(column.foreignKey)
+            .references(`${tableName}.${relatedTable.primary[0]}`)
+        }
+        break
+    }
+  }
+
+  // need to check if any columns have been deleted
+  if (oldTable) {
+    const deletedColumns = Object.entries(oldTable.schema)
+      .filter(
+        ([key, schema]) =>
+          schema.type !== FieldTypes.LINK && table.schema[key] == null
+      )
+      .map(([key]) => key)
+    deletedColumns.forEach(key => {
+      if (oldTable.constrained && oldTable.constrained.indexOf(key) !== -1) {
+        schema.dropForeign(key)
+      }
+      schema.dropColumn(key)
+    })
+  }
+
+  return schema
+}
+
+function buildCreateTable(
+  knex: Knex,
+  table: Table,
+  tables: Record<string, Table>
+): SchemaBuilder {
+  return knex.schema.createTable(table.name, schema => {
+    generateSchema(schema, table, tables)
+  })
+}
+
+function buildUpdateTable(
+  knex: Knex,
+  table: Table,
+  tables: Record<string, Table>,
+  oldTable: Table
+): SchemaBuilder {
+  return knex.schema.alterTable(table.name, schema => {
+    generateSchema(schema, table, tables, oldTable)
+  })
+}
+
+function buildDeleteTable(knex: Knex, table: Table): SchemaBuilder {
+  return knex.schema.dropTable(table.name)
+}
+
+class SqlTableQueryBuilder {
+  private readonly sqlClient: string
+
+  // pass through client to get flavour of SQL
+  constructor(client: string) {
+    this.sqlClient = client
+  }
+
+  getSqlClient(): string {
+    return this.sqlClient
+  }
+
+  /**
+   * @param json the input JSON structure from which an SQL query will be built.
+   * @return {string} the operation that was found in the JSON.
+   */
+  _operation(json: QueryJson): Operation {
+    return json.endpoint.operation
+  }
+
+  _tableQuery(json: QueryJson): any {
+    const client = knex({ client: this.sqlClient })
+    let query
+    if (!json.table || !json.meta || !json.meta.tables) {
+      throw "Cannot execute without table being specified"
+    }
+    switch (this._operation(json)) {
+      case Operation.CREATE_TABLE:
+        query = buildCreateTable(client, json.table, json.meta.tables)
+        break
+      case Operation.UPDATE_TABLE:
+        if (!json.meta || !json.meta.table) {
+          throw "Must specify old table for update"
+        }
+        query = buildUpdateTable(
+          client,
+          json.table,
+          json.meta.tables,
+          json.meta.table
+        )
+        break
+      case Operation.DELETE_TABLE:
+        query = buildDeleteTable(client, json.table)
+        break
+      default:
+        throw "Table operation is of unknown type"
+    }
+    return query.toSQL()
+  }
+}
+
+export default SqlTableQueryBuilder
+module.exports = SqlTableQueryBuilder

package/src/integrations/base/utils.ts

@@ -0,0 +1,22 @@
+import { QueryJson } from "../../definitions/datasource"
+import { Datasource } from "../../definitions/common"
+
+module DatasourceUtils {
+  const { integrations } = require("../index")
+
+  export async function makeExternalQuery(
+    datasource: Datasource,
+    json: QueryJson
+  ) {
+    const Integration = integrations[datasource.source]
+    // query is the opinionated function
+    if (Integration.prototype.query) {
+      const integration = new Integration(datasource.config)
+      return integration.query(json)
+    } else {
+      throw "Datasource does not support query."
+    }
+  }
+
+  module.exports.makeExternalQuery = makeExternalQuery
+}

package/src/integrations/couchdb.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module CouchDBModule {
   const PouchDB = require("pouchdb")
@@ -50,7 +51,7 @@ module CouchDBModule {
     },
   }
 
-  class CouchDBIntegration {
+  class CouchDBIntegration implements IntegrationBase {
     private config: CouchDBConfig
     private client: any
 

package/src/integrations/dynamodb.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module DynamoModule {
   const AWS = require("aws-sdk")
@@ -113,7 +114,7 @@ module DynamoModule {
     },
   }
 
-  class DynamoDBIntegration {
+  class DynamoDBIntegration implements IntegrationBase {
     private config: DynamoDBConfig
     private client: any
 

package/src/integrations/elasticsearch.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module ElasticsearchModule {
   const { Client } = require("@elastic/elasticsearch")
@@ -74,7 +75,7 @@ module ElasticsearchModule {
     },
   }
 
-  class ElasticSearchIntegration {
+  class ElasticSearchIntegration implements IntegrationBase {
     private config: ElasticsearchConfig
     private client: any
 

package/src/integrations/microsoftSqlServer.ts

@@ -1,11 +1,20 @@
 import {
-  Integration,
   DatasourceFieldTypes,
-  QueryTypes,
+  Integration,
+  Operation,
   QueryJson,
+  QueryTypes,
   SqlQuery,
 } from "../definitions/datasource"
-import { getSqlQuery } from "./utils"
+import {
+  getSqlQuery,
+  buildExternalTableId,
+  convertSqlType,
+  finaliseExternalTables,
+  SqlClients,
+} from "./utils"
+import { DatasourcePlus } from "./base/datasourcePlus"
+import { Table, TableSchema } from "../definitions/common"
 
 module MSSQLModule {
   const sqlServer = require("mssql")
@@ -22,6 +31,7 @@ module MSSQLModule {
 
   const SCHEMA: Integration = {
     docs: "https://github.com/tediousjs/node-mssql",
+    plus: true,
     description:
       "Microsoft SQL Server is a relational database management system developed by Microsoft. ",
     friendlyName: "MS SQL Server",
@@ -69,26 +79,84 @@ module MSSQLModule {
     },
   }
 
-  async function internalQuery(client: any, query: SqlQuery) {
+  async function internalQuery(
+    client: any,
+    query: SqlQuery,
+    operation: string | undefined = undefined
+  ) {
+    const request = client.request()
     try {
-      return await client.query(query.sql, query.bindings || {})
+      if (Array.isArray(query.bindings)) {
+        let count = 0
+        for (let binding of query.bindings) {
+          request.input(`p${count++}`, binding)
+        }
+      }
+      // this is a hack to get the inserted ID back,
+      //  no way to do this with Knex nicely
+      const sql =
+        operation === Operation.CREATE
+          ? `${query.sql}; SELECT SCOPE_IDENTITY() AS id;`
+          : query.sql
+      return await request.query(sql)
     } catch (err) {
       // @ts-ignore
       throw new Error(err)
     }
   }
 
-  class SqlServerIntegration extends Sql {
+  class SqlServerIntegration extends Sql implements DatasourcePlus {
     private readonly config: MSSQLConfig
     static pool: any
+    public tables: Record<string, Table> = {}
+    public schemaErrors: Record<string, string> = {}
+
+    MASTER_TABLES = [
+      "spt_fallback_db",
+      "spt_fallback_dev",
+      "spt_fallback_usg",
+      "spt_monitor",
+      "MSreplication_options",
+    ]
+    TABLES_SQL =
+      "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='BASE TABLE'"
+
+    getDefinitionSQL(tableName: string) {
+      return `select *
+              from INFORMATION_SCHEMA.COLUMNS
+              where TABLE_NAME='${tableName}'`
+    }
+
+    getConstraintsSQL(tableName: string) {
+      return `SELECT * FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS AS TC 
+              INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE AS KU
+                ON TC.CONSTRAINT_TYPE = 'PRIMARY KEY' 
+                AND TC.CONSTRAINT_NAME = KU.CONSTRAINT_NAME 
+                AND KU.table_name='${tableName}'
+              ORDER BY 
+                KU.TABLE_NAME,
+                KU.ORDINAL_POSITION;`
+    }
+
+    getAutoColumnsSQL(tableName: string) {
+      return `SELECT 
+              COLUMNPROPERTY(OBJECT_ID(TABLE_SCHEMA+'.'+TABLE_NAME),COLUMN_NAME,'IsComputed') 
+                AS IS_COMPUTED,
+              COLUMNPROPERTY(object_id(TABLE_SCHEMA+'.'+TABLE_NAME), COLUMN_NAME, 'IsIdentity')
+                AS IS_IDENTITY,
+              *
+              FROM INFORMATION_SCHEMA.COLUMNS
+              WHERE TABLE_NAME='${tableName}'`
+    }
 
     constructor(config: MSSQLConfig) {
-      super("mssql")
+      super(SqlClients.MS_SQL)
       this.config = config
       const clientCfg = {
         ...this.config,
         options: {
           encrypt: this.config.encrypt,
+          enableArithAbort: true,
         },
       }
       delete clientCfg.encrypt
@@ -99,14 +167,75 @@ module MSSQLModule {
 
     async connect() {
       try {
-        const client = await this.pool.connect()
-        this.client = client.request()
+        this.client = await this.pool.connect()
       } catch (err) {
         // @ts-ignore
         throw new Error(err)
       }
     }
 
+    async runSQL(sql: string) {
+      return (await internalQuery(this.client, getSqlQuery(sql))).recordset
+    }
+
+    /**
+     * Fetches the tables from the sql server database and assigns them to the datasource.
+     * @param {*} datasourceId - datasourceId to fetch
+     * @param entities - the tables that are to be built
+     */
+    async buildSchema(datasourceId: string, entities: Record<string, Table>) {
+      await this.connect()
+      let tableNames = await this.runSQL(this.TABLES_SQL)
+      if (tableNames == null || !Array.isArray(tableNames)) {
+        throw "Unable to get list of tables in database"
+      }
+      tableNames = tableNames
+        .map((record: any) => record.TABLE_NAME)
+        .filter((name: string) => this.MASTER_TABLES.indexOf(name) === -1)
+
+      const tables: Record<string, Table> = {}
+      for (let tableName of tableNames) {
+        // get the column definition (type)
+        const definition = await this.runSQL(this.getDefinitionSQL(tableName))
+        // find primary key constraints
+        const constraints = await this.runSQL(this.getConstraintsSQL(tableName))
+        // find the computed and identity columns (auto columns)
+        const columns = await this.runSQL(this.getAutoColumnsSQL(tableName))
+        const primaryKeys = constraints
+          .filter(
+            (constraint: any) => constraint.CONSTRAINT_TYPE === "PRIMARY KEY"
+          )
+          .map((constraint: any) => constraint.COLUMN_NAME)
+        const autoColumns = columns
+          .filter((col: any) => col.IS_COMPUTED || col.IS_IDENTITY)
+          .map((col: any) => col.COLUMN_NAME)
+
+        let schema: TableSchema = {}
+        for (let def of definition) {
+          const name = def.COLUMN_NAME
+          if (typeof name !== "string") {
+            continue
+          }
+          const type: string = convertSqlType(def.DATA_TYPE)
+
+          schema[name] = {
+            autocolumn: !!autoColumns.find((col: string) => col === name),
+            name: name,
+            type,
+          }
+        }
+        tables[tableName] = {
+          _id: buildExternalTableId(datasourceId, tableName),
+          primary: primaryKeys,
+          name: tableName,
+          schema,
+        }
+      }
+      const final = finaliseExternalTables(tables, entities)
+      this.tables = final.tables
+      this.schemaErrors = final.errors
+    }
+
     async read(query: SqlQuery | string) {
       await this.connect()
       const response = await internalQuery(this.client, getSqlQuery(query))
@@ -132,10 +261,13 @@ module MSSQLModule {
     }
 
     async query(json: QueryJson) {
-      const operation = this._operation(json).toLowerCase()
-      const input = this._query(json)
-      const response = await internalQuery(this.client, input)
-      return response.recordset ? response.recordset : [{ [operation]: true }]
+      await this.connect()
+      const operation = this._operation(json)
+      const queryFn = (query: any, op: string) =>
+        internalQuery(this.client, query, op)
+      const processFn = (result: any) =>
+        result.recordset ? result.recordset : [{ [operation]: true }]
+      return this.queryWithReturning(json, queryFn, processFn)
     }
   }
 

package/src/integrations/mongodb.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module MongoDBModule {
   const { MongoClient } = require("mongodb")
@@ -62,7 +63,7 @@ module MongoDBModule {
     },
   }
 
-  class MongoIntegration {
+  class MongoIntegration implements IntegrationBase {
     private config: MongoDBConfig
     private client: any
 

package/src/integrations/mysql.ts

@@ -2,22 +2,22 @@ import {
   Integration,
   DatasourceFieldTypes,
   QueryTypes,
-  Operation,
   QueryJson,
   SqlQuery,
 } from "../definitions/datasource"
 import { Table, TableSchema } from "../definitions/common"
-import { getSqlQuery } from "./utils"
+import {
+  getSqlQuery,
+  SqlClients,
+  buildExternalTableId,
+  convertSqlType,
+  finaliseExternalTables,
+} from "./utils"
+import { DatasourcePlus } from "./base/datasourcePlus"
 
 module MySQLModule {
   const mysql = require("mysql2")
   const Sql = require("./base/sql")
-  const {
-    buildExternalTableId,
-    convertType,
-    copyExistingPropsOver,
-  } = require("./utils")
-  const { FieldTypes } = require("../constants")
 
   interface MySQLConfig {
     host: string
@@ -28,30 +28,6 @@ module MySQLModule {
     ssl?: object
   }
 
-  const TYPE_MAP = {
-    text: FieldTypes.LONGFORM,
-    blob: FieldTypes.LONGFORM,
-    enum: FieldTypes.STRING,
-    varchar: FieldTypes.STRING,
-    float: FieldTypes.NUMBER,
-    int: FieldTypes.NUMBER,
-    numeric: FieldTypes.NUMBER,
-    bigint: FieldTypes.NUMBER,
-    mediumint: FieldTypes.NUMBER,
-    decimal: FieldTypes.NUMBER,
-    dec: FieldTypes.NUMBER,
-    double: FieldTypes.NUMBER,
-    real: FieldTypes.NUMBER,
-    fixed: FieldTypes.NUMBER,
-    smallint: FieldTypes.NUMBER,
-    timestamp: FieldTypes.DATETIME,
-    date: FieldTypes.DATETIME,
-    datetime: FieldTypes.DATETIME,
-    time: FieldTypes.DATETIME,
-    tinyint: FieldTypes.BOOLEAN,
-    json: DatasourceFieldTypes.JSON,
-  }
-
   const SCHEMA: Integration = {
     docs: "https://github.com/mysqljs/mysql",
     plus: true,
@@ -131,12 +107,14 @@ module MySQLModule {
     })
   }
 
-  class MySQLIntegration extends Sql {
+  class MySQLIntegration extends Sql implements DatasourcePlus {
     private config: MySQLConfig
     private readonly client: any
+    public tables: Record<string, Table> = {}
+    public schemaErrors: Record<string, string> = {}
 
     constructor(config: MySQLConfig) {
-      super("mysql")
+      super(SqlClients.MY_SQL)
       this.config = config
       if (config.ssl && Object.keys(config.ssl).length === 0) {
         delete config.ssl
@@ -181,14 +159,10 @@ module MySQLModule {
           schema[columnName] = {
             name: columnName,
             autocolumn: isAuto,
-            type: convertType(column.Type, TYPE_MAP),
+            type: convertSqlType(column.Type),
             constraints,
           }
         }
-        // for now just default to first column
-        if (primaryKeys.length === 0) {
-          primaryKeys.push(descResp[0].Field)
-        }
         if (!tables[tableName]) {
           tables[tableName] = {
             _id: buildExternalTableId(datasourceId, tableName),
@@ -197,12 +171,12 @@ module MySQLModule {
             schema,
           }
         }
-
-        copyExistingPropsOver(tableName, tables, entities)
       }
 
       this.client.end()
-      this.tables = tables
+      const final = finaliseExternalTables(tables, entities)
+      this.tables = final.tables
+      this.schemaErrors = final.errors
     }
 
     async create(query: SqlQuery | string) {
@@ -210,7 +184,7 @@ module MySQLModule {
       return results.length ? results : [{ created: true }]
     }
 
-    read(query: SqlQuery | string) {
+    async read(query: SqlQuery | string) {
       return internalQuery(this.client, getSqlQuery(query))
     }
 
@@ -224,60 +198,12 @@ module MySQLModule {
       return results.length ? results : [{ deleted: true }]
     }
 
-    async getReturningRow(json: QueryJson) {
-      if (!json.extra || !json.extra.idFilter) {
-        return {}
-      }
-      const input = this._query({
-        endpoint: {
-          ...json.endpoint,
-          operation: Operation.READ,
-        },
-        fields: [],
-        filters: json.extra.idFilter,
-        paginate: {
-          limit: 1,
-        },
-      })
-      return internalQuery(this.client, input, false)
-    }
-
-    // when creating if an ID has been inserted need to make sure
-    // the id filter is enriched with it before trying to retrieve the row
-    checkLookupKeys(results: any, json: QueryJson) {
-      if (!results?.insertId || !json.meta?.table || !json.meta.table.primary) {
-        return json
-      }
-      const primaryKey = json.meta.table.primary?.[0]
-      json.extra = {
-        idFilter: {
-          equal: {
-            [primaryKey]: results.insertId,
-          },
-        },
-      }
-      return json
-    }
-
     async query(json: QueryJson) {
-      const operation = this._operation(json)
       this.client.connect()
-      const input = this._query(json, { disableReturning: true })
-      let row
-      // need to manage returning, a feature mySQL can't do
-      if (operation === operation.DELETE) {
-        row = this.getReturningRow(json)
-      }
-      const results = await internalQuery(this.client, input, false)
-      // same as delete, manage returning
-      if (operation === Operation.CREATE || operation === Operation.UPDATE) {
-        row = this.getReturningRow(this.checkLookupKeys(results, json))
-      }
+      const queryFn = (query: any) => internalQuery(this.client, query, false)
+      const output = await this.queryWithReturning(json, queryFn)
       this.client.end()
-      if (operation !== Operation.READ) {
-        return row
-      }
-      return results.length ? results : [{ [operation.toLowerCase()]: true }]
+      return output
     }
   }