@budibase/server 0.9.174 → 0.9.176-alpha.2

package/src/api/controllers/table/index.js

@@ -1,16 +1,28 @@
 const CouchDB = require("../../../db")
-const linkRows = require("../../../db/linkedRows")
+const internal = require("./internal")
+const external = require("./external")
 const csvParser = require("../../../utilities/csvParser")
+const { isExternalTable } = require("../../../integrations/utils")
 const {
-  getRowParams,
   getTableParams,
-  generateTableID,
   getDatasourceParams,
   BudibaseInternalDB,
 } = require("../../../db/utils")
-const { FieldTypes } = require("../../../constants")
-const { TableSaveFunctions, getTable } = require("./utils")
+const { getTable } = require("./utils")
 
+function pickApi({ tableId, table }) {
+  if (table && !tableId) {
+    tableId = table._id
+  }
+  if (table && table.type === "external") {
+    return external
+  } else if (tableId && isExternalTable(tableId)) {
+    return external
+  }
+  return internal
+}
+
+// covers both internal and external
 exports.fetch = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
@@ -50,143 +62,23 @@ exports.find = async function (ctx) {
 
 exports.save = async function (ctx) {
   const appId = ctx.appId
-  const db = new CouchDB(appId)
-  const { dataImport, ...rest } = ctx.request.body
-  let tableToSave = {
-    type: "table",
-    _id: generateTableID(),
-    views: {},
-    ...rest,
-  }
-
-  // if the table obj had an _id then it will have been retrieved
-  let oldTable
-  if (ctx.request.body && ctx.request.body._id) {
-    oldTable = await db.get(ctx.request.body._id)
-  }
-
-  // saving a table is a complex operation, involving many different steps, this
-  // has been broken out into a utility to make it more obvious/easier to manipulate
-  const tableSaveFunctions = new TableSaveFunctions({
-    db,
-    ctx,
-    oldTable,
-    dataImport,
-  })
-  tableToSave = await tableSaveFunctions.before(tableToSave)
-
-  // make sure that types don't change of a column, have to remove
-  // the column if you want to change the type
-  if (oldTable && oldTable.schema) {
-    for (let propKey of Object.keys(tableToSave.schema)) {
-      let column = tableToSave.schema[propKey]
-      let oldColumn = oldTable.schema[propKey]
-      if (oldColumn && oldColumn.type === "internal") {
-        oldColumn.type = "auto"
-      }
-      if (oldColumn && oldColumn.type !== column.type) {
-        ctx.throw(400, "Cannot change the type of a column")
-      }
-    }
-  }
-
-  // Don't rename if the name is the same
-  let { _rename } = tableToSave
-  /* istanbul ignore next */
-  if (_rename && _rename.old === _rename.updated) {
-    _rename = null
-    delete tableToSave._rename
-  }
-
-  // rename row fields when table column is renamed
-  /* istanbul ignore next */
-  if (_rename && tableToSave.schema[_rename.updated].type === FieldTypes.LINK) {
-    ctx.throw(400, "Cannot rename a linked column.")
-  }
-
-  tableToSave = await tableSaveFunctions.mid(tableToSave)
-
-  // update schema of non-statistics views when new columns are added
-  for (let view in tableToSave.views) {
-    const tableView = tableToSave.views[view]
-    if (!tableView) continue
-
-    if (tableView.schema.group || tableView.schema.field) continue
-    tableView.schema = tableToSave.schema
-  }
-
-  // update linked rows
-  try {
-    const linkResp = await linkRows.updateLinks({
-      appId,
-      eventType: oldTable
-        ? linkRows.EventType.TABLE_UPDATED
-        : linkRows.EventType.TABLE_SAVE,
-      table: tableToSave,
-      oldTable: oldTable,
-    })
-    if (linkResp != null && linkResp._rev) {
-      tableToSave._rev = linkResp._rev
-    }
-  } catch (err) {
-    ctx.throw(400, err)
-  }
-
-  // don't perform any updates until relationships have been
-  // checked by the updateLinks function
-  const updatedRows = tableSaveFunctions.getUpdatedRows()
-  if (updatedRows && updatedRows.length !== 0) {
-    await db.bulkDocs(updatedRows)
-  }
-  const result = await db.put(tableToSave)
-  tableToSave._rev = result.rev
-
-  tableToSave = await tableSaveFunctions.after(tableToSave)
-
-  ctx.eventEmitter &&
-    ctx.eventEmitter.emitTable(`table:save`, appId, tableToSave)
-
+  const table = ctx.request.body
+  const savedTable = await pickApi({ table }).save(ctx)
   ctx.status = 200
-  ctx.message = `Table ${ctx.request.body.name} saved successfully.`
-  ctx.body = tableToSave
+  ctx.message = `Table ${table.name} saved successfully.`
+  ctx.eventEmitter &&
+    ctx.eventEmitter.emitTable(`table:save`, appId, savedTable)
+  ctx.body = savedTable
 }
 
 exports.destroy = async function (ctx) {
   const appId = ctx.appId
-  const db = new CouchDB(appId)
-  const tableToDelete = await db.get(ctx.params.tableId)
-
-  // Delete all rows for that table
-  const rows = await db.allDocs(
-    getRowParams(ctx.params.tableId, null, {
-      include_docs: true,
-    })
-  )
-  await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
-
-  // update linked rows
-  await linkRows.updateLinks({
-    appId,
-    eventType: linkRows.EventType.TABLE_DELETE,
-    table: tableToDelete,
-  })
-
-  // don't remove the table itself until very end
-  await db.remove(tableToDelete)
-
-  // remove table search index
-  const currentIndexes = await db.getIndexes()
-  const existingIndex = currentIndexes.indexes.find(
-    existing => existing.name === `search:${ctx.params.tableId}`
-  )
-  if (existingIndex) {
-    await db.deleteIndex(existingIndex)
-  }
-
+  const tableId = ctx.params.tableId
+  const deletedTable = await pickApi({ tableId }).destroy(ctx)
   ctx.eventEmitter &&
-    ctx.eventEmitter.emitTable(`table:delete`, appId, tableToDelete)
+    ctx.eventEmitter.emitTable(`table:delete`, appId, deletedTable)
   ctx.status = 200
-  ctx.body = { message: `Table ${ctx.params.tableId} deleted.` }
+  ctx.body = { message: `Table ${tableId} deleted.` }
 }
 
 exports.validateCSVSchema = async function (ctx) {

package/src/api/controllers/table/internal.js

@@ -0,0 +1,138 @@
+const CouchDB = require("../../../db")
+const linkRows = require("../../../db/linkedRows")
+const { getRowParams, generateTableID } = require("../../../db/utils")
+const { FieldTypes } = require("../../../constants")
+const { TableSaveFunctions } = require("./utils")
+
+exports.save = async function (ctx) {
+  const appId = ctx.appId
+  const db = new CouchDB(appId)
+  const { dataImport, ...rest } = ctx.request.body
+  let tableToSave = {
+    type: "table",
+    _id: generateTableID(),
+    views: {},
+    ...rest,
+  }
+
+  // if the table obj had an _id then it will have been retrieved
+  let oldTable
+  if (ctx.request.body && ctx.request.body._id) {
+    oldTable = await db.get(ctx.request.body._id)
+  }
+
+  // saving a table is a complex operation, involving many different steps, this
+  // has been broken out into a utility to make it more obvious/easier to manipulate
+  const tableSaveFunctions = new TableSaveFunctions({
+    db,
+    ctx,
+    oldTable,
+    dataImport,
+  })
+  tableToSave = await tableSaveFunctions.before(tableToSave)
+
+  // make sure that types don't change of a column, have to remove
+  // the column if you want to change the type
+  if (oldTable && oldTable.schema) {
+    for (let propKey of Object.keys(tableToSave.schema)) {
+      let column = tableToSave.schema[propKey]
+      let oldColumn = oldTable.schema[propKey]
+      if (oldColumn && oldColumn.type === "internal") {
+        oldColumn.type = "auto"
+      }
+      if (oldColumn && oldColumn.type !== column.type) {
+        ctx.throw(400, "Cannot change the type of a column")
+      }
+    }
+  }
+
+  // Don't rename if the name is the same
+  let { _rename } = tableToSave
+  /* istanbul ignore next */
+  if (_rename && _rename.old === _rename.updated) {
+    _rename = null
+    delete tableToSave._rename
+  }
+
+  // rename row fields when table column is renamed
+  /* istanbul ignore next */
+  if (_rename && tableToSave.schema[_rename.updated].type === FieldTypes.LINK) {
+    ctx.throw(400, "Cannot rename a linked column.")
+  }
+
+  tableToSave = await tableSaveFunctions.mid(tableToSave)
+
+  // update schema of non-statistics views when new columns are added
+  for (let view in tableToSave.views) {
+    const tableView = tableToSave.views[view]
+    if (!tableView) continue
+
+    if (tableView.schema.group || tableView.schema.field) continue
+    tableView.schema = tableToSave.schema
+  }
+
+  // update linked rows
+  try {
+    const linkResp = await linkRows.updateLinks({
+      appId,
+      eventType: oldTable
+        ? linkRows.EventType.TABLE_UPDATED
+        : linkRows.EventType.TABLE_SAVE,
+      table: tableToSave,
+      oldTable: oldTable,
+    })
+    if (linkResp != null && linkResp._rev) {
+      tableToSave._rev = linkResp._rev
+    }
+  } catch (err) {
+    ctx.throw(400, err)
+  }
+
+  // don't perform any updates until relationships have been
+  // checked by the updateLinks function
+  const updatedRows = tableSaveFunctions.getUpdatedRows()
+  if (updatedRows && updatedRows.length !== 0) {
+    await db.bulkDocs(updatedRows)
+  }
+  const result = await db.put(tableToSave)
+  tableToSave._rev = result.rev
+
+  tableToSave = await tableSaveFunctions.after(tableToSave)
+
+  return tableToSave
+}
+
+exports.destroy = async function (ctx) {
+  const appId = ctx.appId
+  const db = new CouchDB(appId)
+  const tableToDelete = await db.get(ctx.params.tableId)
+
+  // Delete all rows for that table
+  const rows = await db.allDocs(
+    getRowParams(ctx.params.tableId, null, {
+      include_docs: true,
+    })
+  )
+  await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
+
+  // update linked rows
+  await linkRows.updateLinks({
+    appId,
+    eventType: linkRows.EventType.TABLE_DELETE,
+    table: tableToDelete,
+  })
+
+  // don't remove the table itself until very end
+  await db.remove(tableToDelete)
+
+  // remove table search index
+  const currentIndexes = await db.getIndexes()
+  const existingIndex = currentIndexes.indexes.find(
+    existing => existing.name === `search:${ctx.params.tableId}`
+  )
+  if (existingIndex) {
+    await db.deleteIndex(existingIndex)
+  }
+
+  return tableToDelete
+}

package/src/api/controllers/table/utils.js

@@ -315,4 +315,24 @@ exports.checkForViewUpdates = async (db, table, rename, deletedColumns) => {
   }
 }
 
+exports.generateForeignKey = (column, relatedTable) => {
+  return `fk_${relatedTable.name}_${column.fieldName}`
+}
+
+exports.generateJunctionTableName = (column, table, relatedTable) => {
+  return `jt_${table.name}_${relatedTable.name}_${column.name}_${column.fieldName}`
+}
+
+exports.foreignKeyStructure = (keyName, meta = null) => {
+  const structure = {
+    type: FieldTypes.NUMBER,
+    constraints: {},
+    name: keyName,
+  }
+  if (meta) {
+    structure.meta = meta
+  }
+  return structure
+}
+
 exports.TableSaveFunctions = TableSaveFunctions

package/src/api/routes/tests/datasource.spec.js

@@ -26,8 +26,8 @@ describe("/datasources", () => {
         .expect('Content-Type', /json/)
         .expect(200)
 
-      expect(res.res.statusMessage).toEqual("Datasource saved successfully.")
-      expect(res.body.name).toEqual("Test")
+      expect(res.body.datasource.name).toEqual("Test")
+      expect(res.body.errors).toBeUndefined()
     })
   })
 

package/src/api/routes/tests/query.spec.js

@@ -1,7 +1,6 @@
 // mock out postgres for this
 jest.mock("pg")
 
-const { findLastKey } = require("lodash/fp")
 const setup = require("./utilities")
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const { basicQuery, basicDatasource } = setup.structures

package/src/constants/index.js

@@ -62,6 +62,9 @@ exports.DataSourceOperation = {
   READ: "READ",
   UPDATE: "UPDATE",
   DELETE: "DELETE",
+  CREATE_TABLE: "CREATE_TABLE",
+  UPDATE_TABLE: "UPDATE_TABLE",
+  DELETE_TABLE: "DELETE_TABLE",
 }
 
 exports.SortDirection = {
@@ -152,5 +155,9 @@ exports.MetadataTypes = {
   AUTOMATION_TEST_HISTORY: "automationTestHistory",
 }
 
+exports.BuildSchemaErrors = {
+  NO_KEY: "no_key",
+}
+
 // pass through the list from the auth/core lib
 exports.ObjectStoreBuckets = ObjectStoreBuckets

package/src/definitions/common.ts

@@ -17,6 +17,11 @@ export interface FieldSchema {
   autocolumn?: boolean
   throughFrom?: string
   throughTo?: string
+  main?: boolean
+  meta?: {
+    toTable: string
+    toKey: string
+  }
   constraints?: {
     type?: string
     email?: boolean
@@ -36,11 +41,12 @@ export interface TableSchema {
 export interface Table extends Base {
   type?: string
   views?: {}
-  name?: string
+  name: string
   primary?: string[]
   schema: TableSchema
   primaryDisplay?: string
   sourceId?: string
+  constrained?: string[]
 }
 
 export interface Row extends Base {

package/src/definitions/datasource.ts

@@ -5,6 +5,9 @@ export enum Operation {
   READ = "READ",
   UPDATE = "UPDATE",
   DELETE = "DELETE",
+  CREATE_TABLE = "CREATE_TABLE",
+  UPDATE_TABLE = "UPDATE_TABLE",
+  DELETE_TABLE = "DELETE_TABLE",
 }
 
 export enum SortDirection {
@@ -142,8 +145,10 @@ export interface QueryJson {
   sort?: SortJson
   paginate?: PaginationJson
   body?: object
+  table?: Table
   meta?: {
     table?: Table
+    tables?: Record<string, Table>
   }
   extra?: {
     idFilter?: SearchFilters

package/src/integrations/base/datasourcePlus.ts

@@ -0,0 +1,8 @@
+import { Table } from "../../definitions/common"
+
+export interface DatasourcePlus {
+  tables: Record<string, Table>
+  schemaErrors: Record<string, string>
+
+  buildSchema(datasourceId: string, entities: Record<string, Table>): any
+}

package/src/integrations/base/sql.ts

@@ -1,19 +1,24 @@
 import { Knex, knex } from "knex"
-const BASE_LIMIT = 5000
 import {
+  Operation,
   QueryJson,
-  SearchFilters,
   QueryOptions,
-  SortDirection,
-  Operation,
   RelationshipsJson,
+  SearchFilters,
+  SortDirection,
 } from "../../definitions/datasource"
 import { isIsoDateString } from "../utils"
+import SqlTableQueryBuilder from "./sqlTable"
+
+const BASE_LIMIT = 5000
 
 type KnexQuery = Knex.QueryBuilder | Knex
 
 function parseBody(body: any) {
   for (let [key, value] of Object.entries(body)) {
+    if (Array.isArray(value)) {
+      body[key] = JSON.stringify(value)
+    }
     if (typeof value !== "string") {
       continue
     }
@@ -243,23 +248,14 @@ function buildDelete(
   }
 }
 
-class SqlQueryBuilder {
-  private readonly sqlClient: string
+class SqlQueryBuilder extends SqlTableQueryBuilder {
   private readonly limit: number
   // pass through client to get flavour of SQL
   constructor(client: string, limit: number = BASE_LIMIT) {
-    this.sqlClient = client
+    super(client)
     this.limit = limit
   }
 
-  /**
-   * @param json the input JSON structure from which an SQL query will be built.
-   * @return {string} the operation that was found in the JSON.
-   */
-  _operation(json: QueryJson): Operation {
-    return json.endpoint.operation
-  }
-
   /**
    * @param json The JSON query DSL which is to be converted to SQL.
    * @param opts extra options which are to be passed into the query builder, e.g. disableReturning
@@ -267,7 +263,8 @@ class SqlQueryBuilder {
    * @return {{ sql: string, bindings: object }} the query ready to be passed to the driver.
    */
   _query(json: QueryJson, opts: QueryOptions = {}) {
-    const client = knex({ client: this.sqlClient })
+    const sqlClient = this.getSqlClient()
+    const client = knex({ client: sqlClient })
     let query
     switch (this._operation(json)) {
       case Operation.CREATE:
@@ -282,6 +279,8 @@ class SqlQueryBuilder {
       case Operation.DELETE:
         query = buildDelete(client, json, opts)
         break
+      case Operation.CREATE_TABLE: case Operation.UPDATE_TABLE: case Operation.DELETE_TABLE:
+        return this._tableQuery(json)
       default:
         throw `Operation type is not supported by SQL query builder`
     }

package/src/integrations/base/sqlTable.ts

@@ -0,0 +1,167 @@
+import { Knex, knex } from "knex"
+import { Table } from "../../definitions/common"
+import { Operation, QueryJson } from "../../definitions/datasource"
+import { breakExternalTableId } from "../utils"
+import SchemaBuilder = Knex.SchemaBuilder
+import CreateTableBuilder = Knex.CreateTableBuilder
+const { FieldTypes, RelationshipTypes } = require("../../constants")
+
+function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record<string, Table>, oldTable: null | Table = null) {
+  let primaryKey = table && table.primary ? table.primary[0] : null
+  const columns = Object.values(table.schema)
+  // all columns in a junction table will be meta
+  let metaCols = columns.filter(col => col.meta)
+  let isJunction = metaCols.length === columns.length
+  // can't change primary once its set for now
+  if (primaryKey && !oldTable && !isJunction) {
+    schema.increments(primaryKey).primary()
+  } else if (!oldTable && isJunction) {
+    schema.primary(metaCols.map(col => col.name))
+  }
+
+
+  // check if any columns need added
+  const foreignKeys = Object.values(table.schema).map(col => col.foreignKey)
+  for (let [key, column] of Object.entries(table.schema)) {
+    // skip things that are already correct
+    const oldColumn = oldTable ? oldTable.schema[key] : null
+    if ((oldColumn && oldColumn.type === column.type) || (primaryKey === key && !isJunction)) {
+      continue
+    }
+    switch (column.type) {
+      case FieldTypes.STRING: case FieldTypes.OPTIONS: case FieldTypes.LONGFORM:
+        schema.string(key)
+        break
+      case FieldTypes.NUMBER:
+        // if meta is specified then this is a junction table entry
+        if (column.meta && column.meta.toKey && column.meta.toTable) {
+          const { toKey, toTable } = column.meta
+          schema.integer(key).unsigned()
+          schema.foreign(key).references(`${toTable}.${toKey}`)
+        } else if (foreignKeys.indexOf(key) === -1) {
+          schema.float(key)
+        }
+        break
+      case FieldTypes.BOOLEAN:
+        schema.boolean(key)
+        break
+      case FieldTypes.DATETIME:
+        schema.datetime(key)
+        break
+      case FieldTypes.ARRAY:
+        schema.json(key)
+        break
+      case FieldTypes.LINK:
+        // this side of the relationship doesn't need any SQL work
+        if (
+          column.relationshipType !== RelationshipTypes.MANY_TO_ONE &&
+          column.relationshipType !== RelationshipTypes.MANY_TO_MANY
+        ) {
+          if (!column.foreignKey || !column.tableId) {
+            throw "Invalid relationship schema"
+          }
+          const { tableName } = breakExternalTableId(column.tableId)
+          // @ts-ignore
+          const relatedTable = tables[tableName]
+          if (!relatedTable) {
+            throw "Referenced table doesn't exist"
+          }
+          schema.integer(column.foreignKey).unsigned()
+          schema.foreign(column.foreignKey).references(`${tableName}.${relatedTable.primary[0]}`)
+        }
+        break
+    }
+  }
+
+  // need to check if any columns have been deleted
+  if (oldTable) {
+    const deletedColumns = Object.entries(oldTable.schema)
+      .filter(([key, schema]) => schema.type !== FieldTypes.LINK && table.schema[key] == null)
+      .map(([key]) => key)
+    deletedColumns.forEach(key => {
+      if (oldTable.constrained && oldTable.constrained.indexOf(key) !== -1) {
+        schema.dropForeign(key)
+      }
+      schema.dropColumn(key)
+    })
+  }
+
+  return schema
+}
+
+function buildCreateTable(
+  knex: Knex,
+  table: Table,
+  tables: Record<string, Table>,
+): SchemaBuilder {
+  return knex.schema.createTable(table.name, schema => {
+    generateSchema(schema, table, tables)
+  })
+}
+
+function buildUpdateTable(
+  knex: Knex,
+  table: Table,
+  tables: Record<string, Table>,
+  oldTable: Table,
+): SchemaBuilder {
+  return knex.schema.alterTable(table.name, schema => {
+    generateSchema(schema, table, tables, oldTable)
+  })
+}
+
+function buildDeleteTable(
+  knex: Knex,
+  table: Table,
+): SchemaBuilder {
+  return knex.schema.dropTable(table.name)
+}
+
+class SqlTableQueryBuilder {
+  private readonly sqlClient: string
+
+  // pass through client to get flavour of SQL
+  constructor(client: string) {
+    this.sqlClient = client
+  }
+
+  getSqlClient(): string {
+    return this.sqlClient
+  }
+
+  /**
+   * @param json the input JSON structure from which an SQL query will be built.
+   * @return {string} the operation that was found in the JSON.
+   */
+  _operation(json: QueryJson): Operation {
+    return json.endpoint.operation
+  }
+
+  _tableQuery(json: QueryJson): any {
+    const client = knex({ client: this.sqlClient })
+    let query
+    if (!json.table || !json.meta || !json.meta.tables) {
+      throw "Cannot execute without table being specified"
+    }
+    switch (this._operation(json)) {
+      case Operation.CREATE_TABLE:
+        query = buildCreateTable(client, json.table, json.meta.tables)
+        break
+      case Operation.UPDATE_TABLE:
+        if (!json.meta || !json.meta.table) {
+          throw "Must specify old table for update"
+        }
+        query = buildUpdateTable(client, json.table, json.meta.tables, json.meta.table)
+        break
+      case Operation.DELETE_TABLE:
+        query = buildDeleteTable(client, json.table)
+        break
+      default:
+        throw "Table operation is of unknown type"
+    }
+    return query.toSQL()
+  }
+}
+
+export default SqlTableQueryBuilder
+module.exports = SqlTableQueryBuilder