@budibase/server 1.0.44 → 1.0.46-alpha.2

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.44",
+  "version": "1.0.46-alpha.2",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.44",
-    "@budibase/client": "^1.0.44",
-    "@budibase/string-templates": "^1.0.44",
+    "@budibase/backend-core": "^1.0.46-alpha.2",
+    "@budibase/client": "^1.0.46-alpha.2",
+    "@budibase/string-templates": "^1.0.46-alpha.2",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -165,5 +165,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "2eb6bef9f5255148c820c0b3b5d8d92db7c86a50"
+  "gitHead": "74c2b625437c6cace5a8547347470b8343c8fe4f"
 }

package/src/api/controllers/row/utils.js

@@ -61,12 +61,17 @@ exports.validate = async ({ appId, tableId, row, table }) => {
     let res
 
     // Validate.js doesn't seem to handle array
-    if (type === FieldTypes.ARRAY && row[fieldName] && row[fieldName].length) {
-      row[fieldName].map(val => {
-        if (!constraints.inclusion.includes(val)) {
-          errors[fieldName] = "Field not in list"
-        }
-      })
+    if (type === FieldTypes.ARRAY && row[fieldName]) {
+      if (row[fieldName].length) {
+        row[fieldName].map(val => {
+          if (!constraints.inclusion.includes(val)) {
+            errors[fieldName] = "Field not in list"
+          }
+        })
+      } else if (constraints.presence && row[fieldName].length === 0) {
+        // non required MultiSelect creates an empty array, which should not throw errors
+        errors[fieldName] = [`${fieldName} is required`]
+      }
     } else if (type === FieldTypes.JSON && typeof row[fieldName] === "string") {
       // this should only happen if there is an error
       try {

package/src/api/controllers/table/index.js

@@ -2,7 +2,7 @@ const CouchDB = require("../../../db")
 const internal = require("./internal")
 const external = require("./external")
 const csvParser = require("../../../utilities/csvParser")
-const { isExternalTable } = require("../../../integrations/utils")
+const { isExternalTable, isSQL } = require("../../../integrations/utils")
 const {
   getTableParams,
   getDatasourceParams,
@@ -32,8 +32,8 @@ exports.fetch = async function (ctx) {
     })
   )
 
-  const internal = internalTables.rows.map(row => ({
-    ...row.doc,
+  const internal = internalTables.rows.map(tableDoc => ({
+    ...tableDoc.doc,
     type: "internal",
     sourceId: BudibaseInternalDB._id,
   }))
@@ -44,12 +44,18 @@ exports.fetch = async function (ctx) {
     })
   )
 
-  const external = externalTables.rows.flatMap(row => {
-    return Object.values(row.doc.entities || {}).map(entity => ({
-      ...entity,
-      type: "external",
-      sourceId: row.doc._id,
-    }))
+  const external = externalTables.rows.flatMap(tableDoc => {
+    let entities = tableDoc.doc.entities
+    if (entities) {
+      return Object.values(entities).map(entity => ({
+        ...entity,
+        type: "external",
+        sourceId: tableDoc.doc._id,
+        sql: isSQL(tableDoc.doc),
+      }))
+    } else {
+      return []
+    }
   })
 
   ctx.body = [...internal, ...external]

package/src/api/controllers/table/utils.js

@@ -12,6 +12,7 @@ const { USERS_TABLE_SCHEMA, SwitchableTypes } = require("../../../constants")
 const {
   isExternalTable,
   breakExternalTableId,
+  isSQL,
 } = require("../../../integrations/utils")
 const { getViews, saveView } = require("../view/utils")
 const viewTemplate = require("../view/viewBuilder")
@@ -242,7 +243,9 @@ exports.getTable = async (appId, tableId) => {
   const db = new CouchDB(appId)
   if (isExternalTable(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
-    return exports.getExternalTable(appId, datasourceId, tableName)
+    const datasource = await db.get(datasourceId)
+    const table = await exports.getExternalTable(appId, datasourceId, tableName)
+    return { ...table, sql: isSQL(datasource) }
   } else {
     return db.get(tableId)
   }

package/src/integrations/base/sql.ts

@@ -72,16 +72,22 @@ class InternalBuilder {
 
   // right now we only do filters on the specific table being queried
   addFilters(
-    tableName: string,
     query: KnexQuery,
-    filters: SearchFilters | undefined
+    filters: SearchFilters | undefined,
+    opts: { relationship?: boolean; tableName?: string }
   ): KnexQuery {
     function iterate(
       structure: { [key: string]: any },
       fn: (key: string, value: any) => void
     ) {
       for (let [key, value] of Object.entries(structure)) {
-        fn(`${tableName}.${key}`, value)
+        const isRelationshipField = key.includes(".")
+        if (!opts.relationship && !isRelationshipField) {
+          fn(`${opts.tableName}.${key}`, value)
+        }
+        if (opts.relationship && isRelationshipField) {
+          fn(key, value)
+        }
       }
     }
     if (!filters) {
@@ -272,7 +278,7 @@ class InternalBuilder {
     if (foundOffset) {
       query = query.offset(foundOffset)
     }
-    query = this.addFilters(tableName, query, filters)
+    query = this.addFilters(query, filters, { tableName })
     // add sorting to pre-query
     query = this.addSorting(query, json)
     // @ts-ignore
@@ -285,20 +291,21 @@ class InternalBuilder {
       preQuery = this.addSorting(preQuery, json)
     }
     // handle joins
-    return this.addRelationships(
+    query = this.addRelationships(
       knex,
       preQuery,
       selectStatement,
       tableName,
       relationships
     )
+    return this.addFilters(query, filters, { relationship: true })
   }
 
   update(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, body, filters } = json
     let query: KnexQuery = knex(endpoint.entityId)
     const parsedBody = parseBody(body)
-    query = this.addFilters(endpoint.entityId, query, filters)
+    query = this.addFilters(query, filters, { tableName: endpoint.entityId })
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.update(parsedBody)
@@ -310,7 +317,7 @@ class InternalBuilder {
   delete(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, filters } = json
     let query: KnexQuery = knex(endpoint.entityId)
-    query = this.addFilters(endpoint.entityId, query, filters)
+    query = this.addFilters(query, filters, { tableName: endpoint.entityId })
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.delete()

package/src/integrations/tests/sql.spec.js

@@ -119,6 +119,22 @@ describe("SQL query builder", () => {
     })
   })
 
+  it("should allow filtering on a related field", () => {
+    const query = sql._query(generateReadJson({
+      filters: {
+        equal: {
+          age: 10,
+          "task.name": "task 1",
+        },
+      },
+    }))
+    // order of bindings changes because relationship filters occur outside inner query
+    expect(query).toEqual({
+      bindings: [10, limit, "task 1"],
+      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" = $1 limit $2) as "${TABLE_NAME}" where "task"."name" = $3`
+    })
+  })
+
   it("should test an create statement", () => {
     const query = sql._query(generateCreateJson(TABLE_NAME, {
       name: "Michael",

package/src/integrations/utils.ts

@@ -1,6 +1,5 @@
-import { SqlQuery } from "../definitions/datasource"
+import { SourceNames, SqlQuery } from "../definitions/datasource"
 import { Datasource, Table } from "../definitions/common"
-import { SourceNames } from "../definitions/datasource"
 import { DocumentTypes, SEPARATOR } from "../db/utils"
 import { FieldTypes, BuildSchemaErrors, InvalidColumns } from "../constants"
 
@@ -127,7 +126,12 @@ export function isSQL(datasource: Datasource): boolean {
   if (!datasource || !datasource.source) {
     return false
   }
-  const SQL = [SourceNames.POSTGRES, SourceNames.SQL_SERVER, SourceNames.MYSQL]
+  const SQL = [
+    SourceNames.POSTGRES,
+    SourceNames.SQL_SERVER,
+    SourceNames.MYSQL,
+    SourceNames.ORACLE,
+  ]
   return SQL.indexOf(datasource.source) !== -1
 }
 
@@ -139,11 +143,46 @@ export function isIsoDateString(str: string) {
   return d.toISOString() === str
 }
 
-// add the existing relationships from the entities if they exist, to prevent them from being overridden
+/**
+ * This function will determine whether a column is a relationship and whether it
+ * is currently valid. The reason for the validity check is that tables can be deleted
+ * outside of Budibase control and if this is the case it will break Budibase relationships.
+ * The tableIds is a list passed down from the main finalise tables function, which is
+ * based on the tables that have just been fetched. This will only really be used on subsequent
+ * fetches to the first one - if the user is periodically refreshing Budibase knowledge of tables.
+ * @param column The column to check, to see if it is a valid relationship.
+ * @param tableIds The IDs of the tables which currently exist.
+ */
+function shouldCopyRelationship(column: { type: string, tableId?: string }, tableIds: [string]) {
+  return column.type === FieldTypes.LINK && column.tableId && tableIds.includes(column.tableId)
+}
+
+/**
+ * Similar function to the shouldCopyRelationship function, but instead this looks for options and boolean
+ * types. It is possible to switch a string -> options and a number -> boolean (and vice versus) need to make
+ * sure that these get copied over when tables are fetched. Also checks whether they are still valid, if a
+ * column has changed type in the external database then copying it over may not be possible.
+ * @param column The column to check for options or boolean type.
+ * @param fetchedColumn The fetched column to check for the type in the external database.
+ */
+function shouldCopySpecialColumn(column: { type: string }, fetchedColumn: { type: string } | undefined) {
+  return column.type === FieldTypes.OPTIONS ||
+    ((!fetchedColumn || fetchedColumn.type === FieldTypes.NUMBER) && column.type === FieldTypes.BOOLEAN)
+}
+
+/**
+ * Looks for columns which need to be copied over into the new table definitions, like relationships
+ * and options types.
+ * @param tableName The name of the table which is being checked.
+ * @param table The specific table which is being checked.
+ * @param entities All the tables that existed before - the old table definitions.
+ * @param tableIds The IDs of the tables which exist now, to check if anything has been removed.
+ */
 function copyExistingPropsOver(
   tableName: string,
   table: Table,
-  entities: { [key: string]: any }
+  entities: { [key: string]: any },
+  tableIds: [string]
 ) {
   if (entities && entities[tableName]) {
     if (entities[tableName].primaryDisplay) {
@@ -154,11 +193,10 @@ function copyExistingPropsOver(
       if (!existingTableSchema.hasOwnProperty(key)) {
         continue
       }
+      const column = existingTableSchema[key]
       if (
-        existingTableSchema[key].type === FieldTypes.LINK ||
-        existingTableSchema[key].type === FieldTypes.OPTIONS ||
-        ((!table.schema[key] || table.schema[key].type === FieldTypes.NUMBER) &&
-          existingTableSchema[key].type === FieldTypes.BOOLEAN)
+        shouldCopyRelationship(column, tableIds) ||
+        shouldCopySpecialColumn(column, table.schema[key])
       ) {
         table.schema[key] = existingTableSchema[key]
       }
@@ -167,6 +205,13 @@ function copyExistingPropsOver(
   return table
 }
 
+/**
+ * Look through the final table definitions to see if anything needs to be
+ * copied over from the old and if any errors have occurred mark them so
+ * that the user can be made aware.
+ * @param tables The list of tables that have been retrieved from the external database.
+ * @param entities The old list of tables, if there was any to look for definitions in.
+ */
 export function finaliseExternalTables(
   tables: { [key: string]: any },
   entities: { [key: string]: any }
@@ -174,6 +219,8 @@ export function finaliseExternalTables(
   const invalidColumns = Object.values(InvalidColumns)
   let finalTables: { [key: string]: any } = {}
   const errors: { [key: string]: string } = {}
+  // @ts-ignore
+  const tableIds: [string] = Object.values(tables).map(table => table._id)
   for (let [name, table] of Object.entries(tables)) {
     const schemaFields = Object.keys(table.schema)
     // make sure every table has a key
@@ -185,7 +232,7 @@ export function finaliseExternalTables(
       continue
     }
     // make sure all previous props have been added back
-    finalTables[name] = copyExistingPropsOver(name, table, entities)
+    finalTables[name] = copyExistingPropsOver(name, table, entities, tableIds)
   }
   // sort the tables by name
   finalTables = Object.entries(finalTables)

package/src/middleware/currentapp.js

@@ -47,6 +47,15 @@ module.exports = async (ctx, next) => {
     (!ctx.user || !ctx.user.builder || !ctx.user.builder.global)
   ) {
     clearCookie(ctx, Cookies.CurrentApp)
+    // have to set the return url on the server side as client side is not available
+    setCookie(ctx, ctx.url, Cookies.RETURN_URL, {
+      // don't sign so the browser can easily read
+      sign: false,
+      // use the request domain to match how ui handles the return url cookie.
+      // it's important we don't use the shared domain here as the builder
+      // can't delete from it without awareness of the domain.
+      requestDomain: true,
+    })
     return ctx.redirect("/")
   }