@budibase/server 1.0.49-alpha.9 → 1.0.49

package/src/integrations/base/sql.ts

@@ -72,22 +72,16 @@ class InternalBuilder {
 
   // right now we only do filters on the specific table being queried
   addFilters(
+    tableName: string,
     query: KnexQuery,
-    filters: SearchFilters | undefined,
-    opts: { relationship?: boolean; tableName?: string }
+    filters: SearchFilters | undefined
   ): KnexQuery {
     function iterate(
       structure: { [key: string]: any },
       fn: (key: string, value: any) => void
     ) {
       for (let [key, value] of Object.entries(structure)) {
-        const isRelationshipField = key.includes(".")
-        if (!opts.relationship && !isRelationshipField) {
-          fn(`${opts.tableName}.${key}`, value)
-        }
-        if (opts.relationship && isRelationshipField) {
-          fn(key, value)
-        }
+        fn(`${tableName}.${key}`, value)
       }
     }
     if (!filters) {
@@ -191,70 +185,29 @@ class InternalBuilder {
     if (!relationships) {
       return query
     }
-    const tableSets: Record<string, [any]> = {}
-    // aggregate into table sets (all the same to tables)
     for (let relationship of relationships) {
-      const keyObj: { toTable: string; throughTable: string | undefined } = {
-        toTable: relationship.tableName,
-        throughTable: undefined,
-      }
-      if (relationship.through) {
-        keyObj.throughTable = relationship.through
-      }
-      const key = JSON.stringify(keyObj)
-      if (tableSets[key]) {
-        tableSets[key].push(relationship)
-      } else {
-        tableSets[key] = [relationship]
-      }
-    }
-    for (let [key, relationships] of Object.entries(tableSets)) {
-      const { toTable, throughTable } = JSON.parse(key)
-      if (!throughTable) {
+      const from = relationship.from,
+        to = relationship.to,
+        toTable = relationship.tableName
+      if (!relationship.through) {
         // @ts-ignore
-        query = query.join(
+        query = query.leftJoin(
           toTable,
-          function () {
-            for (let relationship of relationships) {
-              const from = relationship.from,
-                to = relationship.to
-              // @ts-ignore
-              this.orOn(`${fromTable}.${from}`, "=", `${toTable}.${to}`)
-            }
-          },
-          "left"
+          `${fromTable}.${from}`,
+          `${toTable}.${to}`
         )
       } else {
+        const throughTable = relationship.through
+        const fromPrimary = relationship.fromPrimary
+        const toPrimary = relationship.toPrimary
         query = query
           // @ts-ignore
-          .join(
+          .leftJoin(
             throughTable,
-            function () {
-              for (let relationship of relationships) {
-                const fromPrimary = relationship.fromPrimary
-                const from = relationship.from
-                // @ts-ignore
-                this.orOn(
-                  `${fromTable}.${fromPrimary}`,
-                  "=",
-                  `${throughTable}.${from}`
-                )
-              }
-            },
-            "left"
-          )
-          .join(
-            toTable,
-            function () {
-              for (let relationship of relationships) {
-                const toPrimary = relationship.toPrimary
-                const to = relationship.to
-                // @ts-ignore
-                this.orOn(`${toTable}.${toPrimary}`, `${throughTable}.${to}`)
-              }
-            },
-            "left"
+            `${fromTable}.${fromPrimary}`,
+            `${throughTable}.${from}`
           )
+          .leftJoin(toTable, `${toTable}.${toPrimary}`, `${throughTable}.${to}`)
       }
     }
     return query.limit(BASE_LIMIT)
@@ -319,7 +272,7 @@ class InternalBuilder {
     if (foundOffset) {
       query = query.offset(foundOffset)
     }
-    query = this.addFilters(query, filters, { tableName })
+    query = this.addFilters(tableName, query, filters)
     // add sorting to pre-query
     query = this.addSorting(query, json)
     // @ts-ignore
@@ -332,21 +285,20 @@ class InternalBuilder {
       preQuery = this.addSorting(preQuery, json)
     }
     // handle joins
-    query = this.addRelationships(
+    return this.addRelationships(
       knex,
       preQuery,
       selectStatement,
       tableName,
       relationships
     )
-    return this.addFilters(query, filters, { relationship: true })
   }
 
   update(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, body, filters } = json
     let query: KnexQuery = knex(endpoint.entityId)
     const parsedBody = parseBody(body)
-    query = this.addFilters(query, filters, { tableName: endpoint.entityId })
+    query = this.addFilters(endpoint.entityId, query, filters)
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.update(parsedBody)
@@ -358,7 +310,7 @@ class InternalBuilder {
   delete(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, filters } = json
     let query: KnexQuery = knex(endpoint.entityId)
-    query = this.addFilters(query, filters, { tableName: endpoint.entityId })
+    query = this.addFilters(endpoint.entityId, query, filters)
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.delete()

package/src/integrations/index.ts

@@ -9,7 +9,6 @@ const airtable = require("./airtable")
 const mysql = require("./mysql")
 const arangodb = require("./arangodb")
 const rest = require("./rest")
-const googlesheets = require("./googlesheets")
 const { SourceNames } = require("../definitions/datasource")
 
 const DEFINITIONS = {
@@ -24,7 +23,6 @@ const DEFINITIONS = {
   [SourceNames.MYSQL]: mysql.schema,
   [SourceNames.ARANGODB]: arangodb.schema,
   [SourceNames.REST]: rest.schema,
-  [SourceNames.GOOGLE_SHEETS]: googlesheets.schema,
 }
 
 const INTEGRATIONS = {
@@ -39,7 +37,6 @@ const INTEGRATIONS = {
   [SourceNames.MYSQL]: mysql.integration,
   [SourceNames.ARANGODB]: arangodb.integration,
   [SourceNames.REST]: rest.integration,
-  [SourceNames.GOOGLE_SHEETS]: googlesheets.integration,
 }
 
 // optionally add oracle integration if the oracle binary can be installed

package/src/integrations/tests/sql.spec.js

@@ -119,22 +119,6 @@ describe("SQL query builder", () => {
     })
   })
 
-  it("should allow filtering on a related field", () => {
-    const query = sql._query(generateReadJson({
-      filters: {
-        equal: {
-          age: 10,
-          "task.name": "task 1",
-        },
-      },
-    }))
-    // order of bindings changes because relationship filters occur outside inner query
-    expect(query).toEqual({
-      bindings: [10, limit, "task 1"],
-      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" = $1 limit $2) as "${TABLE_NAME}" where "task"."name" = $3`
-    })
-  })
-
   it("should test an create statement", () => {
     const query = sql._query(generateCreateJson(TABLE_NAME, {
       name: "Michael",

package/src/integrations/utils.ts

@@ -1,5 +1,6 @@
-import { SourceNames, SqlQuery } from "../definitions/datasource"
+import { SqlQuery } from "../definitions/datasource"
 import { Datasource, Table } from "../definitions/common"
+import { SourceNames } from "../definitions/datasource"
 import { DocumentTypes, SEPARATOR } from "../db/utils"
 import { FieldTypes, BuildSchemaErrors, InvalidColumns } from "../constants"
 
@@ -52,10 +53,7 @@ export function buildExternalTableId(datasourceId: string, tableName: string) {
   return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
 }
 
-export function breakExternalTableId(tableId: string | undefined) {
-  if (!tableId) {
-    return {}
-  }
+export function breakExternalTableId(tableId: string) {
   const parts = tableId.split(DOUBLE_SEPARATOR)
   let tableName = parts.pop()
   // if they need joined
@@ -129,12 +127,7 @@ export function isSQL(datasource: Datasource): boolean {
   if (!datasource || !datasource.source) {
     return false
   }
-  const SQL = [
-    SourceNames.POSTGRES,
-    SourceNames.SQL_SERVER,
-    SourceNames.MYSQL,
-    SourceNames.ORACLE,
-  ]
+  const SQL = [SourceNames.POSTGRES, SourceNames.SQL_SERVER, SourceNames.MYSQL]
   return SQL.indexOf(datasource.source) !== -1
 }
 
@@ -146,60 +139,11 @@ export function isIsoDateString(str: string) {
   return d.toISOString() === str
 }
 
-/**
- * This function will determine whether a column is a relationship and whether it
- * is currently valid. The reason for the validity check is that tables can be deleted
- * outside of Budibase control and if this is the case it will break Budibase relationships.
- * The tableIds is a list passed down from the main finalise tables function, which is
- * based on the tables that have just been fetched. This will only really be used on subsequent
- * fetches to the first one - if the user is periodically refreshing Budibase knowledge of tables.
- * @param column The column to check, to see if it is a valid relationship.
- * @param tableIds The IDs of the tables which currently exist.
- */
-function shouldCopyRelationship(
-  column: { type: string; tableId?: string },
-  tableIds: [string]
-) {
-  return (
-    column.type === FieldTypes.LINK &&
-    column.tableId &&
-    tableIds.includes(column.tableId)
-  )
-}
-
-/**
- * Similar function to the shouldCopyRelationship function, but instead this looks for options and boolean
- * types. It is possible to switch a string -> options and a number -> boolean (and vice versus) need to make
- * sure that these get copied over when tables are fetched. Also checks whether they are still valid, if a
- * column has changed type in the external database then copying it over may not be possible.
- * @param column The column to check for options or boolean type.
- * @param fetchedColumn The fetched column to check for the type in the external database.
- */
-function shouldCopySpecialColumn(
-  column: { type: string },
-  fetchedColumn: { type: string } | undefined
-) {
-  return (
-    column.type === FieldTypes.OPTIONS ||
-    column.type === FieldTypes.ARRAY ||
-    ((!fetchedColumn || fetchedColumn.type === FieldTypes.NUMBER) &&
-      column.type === FieldTypes.BOOLEAN)
-  )
-}
-
-/**
- * Looks for columns which need to be copied over into the new table definitions, like relationships
- * and options types.
- * @param tableName The name of the table which is being checked.
- * @param table The specific table which is being checked.
- * @param entities All the tables that existed before - the old table definitions.
- * @param tableIds The IDs of the tables which exist now, to check if anything has been removed.
- */
+// add the existing relationships from the entities if they exist, to prevent them from being overridden
 function copyExistingPropsOver(
   tableName: string,
   table: Table,
-  entities: { [key: string]: any },
-  tableIds: [string]
+  entities: { [key: string]: any }
 ) {
   if (entities && entities[tableName]) {
     if (entities[tableName].primaryDisplay) {
@@ -210,10 +154,12 @@ function copyExistingPropsOver(
       if (!existingTableSchema.hasOwnProperty(key)) {
         continue
       }
-      const column = existingTableSchema[key]
       if (
-        shouldCopyRelationship(column, tableIds) ||
-        shouldCopySpecialColumn(column, table.schema[key])
+        existingTableSchema[key].type === FieldTypes.LINK ||
+        existingTableSchema[key].type === FieldTypes.OPTIONS ||
+        existingTableSchema[key].type === FieldTypes.ARRAY ||
+        ((!table.schema[key] || table.schema[key].type === FieldTypes.NUMBER) &&
+          existingTableSchema[key].type === FieldTypes.BOOLEAN)
       ) {
         table.schema[key] = existingTableSchema[key]
       }
@@ -222,13 +168,6 @@ function copyExistingPropsOver(
   return table
 }
 
-/**
- * Look through the final table definitions to see if anything needs to be
- * copied over from the old and if any errors have occurred mark them so
- * that the user can be made aware.
- * @param tables The list of tables that have been retrieved from the external database.
- * @param entities The old list of tables, if there was any to look for definitions in.
- */
 export function finaliseExternalTables(
   tables: { [key: string]: any },
   entities: { [key: string]: any }
@@ -236,8 +175,6 @@ export function finaliseExternalTables(
   const invalidColumns = Object.values(InvalidColumns)
   let finalTables: { [key: string]: any } = {}
   const errors: { [key: string]: string } = {}
-  // @ts-ignore
-  const tableIds: [string] = Object.values(tables).map(table => table._id)
   for (let [name, table] of Object.entries(tables)) {
     const schemaFields = Object.keys(table.schema)
     // make sure every table has a key
@@ -249,7 +186,7 @@ export function finaliseExternalTables(
       continue
     }
     // make sure all previous props have been added back
-    finalTables[name] = copyExistingPropsOver(name, table, entities, tableIds)
+    finalTables[name] = copyExistingPropsOver(name, table, entities)
   }
   // sort the tables by name
   finalTables = Object.entries(finalTables)

package/src/middleware/authorized.js

@@ -9,60 +9,11 @@ const {
 } = require("@budibase/backend-core/permissions")
 const builderMiddleware = require("./builder")
 const { isWebhookEndpoint } = require("./utils")
-const { buildCsrfMiddleware } = require("@budibase/backend-core/auth")
-const { getAppId } = require("@budibase/backend-core/context")
 
 function hasResource(ctx) {
   return ctx.resourceId != null
 }
 
-const csrf = buildCsrfMiddleware()
-
-/**
- * Apply authorization to the requested resource:
- * - If this is a builder resource the user must be a builder.
- * - Builders can access all resources.
- * - Otherwise the user must have the required role.
- */
-const checkAuthorized = async (ctx, resourceRoles, permType, permLevel) => {
-  // check if this is a builder api and the user is not a builder
-  const isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global
-  const isBuilderApi = permType === PermissionTypes.BUILDER
-  if (isBuilderApi && !isBuilder) {
-    return ctx.throw(403, "Not Authorized")
-  }
-
-  // check for resource authorization
-  if (!isBuilder) {
-    await checkAuthorizedResource(ctx, resourceRoles, permType, permLevel)
-  }
-}
-
-const checkAuthorizedResource = async (
-  ctx,
-  resourceRoles,
-  permType,
-  permLevel
-) => {
-  // get the user's roles
-  const roleId = ctx.roleId || BUILTIN_ROLE_IDS.PUBLIC
-  const userRoles = await getUserRoleHierarchy(roleId, {
-    idOnly: false,
-  })
-  const permError = "User does not have permission"
-  // check if the user has the required role
-  if (resourceRoles.length > 0) {
-    // deny access if the user doesn't have the required resource role
-    const found = userRoles.find(role => resourceRoles.indexOf(role._id) !== -1)
-    if (!found) {
-      ctx.throw(403, permError)
-    }
-    // fallback to the base permissions when no resource roles are found
-  } else if (!doesHaveBasePermission(permType, permLevel, userRoles)) {
-    ctx.throw(403, permError)
-  }
-}
-
 module.exports =
   (permType, permLevel = null) =>
   async (ctx, next) => {
@@ -80,27 +31,40 @@ module.exports =
     // to find API endpoints which are builder focused
     await builderMiddleware(ctx, permType)
 
-    // get the resource roles
-    let resourceRoles = []
-    const appId = getAppId()
-    if (appId && hasResource(ctx)) {
-      resourceRoles = await getRequiredResourceRole(permLevel, ctx)
-    }
-
-    // if the resource is public, proceed
-    const isPublicResource = resourceRoles.includes(BUILTIN_ROLE_IDS.PUBLIC)
-    if (isPublicResource) {
+    const isAuthed = ctx.isAuthenticated
+    // builders for now have permission to do anything
+    let isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global
+    const isBuilderApi = permType === PermissionTypes.BUILDER
+    if (isBuilder) {
       return next()
+    } else if (isBuilderApi && !isBuilder) {
+      return ctx.throw(403, "Not Authorized")
     }
 
-    // check authenticated
-    if (!ctx.isAuthenticated) {
-      return ctx.throw(403, "Session not authenticated")
+    // need to check this first, in-case public access, don't check authed until last
+    const roleId = ctx.roleId || BUILTIN_ROLE_IDS.PUBLIC
+    const hierarchy = await getUserRoleHierarchy(ctx.appId, roleId, {
+      idOnly: false,
+    })
+    const permError = "User does not have permission"
+    let possibleRoleIds = []
+    if (hasResource(ctx)) {
+      possibleRoleIds = await getRequiredResourceRole(ctx.appId, permLevel, ctx)
+    }
+    // check if we found a role, if not fallback to base permissions
+    if (possibleRoleIds.length > 0) {
+      const found = hierarchy.find(
+        role => possibleRoleIds.indexOf(role._id) !== -1
+      )
+      return found ? next() : ctx.throw(403, permError)
+    } else if (!doesHaveBasePermission(permType, permLevel, hierarchy)) {
+      ctx.throw(403, permError)
     }
 
-    // check authorized
-    await checkAuthorized(ctx, resourceRoles, permType, permLevel)
+    // if they are not authed, then anything using the authorized middleware will fail
+    if (!isAuthed) {
+      ctx.throw(403, "Session not authenticated")
+    }
 
-    // csrf protection
-    return csrf(ctx, next)
+    return next()
   }

package/src/middleware/builder.js

@@ -5,7 +5,7 @@ const {
   checkDebounce,
   setDebounce,
 } = require("../utilities/redis")
-const { getDB } = require("@budibase/backend-core/db")
+const CouchDB = require("../db")
 const { DocumentTypes } = require("../db/utils")
 const { PermissionTypes } = require("@budibase/backend-core/permissions")
 const { app: appCache } = require("@budibase/backend-core/cache")
@@ -48,7 +48,7 @@ async function updateAppUpdatedAt(ctx) {
   if (ctx.method === "GET" || (await checkDebounce(appId))) {
     return
   }
-  const db = getDB(appId)
+  const db = new CouchDB(appId)
   const metadata = await db.get(DocumentTypes.APP_METADATA)
   metadata.updatedAt = new Date().toISOString()
   const response = await db.put(metadata)

package/src/middleware/currentapp.js

@@ -11,9 +11,9 @@ const { generateUserMetadataID, isDevAppID } = require("../db/utils")
 const { dbExists } = require("@budibase/backend-core/db")
 const { isUserInAppTenant } = require("@budibase/backend-core/tenancy")
 const { getCachedSelf } = require("../utilities/global")
+const CouchDB = require("../db")
 const env = require("../environment")
 const { isWebhookEndpoint } = require("./utils")
-const { doInAppContext } = require("@budibase/backend-core/context")
 
 module.exports = async (ctx, next) => {
   // try to get the appID from the request
@@ -31,7 +31,7 @@ module.exports = async (ctx, next) => {
   // check the app exists referenced in cookie
   if (appCookie) {
     const appId = appCookie.appId
-    const exists = await dbExists(appId)
+    const exists = await dbExists(CouchDB, appId)
     if (!exists) {
       clearCookie(ctx, Cookies.CurrentApp)
       return next()
@@ -41,15 +41,13 @@ module.exports = async (ctx, next) => {
   }
 
   // deny access to application preview
-  if (!env.isTest()) {
-    if (
-      isDevAppID(requestAppId) &&
-      !isWebhookEndpoint(ctx) &&
-      (!ctx.user || !ctx.user.builder || !ctx.user.builder.global)
-    ) {
-      clearCookie(ctx, Cookies.CurrentApp)
-      return ctx.redirect("/")
-    }
+  if (
+    isDevAppID(requestAppId) &&
+    !isWebhookEndpoint(ctx) &&
+    (!ctx.user || !ctx.user.builder || !ctx.user.builder.global)
+  ) {
+    clearCookie(ctx, Cookies.CurrentApp)
+    return ctx.redirect("/")
   }
 
   let appId,
@@ -70,46 +68,44 @@ module.exports = async (ctx, next) => {
     return next()
   }
 
-  return doInAppContext(appId, async () => {
-    let noCookieSet = false
-    // if the user not in the right tenant then make sure they have no permissions
-    // need to judge this only based on the request app ID,
-    if (
-      env.MULTI_TENANCY &&
-      ctx.user &&
-      requestAppId &&
-      !isUserInAppTenant(requestAppId)
-    ) {
-      // don't error, simply remove the users rights (they are a public user)
-      delete ctx.user.builder
-      delete ctx.user.admin
-      delete ctx.user.roles
-      roleId = BUILTIN_ROLE_IDS.PUBLIC
-      noCookieSet = true
-    }
+  let noCookieSet = false
+  // if the user not in the right tenant then make sure they have no permissions
+  // need to judge this only based on the request app ID,
+  if (
+    env.MULTI_TENANCY &&
+    ctx.user &&
+    requestAppId &&
+    !isUserInAppTenant(requestAppId)
+  ) {
+    // don't error, simply remove the users rights (they are a public user)
+    delete ctx.user.builder
+    delete ctx.user.admin
+    delete ctx.user.roles
+    roleId = BUILTIN_ROLE_IDS.PUBLIC
+    noCookieSet = true
+  }
 
-    ctx.appId = appId
-    if (roleId) {
-      ctx.roleId = roleId
-      const userId = ctx.user ? generateUserMetadataID(ctx.user._id) : null
-      ctx.user = {
-        ...ctx.user,
-        // override userID with metadata one
-        _id: userId,
-        userId,
-        roleId,
-        role: await getRole(roleId),
-      }
-    }
-    if (
-      (requestAppId !== appId ||
-        appCookie == null ||
-        appCookie.appId !== requestAppId) &&
-      !noCookieSet
-    ) {
-      setCookie(ctx, { appId }, Cookies.CurrentApp)
+  ctx.appId = appId
+  if (roleId) {
+    ctx.roleId = roleId
+    const userId = ctx.user ? generateUserMetadataID(ctx.user._id) : null
+    ctx.user = {
+      ...ctx.user,
+      // override userID with metadata one
+      _id: userId,
+      userId,
+      roleId,
+      role: await getRole(appId, roleId),
     }
+  }
+  if (
+    (requestAppId !== appId ||
+      appCookie == null ||
+      appCookie.appId !== requestAppId) &&
+    !noCookieSet
+  ) {
+    setCookie(ctx, { appId }, Cookies.CurrentApp)
+  }
 
-    return next()
-  })
+  return next()
 }

package/src/middleware/tests/authorized.spec.js

@@ -11,33 +11,26 @@ const authorizedMiddleware = require("../authorized")
 const env = require("../../environment")
 const { PermissionTypes, PermissionLevels } = require("@budibase/backend-core/permissions")
 require("@budibase/backend-core").init(require("../../db"))
-const { doInAppContext } = require("@budibase/backend-core/context")
-
-const APP_ID = ""
 
 class TestConfiguration {
   constructor(role) {
     this.middleware = authorizedMiddleware(role)
     this.next = jest.fn()
     this.throw = jest.fn()
-    this.headers = {}
     this.ctx = {
       headers: {},
       request: {
         url: ""
       },
-      appId: APP_ID,
+      appId: "",
       auth: {},
       next: this.next,
-      throw: this.throw,
-      get: (name) => this.headers[name],
+      throw: this.throw
     }
   }
 
   executeMiddleware() {
-    return doInAppContext(APP_ID, () => {
-      return this.middleware(this.ctx, this.next)
-    })
+    return this.middleware(this.ctx, this.next)
   }
 
   setUser(user) {
@@ -53,7 +46,7 @@ class TestConfiguration {
   }
 
   setAuthenticated(isAuthed) {
-    this.ctx.isAuthenticated = isAuthed
+    this.ctx.auth = { authenticated: isAuthed }
   }
 
   setRequestUrl(url) {
@@ -114,7 +107,7 @@ describe("Authorization middleware", () => {
       expect(config.next).toHaveBeenCalled()
     })
     
-    it("throws if the user does not have builder permissions", async () => {
+    it("throws if the user has only builder permissions", async () => {
       config.setEnvironment(false)
       config.setMiddlewareRequiredPermission(PermissionTypes.BUILDER)
       config.setUser({
@@ -140,7 +133,7 @@ describe("Authorization middleware", () => {
       expect(config.next).toHaveBeenCalled()
     })
 
-    it("throws if the user session is not authenticated", async () => {
+    it("throws if the user session is not authenticated after permission checks", async () => {
       config.setUser({
         role: {
           _id: ""