@budibase/server 1.0.49-alpha.6 → 1.0.49

package/src/db/linkedRows/linkUtils.js

@@ -1,8 +1,8 @@
+const CouchDB = require("../index")
 const Sentry = require("@sentry/node")
 const { ViewNames, getQueryIndex } = require("../utils")
 const { FieldTypes } = require("../../constants")
 const { createLinkView } = require("../views/staticViews")
-const { getAppDB } = require("@budibase/backend-core/context")
 
 /**
  * Only needed so that boolean parameters are being used for includeDocs
@@ -17,6 +17,7 @@ exports.createLinkView = createLinkView
 
 /**
  * Gets the linking documents, not the linked documents themselves.
+ * @param {string} args.appId The instance in which we are searching for linked rows.
  * @param {string} args.tableId The table which we are searching for linked rows against.
  * @param {string|null} args.fieldName The name of column/field which is being altered, only looking for
  * linking documents that are related to it. If this is not specified then the table level will be assumed.
@@ -29,8 +30,8 @@ exports.createLinkView = createLinkView
  * (if any).
  */
 exports.getLinkDocuments = async function (args) {
-  const { tableId, rowId, includeDocs } = args
-  const db = getAppDB()
+  const { appId, tableId, rowId, includeDocs } = args
+  const db = new CouchDB(appId)
   let params
   if (rowId != null) {
     params = { key: [tableId, rowId] }
@@ -67,7 +68,7 @@ exports.getLinkDocuments = async function (args) {
   } catch (err) {
     // check if the view doesn't exist, it should for all new instances
     if (err != null && err.name === "not_found") {
-      await exports.createLinkView()
+      await exports.createLinkView(appId)
       return exports.getLinkDocuments(arguments[0])
     } else {
       /* istanbul ignore next */
@@ -88,8 +89,7 @@ exports.getLinkedTableIDs = table => {
     .map(column => column.tableId)
 }
 
-exports.getLinkedTable = async (id, tables) => {
-  const db = getAppDB()
+exports.getLinkedTable = async (db, id, tables) => {
   let linkedTable = tables.find(table => table._id === id)
   if (linkedTable) {
     return linkedTable

package/src/db/tests/linkController.spec.js

@@ -20,6 +20,7 @@ describe("test the link controller", () => {
 
   function createLinkController(table, row = null, oldTable = null) {
     const linkConfig = {
+      appId: config.getAppId(),
       tableId: table._id,
       table,
     }

package/src/db/tests/linkTests.spec.js

@@ -1,8 +1,8 @@
 const TestConfig = require("../../tests/utilities/TestConfiguration")
-const { basicTable } = require("../../tests/utilities/structures")
+const { basicTable, basicLinkedRow } = require("../../tests/utilities/structures")
 const linkUtils = require("../linkedRows/linkUtils")
+const links = require("../linkedRows")
 const CouchDB = require("../index")
-const { getAppDB } = require("@budibase/backend-core/context")
 
 describe("test link functionality", () => {
   const config = new TestConfig(false)
@@ -11,18 +11,18 @@ describe("test link functionality", () => {
     let db, table
     beforeEach(async () => {
       await config.init()
-      db = getAppDB()
+      db = new CouchDB(config.getAppId())
       table = await config.createTable()
     })
 
     it("should be able to retrieve a linked table from a list", async () => {
-      const retrieved = await linkUtils.getLinkedTable(table._id, [table])
+      const retrieved = await linkUtils.getLinkedTable(db, table._id, [table])
       expect(retrieved._id).toBe(table._id)
     })
 
     it("should be able to retrieve a table from DB and update list", async () => {
       const tables = []
-      const retrieved = await linkUtils.getLinkedTable(table._id, tables)
+      const retrieved = await linkUtils.getLinkedTable(db, table._id, tables)
       expect(retrieved._id).toBe(table._id)
       expect(tables[0]).toBeDefined()
     })
@@ -51,6 +51,7 @@ describe("test link functionality", () => {
       const db = new CouchDB("test")
       await db.put({ _id: "_design/database", views: {} })
       const output = await linkUtils.getLinkDocuments({
+        appId: "test",
         tableId: "test",
         rowId: "test",
         includeDocs: false,

package/src/db/views/staticViews.js

@@ -1,4 +1,4 @@
-const { getAppDB } = require("@budibase/backend-core/context")
+const CouchDB = require("../index")
 const {
   DocumentTypes,
   SEPARATOR,
@@ -21,11 +21,12 @@ const SCREEN_PREFIX = DocumentTypes.SCREEN + SEPARATOR
 /**
  * Creates the link view for the instance, this will overwrite the existing one, but this should only
  * be called if it is found that the view does not exist.
+ * @param {string} appId The instance to which the view should be added.
  * @returns {Promise<void>} The view now exists, please note that the next view of this query will actually build it,
  * so it may be slow.
  */
-exports.createLinkView = async () => {
-  const db = getAppDB()
+exports.createLinkView = async appId => {
+  const db = new CouchDB(appId)
   const designDoc = await db.get("_design/database")
   const view = {
     map: function (doc) {
@@ -56,8 +57,8 @@ exports.createLinkView = async () => {
   await db.put(designDoc)
 }
 
-exports.createRoutingView = async () => {
-  const db = getAppDB()
+exports.createRoutingView = async appId => {
+  const db = new CouchDB(appId)
   const designDoc = await db.get("_design/database")
   const view = {
     // if using variables in a map function need to inject them before use
@@ -77,8 +78,8 @@ exports.createRoutingView = async () => {
   await db.put(designDoc)
 }
 
-async function searchIndex(indexName, fnString) {
-  const db = getAppDB()
+async function searchIndex(appId, indexName, fnString) {
+  const db = new CouchDB(appId)
   const designDoc = await db.get("_design/database")
   designDoc.indexes = {
     [indexName]: {
@@ -89,8 +90,9 @@ async function searchIndex(indexName, fnString) {
   await db.put(designDoc)
 }
 
-exports.createAllSearchIndex = async () => {
+exports.createAllSearchIndex = async appId => {
   await searchIndex(
+    appId,
     SearchIndexes.ROWS,
     function (doc) {
       function idx(input, prev) {

package/src/definitions/common.ts

@@ -17,8 +17,6 @@ export interface FieldSchema {
   autocolumn?: boolean
   throughFrom?: string
   throughTo?: string
-  formula?: string
-  formulaType?: string
   main?: boolean
   meta?: {
     toTable: string
@@ -48,7 +46,6 @@ export interface Table extends Base {
   schema: TableSchema
   primaryDisplay?: string
   sourceId?: string
-  relatedFormula?: string[]
   constrained?: string[]
 }
 

package/src/definitions/datasource.ts

@@ -47,7 +47,6 @@ export enum SourceNames {
   ARANGODB = "ARANGODB",
   REST = "REST",
   ORACLE = "ORACLE",
-  GOOGLE_SHEETS = "GOOGLE_SHEETS",
 }
 
 export enum IncludeRelationships {
@@ -87,8 +86,6 @@ export interface ExtraQueryConfig {
 export interface Integration {
   docs: string
   plus?: boolean
-  auth?: { type: string }
-  relationships?: boolean
   description: string
   friendlyName: string
   datasource: {}

package/src/environment.js

@@ -2,8 +2,7 @@ function isTest() {
   return (
     process.env.NODE_ENV === "jest" ||
     process.env.NODE_ENV === "cypress" ||
-    (process.env.JEST_WORKER_ID != null &&
-      process.env.JEST_WORKER_ID !== "null")
+    process.env.JEST_WORKER_ID != null
   )
 }
 
@@ -39,12 +38,10 @@ module.exports = {
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
   USE_QUOTAS: process.env.USE_QUOTAS,
-  EXCLUDE_QUOTAS_TENANTS: process.env.EXCLUDE_QUOTAS_TENANTS,
   REDIS_URL: process.env.REDIS_URL,
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
-  HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS,
   // environment
   NODE_ENV: process.env.NODE_ENV,
   JEST_WORKER_ID: process.env.JEST_WORKER_ID,

package/src/integrations/base/sql.ts

@@ -72,22 +72,16 @@ class InternalBuilder {
 
   // right now we only do filters on the specific table being queried
   addFilters(
+    tableName: string,
     query: KnexQuery,
-    filters: SearchFilters | undefined,
-    opts: { relationship?: boolean; tableName?: string }
+    filters: SearchFilters | undefined
   ): KnexQuery {
     function iterate(
       structure: { [key: string]: any },
       fn: (key: string, value: any) => void
     ) {
       for (let [key, value] of Object.entries(structure)) {
-        const isRelationshipField = key.includes(".")
-        if (!opts.relationship && !isRelationshipField) {
-          fn(`${opts.tableName}.${key}`, value)
-        }
-        if (opts.relationship && isRelationshipField) {
-          fn(key, value)
-        }
+        fn(`${tableName}.${key}`, value)
       }
     }
     if (!filters) {
@@ -278,7 +272,7 @@ class InternalBuilder {
     if (foundOffset) {
       query = query.offset(foundOffset)
     }
-    query = this.addFilters(query, filters, { tableName })
+    query = this.addFilters(tableName, query, filters)
     // add sorting to pre-query
     query = this.addSorting(query, json)
     // @ts-ignore
@@ -291,21 +285,20 @@ class InternalBuilder {
       preQuery = this.addSorting(preQuery, json)
     }
     // handle joins
-    query = this.addRelationships(
+    return this.addRelationships(
       knex,
       preQuery,
       selectStatement,
       tableName,
       relationships
     )
-    return this.addFilters(query, filters, { relationship: true })
   }
 
   update(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, body, filters } = json
     let query: KnexQuery = knex(endpoint.entityId)
     const parsedBody = parseBody(body)
-    query = this.addFilters(query, filters, { tableName: endpoint.entityId })
+    query = this.addFilters(endpoint.entityId, query, filters)
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.update(parsedBody)
@@ -317,7 +310,7 @@ class InternalBuilder {
   delete(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, filters } = json
     let query: KnexQuery = knex(endpoint.entityId)
-    query = this.addFilters(query, filters, { tableName: endpoint.entityId })
+    query = this.addFilters(endpoint.entityId, query, filters)
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.delete()

package/src/integrations/index.ts

@@ -9,7 +9,6 @@ const airtable = require("./airtable")
 const mysql = require("./mysql")
 const arangodb = require("./arangodb")
 const rest = require("./rest")
-const googlesheets = require("./googlesheets")
 const { SourceNames } = require("../definitions/datasource")
 
 const DEFINITIONS = {
@@ -24,7 +23,6 @@ const DEFINITIONS = {
   [SourceNames.MYSQL]: mysql.schema,
   [SourceNames.ARANGODB]: arangodb.schema,
   [SourceNames.REST]: rest.schema,
-  [SourceNames.GOOGLE_SHEETS]: googlesheets.schema,
 }
 
 const INTEGRATIONS = {
@@ -39,7 +37,6 @@ const INTEGRATIONS = {
   [SourceNames.MYSQL]: mysql.integration,
   [SourceNames.ARANGODB]: arangodb.integration,
   [SourceNames.REST]: rest.integration,
-  [SourceNames.GOOGLE_SHEETS]: googlesheets.integration,
 }
 
 // optionally add oracle integration if the oracle binary can be installed

package/src/integrations/tests/sql.spec.js

@@ -119,22 +119,6 @@ describe("SQL query builder", () => {
     })
   })
 
-  it("should allow filtering on a related field", () => {
-    const query = sql._query(generateReadJson({
-      filters: {
-        equal: {
-          age: 10,
-          "task.name": "task 1",
-        },
-      },
-    }))
-    // order of bindings changes because relationship filters occur outside inner query
-    expect(query).toEqual({
-      bindings: [10, limit, "task 1"],
-      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" = $1 limit $2) as "${TABLE_NAME}" where "task"."name" = $3`
-    })
-  })
-
   it("should test an create statement", () => {
     const query = sql._query(generateCreateJson(TABLE_NAME, {
       name: "Michael",

package/src/integrations/utils.ts

@@ -1,5 +1,6 @@
-import { SourceNames, SqlQuery } from "../definitions/datasource"
+import { SqlQuery } from "../definitions/datasource"
 import { Datasource, Table } from "../definitions/common"
+import { SourceNames } from "../definitions/datasource"
 import { DocumentTypes, SEPARATOR } from "../db/utils"
 import { FieldTypes, BuildSchemaErrors, InvalidColumns } from "../constants"
 
@@ -52,10 +53,7 @@ export function buildExternalTableId(datasourceId: string, tableName: string) {
   return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
 }
 
-export function breakExternalTableId(tableId: string | undefined) {
-  if (!tableId) {
-    return {}
-  }
+export function breakExternalTableId(tableId: string) {
   const parts = tableId.split(DOUBLE_SEPARATOR)
   let tableName = parts.pop()
   // if they need joined
@@ -129,12 +127,7 @@ export function isSQL(datasource: Datasource): boolean {
   if (!datasource || !datasource.source) {
     return false
   }
-  const SQL = [
-    SourceNames.POSTGRES,
-    SourceNames.SQL_SERVER,
-    SourceNames.MYSQL,
-    SourceNames.ORACLE,
-  ]
+  const SQL = [SourceNames.POSTGRES, SourceNames.SQL_SERVER, SourceNames.MYSQL]
   return SQL.indexOf(datasource.source) !== -1
 }
 
@@ -146,60 +139,11 @@ export function isIsoDateString(str: string) {
   return d.toISOString() === str
 }
 
-/**
- * This function will determine whether a column is a relationship and whether it
- * is currently valid. The reason for the validity check is that tables can be deleted
- * outside of Budibase control and if this is the case it will break Budibase relationships.
- * The tableIds is a list passed down from the main finalise tables function, which is
- * based on the tables that have just been fetched. This will only really be used on subsequent
- * fetches to the first one - if the user is periodically refreshing Budibase knowledge of tables.
- * @param column The column to check, to see if it is a valid relationship.
- * @param tableIds The IDs of the tables which currently exist.
- */
-function shouldCopyRelationship(
-  column: { type: string; tableId?: string },
-  tableIds: [string]
-) {
-  return (
-    column.type === FieldTypes.LINK &&
-    column.tableId &&
-    tableIds.includes(column.tableId)
-  )
-}
-
-/**
- * Similar function to the shouldCopyRelationship function, but instead this looks for options and boolean
- * types. It is possible to switch a string -> options and a number -> boolean (and vice versus) need to make
- * sure that these get copied over when tables are fetched. Also checks whether they are still valid, if a
- * column has changed type in the external database then copying it over may not be possible.
- * @param column The column to check for options or boolean type.
- * @param fetchedColumn The fetched column to check for the type in the external database.
- */
-function shouldCopySpecialColumn(
-  column: { type: string },
-  fetchedColumn: { type: string } | undefined
-) {
-  return (
-    column.type === FieldTypes.OPTIONS ||
-    column.type === FieldTypes.ARRAY ||
-    ((!fetchedColumn || fetchedColumn.type === FieldTypes.NUMBER) &&
-      column.type === FieldTypes.BOOLEAN)
-  )
-}
-
-/**
- * Looks for columns which need to be copied over into the new table definitions, like relationships
- * and options types.
- * @param tableName The name of the table which is being checked.
- * @param table The specific table which is being checked.
- * @param entities All the tables that existed before - the old table definitions.
- * @param tableIds The IDs of the tables which exist now, to check if anything has been removed.
- */
+// add the existing relationships from the entities if they exist, to prevent them from being overridden
 function copyExistingPropsOver(
   tableName: string,
   table: Table,
-  entities: { [key: string]: any },
-  tableIds: [string]
+  entities: { [key: string]: any }
 ) {
   if (entities && entities[tableName]) {
     if (entities[tableName].primaryDisplay) {
@@ -210,10 +154,12 @@ function copyExistingPropsOver(
       if (!existingTableSchema.hasOwnProperty(key)) {
         continue
       }
-      const column = existingTableSchema[key]
       if (
-        shouldCopyRelationship(column, tableIds) ||
-        shouldCopySpecialColumn(column, table.schema[key])
+        existingTableSchema[key].type === FieldTypes.LINK ||
+        existingTableSchema[key].type === FieldTypes.OPTIONS ||
+        existingTableSchema[key].type === FieldTypes.ARRAY ||
+        ((!table.schema[key] || table.schema[key].type === FieldTypes.NUMBER) &&
+          existingTableSchema[key].type === FieldTypes.BOOLEAN)
       ) {
         table.schema[key] = existingTableSchema[key]
       }
@@ -222,13 +168,6 @@ function copyExistingPropsOver(
   return table
 }
 
-/**
- * Look through the final table definitions to see if anything needs to be
- * copied over from the old and if any errors have occurred mark them so
- * that the user can be made aware.
- * @param tables The list of tables that have been retrieved from the external database.
- * @param entities The old list of tables, if there was any to look for definitions in.
- */
 export function finaliseExternalTables(
   tables: { [key: string]: any },
   entities: { [key: string]: any }
@@ -236,8 +175,6 @@ export function finaliseExternalTables(
   const invalidColumns = Object.values(InvalidColumns)
   let finalTables: { [key: string]: any } = {}
   const errors: { [key: string]: string } = {}
-  // @ts-ignore
-  const tableIds: [string] = Object.values(tables).map(table => table._id)
   for (let [name, table] of Object.entries(tables)) {
     const schemaFields = Object.keys(table.schema)
     // make sure every table has a key
@@ -249,7 +186,7 @@ export function finaliseExternalTables(
       continue
     }
     // make sure all previous props have been added back
-    finalTables[name] = copyExistingPropsOver(name, table, entities, tableIds)
+    finalTables[name] = copyExistingPropsOver(name, table, entities)
   }
   // sort the tables by name
   finalTables = Object.entries(finalTables)

package/src/middleware/authorized.js

@@ -9,60 +9,11 @@ const {
 } = require("@budibase/backend-core/permissions")
 const builderMiddleware = require("./builder")
 const { isWebhookEndpoint } = require("./utils")
-const { buildCsrfMiddleware } = require("@budibase/backend-core/auth")
-const { getAppId } = require("@budibase/backend-core/context")
 
 function hasResource(ctx) {
   return ctx.resourceId != null
 }
 
-const csrf = buildCsrfMiddleware()
-
-/**
- * Apply authorization to the requested resource:
- * - If this is a builder resource the user must be a builder.
- * - Builders can access all resources.
- * - Otherwise the user must have the required role.
- */
-const checkAuthorized = async (ctx, resourceRoles, permType, permLevel) => {
-  // check if this is a builder api and the user is not a builder
-  const isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global
-  const isBuilderApi = permType === PermissionTypes.BUILDER
-  if (isBuilderApi && !isBuilder) {
-    return ctx.throw(403, "Not Authorized")
-  }
-
-  // check for resource authorization
-  if (!isBuilder) {
-    await checkAuthorizedResource(ctx, resourceRoles, permType, permLevel)
-  }
-}
-
-const checkAuthorizedResource = async (
-  ctx,
-  resourceRoles,
-  permType,
-  permLevel
-) => {
-  // get the user's roles
-  const roleId = ctx.roleId || BUILTIN_ROLE_IDS.PUBLIC
-  const userRoles = await getUserRoleHierarchy(roleId, {
-    idOnly: false,
-  })
-  const permError = "User does not have permission"
-  // check if the user has the required role
-  if (resourceRoles.length > 0) {
-    // deny access if the user doesn't have the required resource role
-    const found = userRoles.find(role => resourceRoles.indexOf(role._id) !== -1)
-    if (!found) {
-      ctx.throw(403, permError)
-    }
-    // fallback to the base permissions when no resource roles are found
-  } else if (!doesHaveBasePermission(permType, permLevel, userRoles)) {
-    ctx.throw(403, permError)
-  }
-}
-
 module.exports =
   (permType, permLevel = null) =>
   async (ctx, next) => {
@@ -80,27 +31,40 @@ module.exports =
     // to find API endpoints which are builder focused
     await builderMiddleware(ctx, permType)
 
-    // get the resource roles
-    let resourceRoles = []
-    const appId = getAppId()
-    if (appId && hasResource(ctx)) {
-      resourceRoles = await getRequiredResourceRole(permLevel, ctx)
-    }
-
-    // if the resource is public, proceed
-    const isPublicResource = resourceRoles.includes(BUILTIN_ROLE_IDS.PUBLIC)
-    if (isPublicResource) {
+    const isAuthed = ctx.isAuthenticated
+    // builders for now have permission to do anything
+    let isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global
+    const isBuilderApi = permType === PermissionTypes.BUILDER
+    if (isBuilder) {
       return next()
+    } else if (isBuilderApi && !isBuilder) {
+      return ctx.throw(403, "Not Authorized")
     }
 
-    // check authenticated
-    if (!ctx.isAuthenticated) {
-      return ctx.throw(403, "Session not authenticated")
+    // need to check this first, in-case public access, don't check authed until last
+    const roleId = ctx.roleId || BUILTIN_ROLE_IDS.PUBLIC
+    const hierarchy = await getUserRoleHierarchy(ctx.appId, roleId, {
+      idOnly: false,
+    })
+    const permError = "User does not have permission"
+    let possibleRoleIds = []
+    if (hasResource(ctx)) {
+      possibleRoleIds = await getRequiredResourceRole(ctx.appId, permLevel, ctx)
+    }
+    // check if we found a role, if not fallback to base permissions
+    if (possibleRoleIds.length > 0) {
+      const found = hierarchy.find(
+        role => possibleRoleIds.indexOf(role._id) !== -1
+      )
+      return found ? next() : ctx.throw(403, permError)
+    } else if (!doesHaveBasePermission(permType, permLevel, hierarchy)) {
+      ctx.throw(403, permError)
     }
 
-    // check authorized
-    await checkAuthorized(ctx, resourceRoles, permType, permLevel)
+    // if they are not authed, then anything using the authorized middleware will fail
+    if (!isAuthed) {
+      ctx.throw(403, "Session not authenticated")
+    }
 
-    // csrf protection
-    return csrf(ctx, next)
+    return next()
   }

package/src/middleware/builder.js

@@ -5,7 +5,7 @@ const {
   checkDebounce,
   setDebounce,
 } = require("../utilities/redis")
-const { getDB } = require("@budibase/backend-core/db")
+const CouchDB = require("../db")
 const { DocumentTypes } = require("../db/utils")
 const { PermissionTypes } = require("@budibase/backend-core/permissions")
 const { app: appCache } = require("@budibase/backend-core/cache")
@@ -48,7 +48,7 @@ async function updateAppUpdatedAt(ctx) {
   if (ctx.method === "GET" || (await checkDebounce(appId))) {
     return
   }
-  const db = getDB(appId)
+  const db = new CouchDB(appId)
   const metadata = await db.get(DocumentTypes.APP_METADATA)
   metadata.updatedAt = new Date().toISOString()
   const response = await db.put(metadata)