@budibase/server 1.0.49-alpha.0 → 1.0.49-alpha.12

package/src/api/controllers/layout.js

@@ -2,11 +2,11 @@ const {
   EMPTY_LAYOUT,
   BASE_LAYOUT_PROP_IDS,
 } = require("../../constants/layouts")
-const CouchDB = require("../../db")
 const { generateLayoutID, getScreenParams } = require("../../db/utils")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 exports.save = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   let layout = ctx.request.body
 
   if (!layout.props) {
@@ -26,7 +26,7 @@ exports.save = async function (ctx) {
 }
 
 exports.destroy = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const layoutId = ctx.params.layoutId,
     layoutRev = ctx.params.layoutRev
 

package/src/api/controllers/metadata.js

@@ -1,7 +1,7 @@
 const { MetadataTypes } = require("../../constants")
-const CouchDB = require("../../db")
 const { generateMetadataID } = require("../../db/utils")
 const { saveEntityMetadata, deleteEntityMetadata } = require("../../utilities")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 exports.getTypes = async ctx => {
   ctx.body = {
@@ -14,17 +14,12 @@ exports.saveMetadata = async ctx => {
   if (type === MetadataTypes.AUTOMATION_TEST_HISTORY) {
     ctx.throw(400, "Cannot save automation history type")
   }
-  ctx.body = await saveEntityMetadata(
-    ctx.appId,
-    type,
-    entityId,
-    ctx.request.body
-  )
+  ctx.body = await saveEntityMetadata(type, entityId, ctx.request.body)
 }
 
 exports.deleteMetadata = async ctx => {
   const { type, entityId } = ctx.params
-  await deleteEntityMetadata(ctx.appId, type, entityId)
+  await deleteEntityMetadata(type, entityId)
   ctx.body = {
     message: "Metadata deleted successfully",
   }
@@ -32,7 +27,7 @@ exports.deleteMetadata = async ctx => {
 
 exports.getMetadata = async ctx => {
   const { type, entityId } = ctx.params
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const id = generateMetadataID(type, entityId)
   try {
     ctx.body = await db.get(id)

package/src/api/controllers/migrations.js

@@ -0,0 +1,13 @@
+const { migrate, MIGRATIONS } = require("../../migrations")
+
+exports.migrate = async ctx => {
+  const options = ctx.request.body
+  // don't await as can take a while, just return
+  migrate(options)
+  ctx.status = 200
+}
+
+exports.fetchDefinitions = async ctx => {
+  ctx.body = MIGRATIONS
+  ctx.status = 200
+}

package/src/api/controllers/permission.js

@@ -6,12 +6,12 @@ const {
   getBuiltinRoles,
 } = require("@budibase/backend-core/roles")
 const { getRoleParams } = require("../../db/utils")
-const CouchDB = require("../../db")
 const {
   CURRENTLY_SUPPORTED_LEVELS,
   getBasePermissions,
 } = require("../../utilities/security")
 const { removeFromArray } = require("../../utilities")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 const PermissionUpdateType = {
   REMOVE: "remove",
@@ -35,7 +35,7 @@ async function updatePermissionOnRole(
   { roleId, resourceId, level },
   updateType
 ) {
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const remove = updateType === PermissionUpdateType.REMOVE
   const isABuiltin = isBuiltin(roleId)
   const dbRoleId = getDBRoleID(roleId)
@@ -106,7 +106,7 @@ exports.fetchLevels = function (ctx) {
 }
 
 exports.fetch = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const roles = await getAllDBRoles(db)
   let permissions = {}
   // create an object with structure role ID -> resource ID -> level
@@ -133,7 +133,7 @@ exports.fetch = async function (ctx) {
 
 exports.getResourcePerms = async function (ctx) {
   const resourceId = ctx.params.resourceId
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const body = await db.allDocs(
     getRoleParams(null, {
       include_docs: true,

package/src/api/controllers/query/import/index.ts

@@ -1,10 +1,11 @@
-import CouchDB from "../../../../db"
 import { queryValidation } from "../validation"
 import { generateQueryID } from "../../../../db/utils"
 import { ImportInfo, ImportSource } from "./sources/base"
 import { OpenAPI2 } from "./sources/openapi2"
 import { Query } from "./../../../../definitions/common"
 import { Curl } from "./sources/curl"
+// @ts-ignore
+import { getAppDB } from "@budibase/backend-core/context"
 interface ImportResult {
   errorQueries: Query[]
   queries: Query[]
@@ -33,10 +34,7 @@ export class RestImporter {
     return this.source.getInfo()
   }
 
-  importQueries = async (
-    appId: string,
-    datasourceId: string
-  ): Promise<ImportResult> => {
+  importQueries = async (datasourceId: string): Promise<ImportResult> => {
     // constuct the queries
     let queries = await this.source.getQueries(datasourceId)
 
@@ -58,7 +56,7 @@ export class RestImporter {
       })
 
     // persist queries
-    const db = new CouchDB(appId)
+    const db = getAppDB()
     const response = await db.bulkDocs(queries)
 
     // create index to seperate queries and errors

package/src/api/controllers/query/import/tests/index.spec.js

@@ -6,6 +6,7 @@ const db = jest.fn(() => {
   }
 })
 jest.mock("../../../../../db", () => db)
+require("@budibase/backend-core").init(require("../../../../../db"))
 
 const { RestImporter } = require("../index")
 
@@ -77,7 +78,7 @@ describe("Rest Importer", () => {
   const testImportQueries = async (key, data, assertions) => {
     await init(data)
     bulkDocs.mockReturnValue([])
-    const importResult = await restImporter.importQueries("appId", "datasourceId")
+    const importResult = await restImporter.importQueries("datasourceId")
     expect(importResult.errorQueries.length).toBe(0)
     expect(importResult.queries.length).toBe(assertions[key].count)
     expect(bulkDocs).toHaveBeenCalledTimes(1)

package/src/api/controllers/query/index.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../../../db")
 const {
   generateQueryID,
   getQueryParams,
@@ -10,6 +9,7 @@ const { save: saveDatasource } = require("../datasource")
 const { RestImporter } = require("./import")
 const { invalidateDynamicVariables } = require("../../../threads/utils")
 const environment = require("../../../environment")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 const Runner = new Thread(ThreadType.QUERY, {
   timeoutMs: environment.QUERY_THREAD_TIMEOUT || 10000,
@@ -28,7 +28,7 @@ function enrichQueries(input) {
 }
 
 exports.fetch = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
 
   const body = await db.allDocs(
     getQueryParams(null, {
@@ -69,7 +69,7 @@ exports.import = async ctx => {
     datasourceId = body.datasourceId
   }
 
-  const importResult = await importer.importQueries(ctx.appId, datasourceId)
+  const importResult = await importer.importQueries(datasourceId)
 
   ctx.body = {
     ...importResult,
@@ -79,7 +79,7 @@ exports.import = async ctx => {
 }
 
 exports.save = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const query = ctx.request.body
 
   if (!query._id) {
@@ -94,7 +94,7 @@ exports.save = async function (ctx) {
 }
 
 exports.find = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const query = enrichQueries(await db.get(ctx.params.queryId))
   // remove properties that could be dangerous in real app
   if (isProdAppID(ctx.appId)) {
@@ -105,7 +105,7 @@ exports.find = async function (ctx) {
 }
 
 exports.preview = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
 
   const datasource = await db.get(ctx.request.body.datasourceId)
   // preview may not have a queryId as it hasn't been saved, but if it does
@@ -136,7 +136,7 @@ exports.preview = async function (ctx) {
 }
 
 async function execute(ctx, opts = { rowsOnly: false }) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
 
   const query = await db.get(ctx.params.queryId)
   const datasource = await db.get(query.datasourceId)
@@ -181,7 +181,8 @@ exports.executeV2 = async function (ctx) {
   return execute(ctx, { rowsOnly: false })
 }
 
-const removeDynamicVariables = async (db, queryId) => {
+const removeDynamicVariables = async queryId => {
+  const db = getAppDB()
   const query = await db.get(queryId)
   const datasource = await db.get(query.datasourceId)
   const dynamicVariables = datasource.config.dynamicVariables
@@ -202,8 +203,8 @@ const removeDynamicVariables = async (db, queryId) => {
 }
 
 exports.destroy = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
-  await removeDynamicVariables(db, ctx.params.queryId)
+  const db = getAppDB()
+  await removeDynamicVariables(ctx.params.queryId)
   await db.remove(ctx.params.queryId, ctx.params.revId)
   ctx.message = `Query deleted.`
   ctx.status = 200

package/src/api/controllers/role.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../../db")
 const {
   Role,
   getRole,
@@ -10,6 +9,7 @@ const {
   getUserMetadataParams,
   InternalTables,
 } = require("../../db/utils")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 const UpdateRolesOptions = {
   CREATED: "created",
@@ -40,15 +40,15 @@ async function updateRolesOnUserTable(db, roleId, updateOption) {
 }
 
 exports.fetch = async function (ctx) {
-  ctx.body = await getAllRoles(ctx.appId)
+  ctx.body = await getAllRoles()
 }
 
 exports.find = async function (ctx) {
-  ctx.body = await getRole(ctx.appId, ctx.params.roleId)
+  ctx.body = await getRole(ctx.params.roleId)
 }
 
 exports.save = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   let { _id, name, inherits, permissionId } = ctx.request.body
   if (!_id) {
     _id = generateRoleID()
@@ -69,7 +69,7 @@ exports.save = async function (ctx) {
 }
 
 exports.destroy = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const roleId = ctx.params.roleId
   if (isBuiltin(roleId)) {
     ctx.throw(400, "Cannot delete builtin role.")

package/src/api/controllers/routing.js

@@ -39,12 +39,11 @@ Routing.prototype.addScreenId = function (fullpath, roleId, screenId) {
 
 /**
  * Gets the full routing structure by querying the routing view and processing the result into the tree.
- * @param {string} appId The application to produce the routing structure for.
  * @returns {Promise<object>} The routing structure, this is the full structure designed for use in the builder,
  * if the client routing is required then the updateRoutingStructureForUserRole should be used.
  */
-async function getRoutingStructure(appId) {
-  const screenRoutes = await getRoutingInfo(appId)
+async function getRoutingStructure() {
+  const screenRoutes = await getRoutingInfo()
   const routing = new Routing()
 
   for (let screenRoute of screenRoutes) {
@@ -57,13 +56,13 @@ async function getRoutingStructure(appId) {
 }
 
 exports.fetch = async ctx => {
-  ctx.body = await getRoutingStructure(ctx.appId)
+  ctx.body = await getRoutingStructure()
 }
 
 exports.clientFetch = async ctx => {
-  const routing = await getRoutingStructure(ctx.appId)
+  const routing = await getRoutingStructure()
   let roleId = ctx.user.role._id
-  const roleIds = await getUserRoleHierarchy(ctx.appId, roleId)
+  const roleIds = await getUserRoleHierarchy(roleId)
   for (let topLevel of Object.values(routing.routes)) {
     for (let subpathKey of Object.keys(topLevel.subpaths)) {
       let found = false

package/src/api/controllers/row/ExternalRequest.ts

@@ -19,6 +19,19 @@ import {
   isRowId,
   convertRowId,
 } from "../../../integrations/utils"
+import { getDatasourceAndQuery } from "./utils"
+import {
+  DataSourceOperation,
+  FieldTypes,
+  RelationshipTypes,
+} from "../../../constants"
+import { breakExternalTableId, isSQL } from "../../../integrations/utils"
+import { processObjectSync } from "@budibase/string-templates"
+// @ts-ignore
+import { cloneDeep } from "lodash/fp"
+import { processFormulas } from "../../../utilities/rowProcessor/utils"
+// @ts-ignore
+import { getAppDB } from "@budibase/backend-core/context"
 
 interface ManyRelationship {
   tableId?: string
@@ -38,18 +51,6 @@ interface RunConfig {
 }
 
 module External {
-  const { getDatasourceAndQuery } = require("./utils")
-  const {
-    DataSourceOperation,
-    FieldTypes,
-    RelationshipTypes,
-  } = require("../../../constants")
-  const { breakExternalTableId, isSQL } = require("../../../integrations/utils")
-  const { processObjectSync } = require("@budibase/string-templates")
-  const { cloneDeep } = require("lodash/fp")
-  const CouchDB = require("../../../db")
-  const { processFormulas } = require("../../../utilities/rowProcessor/utils")
-
   function buildFilters(
     id: string | undefined,
     filters: SearchFilters,
@@ -183,7 +184,7 @@ module External {
     thisRow._id = generateIdForRow(row, table)
     thisRow.tableId = table._id
     thisRow._rev = "rev"
-    return thisRow
+    return processFormulas(table, thisRow)
   }
 
   function fixArrayTypes(row: Row, table: Table) {
@@ -210,19 +211,12 @@ module External {
   }
 
   class ExternalRequest {
-    private readonly appId: string
     private operation: Operation
     private tableId: string
     private datasource: Datasource
     private tables: { [key: string]: Table } = {}
 
-    constructor(
-      appId: string,
-      operation: Operation,
-      tableId: string,
-      datasource: Datasource
-    ) {
-      this.appId = appId
+    constructor(operation: Operation, tableId: string, datasource: Datasource) {
       this.operation = operation
       this.tableId = tableId
       this.datasource = datasource
@@ -231,12 +225,14 @@ module External {
       }
     }
 
-    getTable(tableId: string | undefined): Table {
+    getTable(tableId: string | undefined): Table | undefined {
       if (!tableId) {
         throw "Table ID is unknown, cannot find table"
       }
       const { tableName } = breakExternalTableId(tableId)
-      return this.tables[tableName]
+      if (tableName) {
+        return this.tables[tableName]
+      }
     }
 
     inputProcessing(row: Row | undefined, table: Table) {
@@ -272,9 +268,11 @@ module External {
           newRow[key] = row[key]
           continue
         }
-        const { tableName: linkTableName } = breakExternalTableId(field.tableId)
+        const { tableName: linkTableName } = breakExternalTableId(
+          field?.tableId
+        )
         // table has to exist for many to many
-        if (!this.tables[linkTableName]) {
+        if (!linkTableName || !this.tables[linkTableName]) {
           continue
         }
         const linkTable = this.tables[linkTableName]
@@ -329,8 +327,12 @@ module External {
      * This iterates through the returned rows and works out what elements of the rows
      * actually match up to another row (based on primary keys) - this is pretty specific
      * to SQL and the way that SQL relationships are returned based on joins.
+     * This is complicated, but the idea is that when a SQL query returns all the relations
+     * will be separate rows, with all of the data in each row. We have to decipher what comes
+     * from where (which tables) and how to convert that into budibase columns.
      */
     updateRelationshipColumns(
+      table: Table,
       row: Row,
       rows: { [key: string]: Row },
       relationships: RelationshipsJson[]
@@ -341,6 +343,13 @@ module External {
         if (!linkedTable) {
           continue
         }
+        const fromColumn = `${table.name}.${relationship.from}`
+        const toColumn = `${linkedTable.name}.${relationship.to}`
+        // this is important when working with multiple relationships
+        // between the same tables, don't want to overlap/multiply the relations
+        if (!relationship.through && row[fromColumn] !== row[toColumn]) {
+          continue
+        }
         let linked = basicProcessing(row, linkedTable)
         if (!linked._id) {
           continue
@@ -388,6 +397,7 @@ module External {
         // this is a relationship of some sort
         if (finalRows[rowId]) {
           finalRows = this.updateRelationshipColumns(
+            table,
             row,
             finalRows,
             relationships
@@ -401,6 +411,7 @@ module External {
         finalRows[thisRow._id] = thisRow
         // do this at end once its been added to the final rows
         finalRows = this.updateRelationshipColumns(
+          table,
           row,
           finalRows,
           relationships
@@ -422,7 +433,7 @@ module External {
         }
         const { tableName: linkTableName } = breakExternalTableId(field.tableId)
         // no table to link to, this is not a valid relationships
-        if (!this.tables[linkTableName]) {
+        if (!linkTableName || !this.tables[linkTableName]) {
           continue
         }
         const linkTable = this.tables[linkTableName]
@@ -460,6 +471,9 @@ module External {
     async lookupRelations(tableId: string, row: Row) {
       const related: { [key: string]: any } = {}
       const { tableName } = breakExternalTableId(tableId)
+      if (!tableName) {
+        return related
+      }
       const table = this.tables[tableName]
       // @ts-ignore
       const primaryKey = table.primary[0]
@@ -484,7 +498,7 @@ module External {
         if (!lookupField || !row[lookupField]) {
           continue
         }
-        const response = await getDatasourceAndQuery(this.appId, {
+        const response = await getDatasourceAndQuery({
           endpoint: getEndpoint(tableId, DataSourceOperation.READ),
           filters: {
             equal: {
@@ -515,28 +529,30 @@ module External {
       row: Row,
       relationships: ManyRelationship[]
     ) {
-      const { appId } = this
       // if we're creating (in a through table) need to wipe the existing ones first
       const promises = []
       const related = await this.lookupRelations(mainTableId, row)
       for (let relationship of relationships) {
         const { key, tableId, isUpdate, id, ...rest } = relationship
-        const body = processObjectSync(rest, row)
+        const body: { [key: string]: any } = processObjectSync(rest, row, {})
         const linkTable = this.getTable(tableId)
         // @ts-ignore
-        const linkPrimary = linkTable.primary[0]
-        const rows = related[key]?.rows || []
+        const linkPrimary = linkTable?.primary[0]
+        if (!linkTable || !linkPrimary) {
+          return
+        }
+        const rows = related[key].rows || []
         const found = rows.find(
           (row: { [key: string]: any }) =>
             row[linkPrimary] === relationship.id ||
-            row[linkPrimary] === body[linkPrimary]
+            row[linkPrimary] === body?.[linkPrimary]
         )
         const operation = isUpdate
           ? DataSourceOperation.UPDATE
           : DataSourceOperation.CREATE
         if (!found) {
           promises.push(
-            getDatasourceAndQuery(appId, {
+            getDatasourceAndQuery({
               endpoint: getEndpoint(tableId, operation),
               // if we're doing many relationships then we're writing, only one response
               body,
@@ -552,9 +568,12 @@ module External {
       for (let [colName, { isMany, rows, tableId }] of Object.entries(
         related
       )) {
-        const table: Table = this.getTable(tableId)
+        const table: Table | undefined = this.getTable(tableId)
         // if its not the foreign key skip it, nothing to do
-        if (table.primary && table.primary.indexOf(colName) !== -1) {
+        if (
+          !table ||
+          (table.primary && table.primary.indexOf(colName) !== -1)
+        ) {
           continue
         }
         for (let row of rows) {
@@ -566,7 +585,7 @@ module External {
               : DataSourceOperation.UPDATE
             const body = isMany ? null : { [colName]: null }
             promises.push(
-              getDatasourceAndQuery(this.appId, {
+              getDatasourceAndQuery({
                 endpoint: getEndpoint(tableId, op),
                 body,
                 filters,
@@ -605,20 +624,25 @@ module External {
           continue
         }
         const { tableName: linkTableName } = breakExternalTableId(field.tableId)
-        const linkTable = this.tables[linkTableName]
-        if (linkTable) {
-          const linkedFields = extractRealFields(linkTable, fields)
-          fields = fields.concat(linkedFields)
+        if (linkTableName) {
+          const linkTable = this.tables[linkTableName]
+          if (linkTable) {
+            const linkedFields = extractRealFields(linkTable, fields)
+            fields = fields.concat(linkedFields)
+          }
         }
       }
       return fields
     }
 
     async run(config: RunConfig) {
-      const { appId, operation, tableId } = this
+      const { operation, tableId } = this
       let { datasourceId, tableName } = breakExternalTableId(tableId)
+      if (!tableName) {
+        throw "Unable to run without a table name"
+      }
       if (!this.datasource) {
-        const db = new CouchDB(appId)
+        const db = getAppDB()
         this.datasource = await db.get(datasourceId)
         if (!this.datasource || !this.datasource.entities) {
           throw "No tables found, fetch tables before query."
@@ -670,7 +694,7 @@ module External {
         },
       }
       // can't really use response right now
-      const response = await getDatasourceAndQuery(appId, json)
+      const response = await getDatasourceAndQuery(json)
       // handle many to many relationships now if we know the ID (could be auto increment)
       if (
         operation !== DataSourceOperation.READ &&