@budibase/server 1.0.49-alpha.6 → 1.0.49

package/src/api/controllers/row/internalSearch.js

@@ -1,14 +1,14 @@
 const { SearchIndexes } = require("../../../db/utils")
 const fetch = require("node-fetch")
 const { getCouchUrl } = require("@budibase/backend-core/db")
-const { getAppId } = require("@budibase/backend-core/context")
 
 /**
  * Class to build lucene query URLs.
  * Optionally takes a base lucene query object.
  */
 class QueryBuilder {
-  constructor(base) {
+  constructor(appId, base) {
+    this.appId = appId
     this.query = {
       string: {},
       fuzzy: {},
@@ -37,30 +37,22 @@ class QueryBuilder {
   }
 
   setLimit(limit) {
-    if (limit != null) {
-      this.limit = limit
-    }
+    this.limit = limit
     return this
   }
 
   setSort(sort) {
-    if (sort != null) {
-      this.sort = sort
-    }
+    this.sort = sort
     return this
   }
 
   setSortOrder(sortOrder) {
-    if (sortOrder != null) {
-      this.sortOrder = sortOrder
-    }
+    this.sortOrder = sortOrder
     return this
   }
 
   setSortType(sortType) {
-    if (sortType != null) {
-      this.sortType = sortType
-    }
+    this.sortType = sortType
     return this
   }
 
@@ -241,8 +233,7 @@ class QueryBuilder {
   }
 
   async run() {
-    const appId = getAppId()
-    const url = `${getCouchUrl()}/${appId}/_design/database/_search/${
+    const url = `${getCouchUrl()}/${this.appId}/_design/database/_search/${
       SearchIndexes.ROWS
     }`
     const body = this.buildSearchBody()
@@ -279,6 +270,7 @@ const runQuery = async (url, body) => {
  * Gets round the fixed limit of 200 results from a query by fetching as many
  * pages as required and concatenating the results. This recursively operates
  * until enough results have been found.
+ * @param appId {string} The app ID to search
  * @param query {object} The JSON query structure
  * @param params {object} The search params including:
  *   tableId {string} The table ID to search
@@ -291,7 +283,7 @@ const runQuery = async (url, body) => {
  *   rows {array|null} Current results in the recursive search
  * @returns {Promise<*[]|*>}
  */
-const recursiveSearch = async (query, params) => {
+const recursiveSearch = async (appId, query, params) => {
   const bookmark = params.bookmark
   const rows = params.rows || []
   if (rows.length >= params.limit) {
@@ -301,7 +293,7 @@ const recursiveSearch = async (query, params) => {
   if (rows.length > params.limit - 200) {
     pageSize = params.limit - rows.length
   }
-  const page = await new QueryBuilder(query)
+  const page = await new QueryBuilder(appId, query)
     .setVersion(params.version)
     .setTable(params.tableId)
     .setBookmark(bookmark)
@@ -321,13 +313,14 @@ const recursiveSearch = async (query, params) => {
     bookmark: page.bookmark,
     rows: [...rows, ...page.rows],
   }
-  return await recursiveSearch(query, newParams)
+  return await recursiveSearch(appId, query, newParams)
 }
 
 /**
  * Performs a paginated search. A bookmark will be returned to allow the next
  * page to be fetched. There is a max limit off 200 results per page in a
  * paginated search.
+ * @param appId {string} The app ID to search
  * @param query {object} The JSON query structure
  * @param params {object} The search params including:
  *   tableId {string} The table ID to search
@@ -339,13 +332,13 @@ const recursiveSearch = async (query, params) => {
  *   bookmark {string} The bookmark to resume from
  * @returns {Promise<{hasNextPage: boolean, rows: *[]}>}
  */
-exports.paginatedSearch = async (query, params) => {
+exports.paginatedSearch = async (appId, query, params) => {
   let limit = params.limit
   if (limit == null || isNaN(limit) || limit < 0) {
     limit = 50
   }
   limit = Math.min(limit, 200)
-  const search = new QueryBuilder(query)
+  const search = new QueryBuilder(appId, query)
     .setVersion(params.version)
     .setTable(params.tableId)
     .setSort(params.sort)
@@ -374,6 +367,7 @@ exports.paginatedSearch = async (query, params) => {
  * desired amount of results. There is a limit of 1000 results to avoid
  * heavy performance hits, and to avoid client components breaking from
  * handling too much data.
+ * @param appId {string} The app ID to search
  * @param query {object} The JSON query structure
  * @param params {object} The search params including:
  *   tableId {string} The table ID to search
@@ -384,12 +378,12 @@ exports.paginatedSearch = async (query, params) => {
  *   limit {number} The desired number of results
  * @returns {Promise<{rows: *}>}
  */
-exports.fullSearch = async (query, params) => {
+exports.fullSearch = async (appId, query, params) => {
   let limit = params.limit
   if (limit == null || isNaN(limit) || limit < 0) {
     limit = 1000
   }
   params.limit = Math.min(limit, 1000)
-  const rows = await recursiveSearch(query, params)
+  const rows = await recursiveSearch(appId, query, params)
   return { rows }
 }

package/src/api/controllers/row/utils.js

@@ -1,11 +1,11 @@
 const validateJs = require("validate.js")
 const { cloneDeep } = require("lodash/fp")
+const CouchDB = require("../../../db")
 const { InternalTables } = require("../../../db/utils")
 const userController = require("../user")
 const { FieldTypes } = require("../../../constants")
 const { processStringSync } = require("@budibase/string-templates")
 const { makeExternalQuery } = require("../../../integrations/base/utils")
-const { getAppDB } = require("@budibase/backend-core/context")
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value) {
@@ -17,15 +17,14 @@ validateJs.extend(validateJs.validators.datetime, {
   },
 })
 
-exports.getDatasourceAndQuery = async json => {
+exports.getDatasourceAndQuery = async (appId, json) => {
   const datasourceId = json.endpoint.datasourceId
-  const db = getAppDB()
+  const db = new CouchDB(appId)
   const datasource = await db.get(datasourceId)
   return makeExternalQuery(datasource, json)
 }
 
-exports.findRow = async (ctx, tableId, rowId) => {
-  const db = getAppDB()
+exports.findRow = async (ctx, db, tableId, rowId) => {
   let row
   // TODO remove special user case in future
   if (tableId === InternalTables.USER_METADATA) {
@@ -43,9 +42,9 @@ exports.findRow = async (ctx, tableId, rowId) => {
   return row
 }
 
-exports.validate = async ({ tableId, row, table }) => {
+exports.validate = async ({ appId, tableId, row, table }) => {
   if (!table) {
-    const db = getAppDB()
+    const db = new CouchDB(appId)
     table = await db.get(tableId)
   }
   const errors = {}
@@ -59,16 +58,22 @@ exports.validate = async ({ tableId, row, table }) => {
     let res
 
     // Validate.js doesn't seem to handle array
-    if (type === FieldTypes.ARRAY && row[fieldName]) {
-      if (row[fieldName].length) {
+    if (type === FieldTypes.ARRAY) {
+      const hasValues =
+        Array.isArray(row[fieldName]) && row[fieldName].length > 0
+
+      // Check values are valid if values are specified
+      if (hasValues) {
         row[fieldName].map(val => {
           if (!constraints.inclusion.includes(val)) {
-            errors[fieldName] = "Field not in list"
+            errors[fieldName] = "Value not in list"
           }
         })
-      } else if (constraints.presence && row[fieldName].length === 0) {
-        // non required MultiSelect creates an empty array, which should not throw errors
-        errors[fieldName] = [`${fieldName} is required`]
+      }
+
+      // Check for required constraint
+      if (constraints.presence === true && !hasValues) {
+        errors[fieldName] = "Required field"
       }
     } else if (type === FieldTypes.JSON && typeof row[fieldName] === "string") {
       // this should only happen if there is an error

package/src/api/controllers/screen.js

@@ -1,9 +1,10 @@
+const CouchDB = require("../../db")
 const { getScreenParams, generateScreenID } = require("../../db/utils")
 const { AccessController } = require("@budibase/backend-core/roles")
-const { getAppDB } = require("@budibase/backend-core/context")
 
 exports.fetch = async ctx => {
-  const db = getAppDB()
+  const appId = ctx.appId
+  const db = new CouchDB(appId)
 
   const screens = (
     await db.allDocs(
@@ -13,14 +14,15 @@ exports.fetch = async ctx => {
     )
   ).rows.map(element => element.doc)
 
-  ctx.body = await new AccessController().checkScreensAccess(
+  ctx.body = await new AccessController(appId).checkScreensAccess(
     screens,
     ctx.user.role._id
   )
 }
 
 exports.save = async ctx => {
-  const db = getAppDB()
+  const appId = ctx.appId
+  const db = new CouchDB(appId)
   let screen = ctx.request.body
 
   if (!screen._id) {
@@ -37,7 +39,7 @@ exports.save = async ctx => {
 }
 
 exports.destroy = async ctx => {
-  const db = getAppDB()
+  const db = new CouchDB(ctx.appId)
   await db.remove(ctx.params.screenId, ctx.params.screenRev)
   ctx.body = {
     message: "Screen deleted successfully",

package/src/api/controllers/static/index.js

@@ -5,7 +5,8 @@ const { resolve, join } = require("../../../utilities/centralPath")
 const uuid = require("uuid")
 const { ObjectStoreBuckets } = require("../../../constants")
 const { processString } = require("@budibase/string-templates")
-const { getAllApps } = require("@budibase/backend-core/db")
+const { getDeployedApps } = require("../../../utilities/workerRequests")
+const CouchDB = require("../../../db")
 const {
   loadHandlebarsFile,
   NODE_MODULES_PATH,
@@ -16,9 +17,6 @@ const { clientLibraryPath } = require("../../../utilities")
 const { upload } = require("../../../utilities/fileSystem")
 const { attachmentsRelativeURL } = require("../../../utilities")
 const { DocumentTypes } = require("../../../db/utils")
-const { getAppDB } = require("@budibase/backend-core/context")
-const AWS = require("aws-sdk")
-const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1"
 
 async function prepareUpload({ s3Key, bucket, metadata, file }) {
   const response = await upload({
@@ -39,18 +37,12 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) {
   }
 }
 
-async function getAppIdFromUrl(ctx) {
-  // the "appId" component of the URL can be the id or the custom url
+async function checkForSelfHostedURL(ctx) {
+  // the "appId" component of the URL may actually be a specific self hosted URL
   let possibleAppUrl = `/${encodeURI(ctx.params.appId).toLowerCase()}`
-
-  // search prod apps for a url that matches, exclude dev where id is always used
-  const apps = await getAllApps({ dev: false })
-  const app = apps.filter(
-    a => a.url && a.url.toLowerCase() === possibleAppUrl
-  )[0]
-
-  if (app && app.appId) {
-    return app.appId
+  const apps = await getDeployedApps()
+  if (apps[possibleAppUrl] && apps[possibleAppUrl].appId) {
+    return apps[possibleAppUrl].appId
   } else {
     return ctx.params.appId
   }
@@ -83,9 +75,12 @@ exports.uploadFile = async function (ctx) {
 }
 
 exports.serveApp = async function (ctx) {
-  let appId = await getAppIdFromUrl(ctx)
+  let appId = ctx.params.appId
+  if (env.SELF_HOSTED) {
+    appId = await checkForSelfHostedURL(ctx)
+  }
   const App = require("./templates/BudibaseApp.svelte").default
-  const db = getAppDB({ skip_setup: true })
+  const db = new CouchDB(appId, { skip_setup: true })
   const appInfo = await db.get(DocumentTypes.APP_METADATA)
 
   const { head, html, css } = App.render({
@@ -109,51 +104,3 @@ exports.serveClientLibrary = async function (ctx) {
     root: join(NODE_MODULES_PATH, "@budibase", "client", "dist"),
   })
 }
-
-exports.getSignedUploadURL = async function (ctx) {
-  const database = getAppDB()
-
-  // Ensure datasource is valid
-  let datasource
-  try {
-    const { datasourceId } = ctx.params
-    datasource = await database.get(datasourceId)
-    if (!datasource) {
-      ctx.throw(400, "The specified datasource could not be found")
-    }
-  } catch (error) {
-    ctx.throw(400, "The specified datasource could not be found")
-  }
-
-  // Ensure we aren't using a custom endpoint
-  if (datasource?.config?.endpoint) {
-    ctx.throw(400, "S3 datasources with custom endpoints are not supported")
-  }
-
-  // Determine type of datasource and generate signed URL
-  let signedUrl
-  let publicUrl
-  if (datasource.source === "S3") {
-    const { bucket, key } = ctx.request.body || {}
-    if (!bucket || !key) {
-      ctx.throw(400, "bucket and key values are required")
-      return
-    }
-    try {
-      const s3 = new AWS.S3({
-        region: AWS_REGION,
-        accessKeyId: datasource?.config?.accessKeyId,
-        secretAccessKey: datasource?.config?.secretAccessKey,
-        apiVersion: "2006-03-01",
-        signatureVersion: "v4",
-      })
-      const params = { Bucket: bucket, Key: key }
-      signedUrl = s3.getSignedUrl("putObject", params)
-      publicUrl = `https://${bucket}.s3.${AWS_REGION}.amazonaws.com/${key}`
-    } catch (error) {
-      ctx.throw(400, error)
-    }
-  }
-
-  ctx.body = { signedUrl, publicUrl }
-}

package/src/api/controllers/table/external.js

@@ -1,3 +1,4 @@
+const CouchDB = require("../../../db")
 const {
   buildExternalTableId,
   breakExternalTableId,
@@ -18,7 +19,6 @@ const { makeExternalQuery } = require("../../../integrations/base/utils")
 const { cloneDeep } = require("lodash/fp")
 const csvParser = require("../../../utilities/csvParser")
 const { handleRequest } = require("../row/external")
-const { getAppDB } = require("@budibase/backend-core/context")
 
 async function makeTableRequest(
   datasource,
@@ -159,6 +159,7 @@ function isRelationshipSetup(column) {
 }
 
 exports.save = async function (ctx) {
+  const appId = ctx.appId
   const table = ctx.request.body
   // can't do this right now
   delete table.dataImport
@@ -175,14 +176,14 @@ exports.save = async function (ctx) {
 
   let oldTable
   if (ctx.request.body && ctx.request.body._id) {
-    oldTable = await getTable(ctx.request.body._id)
+    oldTable = await getTable(appId, ctx.request.body._id)
   }
 
   if (hasTypeChanged(tableToSave, oldTable)) {
     ctx.throw(400, "A column type has changed.")
   }
 
-  const db = getAppDB()
+  const db = new CouchDB(appId)
   const datasource = await db.get(datasourceId)
   const oldTables = cloneDeep(datasource.entities)
   const tables = datasource.entities
@@ -266,13 +267,14 @@ exports.save = async function (ctx) {
 }
 
 exports.destroy = async function (ctx) {
-  const tableToDelete = await getTable(ctx.params.tableId)
+  const appId = ctx.appId
+  const tableToDelete = await getTable(appId, ctx.params.tableId)
   if (!tableToDelete || !tableToDelete.created) {
     ctx.throw(400, "Cannot delete tables which weren't created in Budibase.")
   }
   const datasourceId = getDatasourceId(tableToDelete)
 
-  const db = getAppDB()
+  const db = new CouchDB(appId)
   const datasource = await db.get(datasourceId)
   const tables = datasource.entities
 
@@ -288,7 +290,8 @@ exports.destroy = async function (ctx) {
 }
 
 exports.bulkImport = async function (ctx) {
-  const table = await getTable(ctx.params.tableId)
+  const appId = ctx.appId
+  const table = await getTable(appId, ctx.params.tableId)
   const { dataImport } = ctx.request.body
   if (!dataImport || !dataImport.schema || !dataImport.csvString) {
     ctx.throw(400, "Provided data import information is invalid.")
@@ -297,7 +300,7 @@ exports.bulkImport = async function (ctx) {
     ...dataImport,
     existingTable: table,
   })
-  await handleRequest(DataSourceOperation.BULK_CREATE, table._id, {
+  await handleRequest(appId, DataSourceOperation.BULK_CREATE, table._id, {
     rows,
   })
   return table

package/src/api/controllers/table/index.js

@@ -1,10 +1,14 @@
+const CouchDB = require("../../../db")
 const internal = require("./internal")
 const external = require("./external")
 const csvParser = require("../../../utilities/csvParser")
-const { isExternalTable, isSQL } = require("../../../integrations/utils")
-const { getDatasourceParams } = require("../../../db/utils")
-const { getAppDB } = require("@budibase/backend-core/context")
-const { getTable, getAllInternalTables } = require("./utils")
+const { isExternalTable } = require("../../../integrations/utils")
+const {
+  getTableParams,
+  getDatasourceParams,
+  BudibaseInternalDB,
+} = require("../../../db/utils")
+const { getTable } = require("./utils")
 
 function pickApi({ tableId, table }) {
   if (table && !tableId) {
@@ -20,9 +24,19 @@ function pickApi({ tableId, table }) {
 
 // covers both internal and external
 exports.fetch = async function (ctx) {
-  const db = getAppDB()
+  const db = new CouchDB(ctx.appId)
 
-  const internal = await getAllInternalTables()
+  const internalTables = await db.allDocs(
+    getTableParams(null, {
+      include_docs: true,
+    })
+  )
+
+  const internal = internalTables.rows.map(row => ({
+    ...row.doc,
+    type: "internal",
+    sourceId: BudibaseInternalDB._id,
+  }))
 
   const externalTables = await db.allDocs(
     getDatasourceParams("plus", {
@@ -30,18 +44,12 @@ exports.fetch = async function (ctx) {
     })
   )
 
-  const external = externalTables.rows.flatMap(tableDoc => {
-    let entities = tableDoc.doc.entities
-    if (entities) {
-      return Object.values(entities).map(entity => ({
-        ...entity,
-        type: "external",
-        sourceId: tableDoc.doc._id,
-        sql: isSQL(tableDoc.doc),
-      }))
-    } else {
-      return []
-    }
+  const external = externalTables.rows.flatMap(row => {
+    return Object.values(row.doc.entities || {}).map(entity => ({
+      ...entity,
+      type: "external",
+      sourceId: row.doc._id,
+    }))
   })
 
   ctx.body = [...internal, ...external]
@@ -49,7 +57,7 @@ exports.fetch = async function (ctx) {
 
 exports.find = async function (ctx) {
   const tableId = ctx.params.id
-  ctx.body = await getTable(tableId)
+  ctx.body = await getTable(ctx.appId, tableId)
 }
 
 exports.save = async function (ctx) {
@@ -88,7 +96,7 @@ exports.validateCSVSchema = async function (ctx) {
   const { csvString, schema = {}, tableId } = ctx.request.body
   let existingTable
   if (tableId) {
-    existingTable = await getTable(tableId)
+    existingTable = await getTable(ctx.appId, tableId)
   }
   let result = await csvParser.parse(csvString, schema)
   if (existingTable) {

package/src/api/controllers/table/internal.js

@@ -1,3 +1,4 @@
+const CouchDB = require("../../../db")
 const linkRows = require("../../../db/linkedRows")
 const { getRowParams, generateTableID } = require("../../../db/utils")
 const { FieldTypes } = require("../../../constants")
@@ -7,14 +8,10 @@ const {
   getTable,
   handleDataImport,
 } = require("./utils")
-const usageQuota = require("../../../utilities/usageQuota")
-const { getAppDB } = require("@budibase/backend-core/context")
-const env = require("../../../environment")
-const { cleanupAttachments } = require("../../../utilities/rowProcessor")
-const { runStaticFormulaChecks } = require("./bulkFormula")
 
 exports.save = async function (ctx) {
-  const db = getAppDB()
+  const appId = ctx.appId
+  const db = new CouchDB(appId)
   const { dataImport, ...rest } = ctx.request.body
   let tableToSave = {
     type: "table",
@@ -36,7 +33,8 @@ exports.save = async function (ctx) {
   // saving a table is a complex operation, involving many different steps, this
   // has been broken out into a utility to make it more obvious/easier to manipulate
   const tableSaveFunctions = new TableSaveFunctions({
-    user: ctx.user,
+    db,
+    ctx,
     oldTable,
     dataImport,
   })
@@ -81,6 +79,7 @@ exports.save = async function (ctx) {
   // update linked rows
   try {
     const linkResp = await linkRows.updateLinks({
+      appId,
       eventType: oldTable
         ? linkRows.EventType.TABLE_UPDATED
         : linkRows.EventType.TABLE_SAVE,
@@ -104,13 +103,13 @@ exports.save = async function (ctx) {
   tableToSave._rev = result.rev
 
   tableToSave = await tableSaveFunctions.after(tableToSave)
-  // has to run after, make sure it has _id
-  await runStaticFormulaChecks(tableToSave, { oldTable })
+
   return tableToSave
 }
 
 exports.destroy = async function (ctx) {
-  const db = getAppDB()
+  const appId = ctx.appId
+  const db = new CouchDB(appId)
   const tableToDelete = await db.get(ctx.params.tableId)
 
   // Delete all rows for that table
@@ -120,10 +119,10 @@ exports.destroy = async function (ctx) {
     })
   )
   await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
-  await usageQuota.update(usageQuota.Properties.ROW, -rows.rows.length)
 
   // update linked rows
   await linkRows.updateLinks({
+    appId,
     eventType: linkRows.EventType.TABLE_DELETE,
     table: tableToDelete,
   })
@@ -132,25 +131,21 @@ exports.destroy = async function (ctx) {
   await db.remove(tableToDelete)
 
   // remove table search index
-  if (!env.isTest()) {
-    const currentIndexes = await db.getIndexes()
-    const existingIndex = currentIndexes.indexes.find(
-      existing => existing.name === `search:${ctx.params.tableId}`
-    )
-    if (existingIndex) {
-      await db.deleteIndex(existingIndex)
-    }
+  const currentIndexes = await db.getIndexes()
+  const existingIndex = currentIndexes.indexes.find(
+    existing => existing.name === `search:${ctx.params.tableId}`
+  )
+  if (existingIndex) {
+    await db.deleteIndex(existingIndex)
   }
 
-  // has to run after, make sure it has _id
-  await runStaticFormulaChecks(tableToDelete, { deletion: true })
-  await cleanupAttachments(tableToDelete, { rows })
   return tableToDelete
 }
 
 exports.bulkImport = async function (ctx) {
-  const table = await getTable(ctx.params.tableId)
+  const appId = ctx.appId
+  const table = await getTable(appId, ctx.params.tableId)
   const { dataImport } = ctx.request.body
-  await handleDataImport(ctx.user, table, dataImport)
+  await handleDataImport(appId, ctx.user, table, dataImport)
   return table
 }