@budibase/server 1.0.49 → 1.0.50-alpha.0

package/src/api/controllers/row/utils.js

@@ -1,11 +1,11 @@
 const validateJs = require("validate.js")
 const { cloneDeep } = require("lodash/fp")
-const CouchDB = require("../../../db")
 const { InternalTables } = require("../../../db/utils")
 const userController = require("../user")
 const { FieldTypes } = require("../../../constants")
 const { processStringSync } = require("@budibase/string-templates")
 const { makeExternalQuery } = require("../../../integrations/base/utils")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value) {
@@ -17,14 +17,15 @@ validateJs.extend(validateJs.validators.datetime, {
   },
 })
 
-exports.getDatasourceAndQuery = async (appId, json) => {
+exports.getDatasourceAndQuery = async json => {
   const datasourceId = json.endpoint.datasourceId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const datasource = await db.get(datasourceId)
   return makeExternalQuery(datasource, json)
 }
 
-exports.findRow = async (ctx, db, tableId, rowId) => {
+exports.findRow = async (ctx, tableId, rowId) => {
+  const db = getAppDB()
   let row
   // TODO remove special user case in future
   if (tableId === InternalTables.USER_METADATA) {
@@ -42,9 +43,9 @@ exports.findRow = async (ctx, db, tableId, rowId) => {
   return row
 }
 
-exports.validate = async ({ appId, tableId, row, table }) => {
+exports.validate = async ({ tableId, row, table }) => {
   if (!table) {
-    const db = new CouchDB(appId)
+    const db = getAppDB()
     table = await db.get(tableId)
   }
   const errors = {}
@@ -58,22 +59,16 @@ exports.validate = async ({ appId, tableId, row, table }) => {
     let res
 
     // Validate.js doesn't seem to handle array
-    if (type === FieldTypes.ARRAY) {
-      const hasValues =
-        Array.isArray(row[fieldName]) && row[fieldName].length > 0
-
-      // Check values are valid if values are specified
-      if (hasValues) {
+    if (type === FieldTypes.ARRAY && row[fieldName]) {
+      if (row[fieldName].length) {
         row[fieldName].map(val => {
           if (!constraints.inclusion.includes(val)) {
-            errors[fieldName] = "Value not in list"
+            errors[fieldName] = "Field not in list"
           }
         })
-      }
-
-      // Check for required constraint
-      if (constraints.presence === true && !hasValues) {
-        errors[fieldName] = "Required field"
+      } else if (constraints.presence && row[fieldName].length === 0) {
+        // non required MultiSelect creates an empty array, which should not throw errors
+        errors[fieldName] = [`${fieldName} is required`]
       }
     } else if (type === FieldTypes.JSON && typeof row[fieldName] === "string") {
       // this should only happen if there is an error

package/src/api/controllers/screen.js

@@ -1,10 +1,9 @@
-const CouchDB = require("../../db")
 const { getScreenParams, generateScreenID } = require("../../db/utils")
 const { AccessController } = require("@budibase/backend-core/roles")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 exports.fetch = async ctx => {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
 
   const screens = (
     await db.allDocs(
@@ -14,15 +13,14 @@ exports.fetch = async ctx => {
     )
   ).rows.map(element => element.doc)
 
-  ctx.body = await new AccessController(appId).checkScreensAccess(
+  ctx.body = await new AccessController().checkScreensAccess(
     screens,
     ctx.user.role._id
   )
 }
 
 exports.save = async ctx => {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   let screen = ctx.request.body
 
   if (!screen._id) {
@@ -39,7 +37,7 @@ exports.save = async ctx => {
 }
 
 exports.destroy = async ctx => {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   await db.remove(ctx.params.screenId, ctx.params.screenRev)
   ctx.body = {
     message: "Screen deleted successfully",

package/src/api/controllers/static/index.js

@@ -5,8 +5,7 @@ const { resolve, join } = require("../../../utilities/centralPath")
 const uuid = require("uuid")
 const { ObjectStoreBuckets } = require("../../../constants")
 const { processString } = require("@budibase/string-templates")
-const { getDeployedApps } = require("../../../utilities/workerRequests")
-const CouchDB = require("../../../db")
+const { getAllApps } = require("@budibase/backend-core/db")
 const {
   loadHandlebarsFile,
   NODE_MODULES_PATH,
@@ -17,6 +16,9 @@ const { clientLibraryPath } = require("../../../utilities")
 const { upload } = require("../../../utilities/fileSystem")
 const { attachmentsRelativeURL } = require("../../../utilities")
 const { DocumentTypes } = require("../../../db/utils")
+const { getAppDB, updateAppId } = require("@budibase/backend-core/context")
+const AWS = require("aws-sdk")
+const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1"
 
 async function prepareUpload({ s3Key, bucket, metadata, file }) {
   const response = await upload({
@@ -37,15 +39,19 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) {
   }
 }
 
-async function checkForSelfHostedURL(ctx) {
-  // the "appId" component of the URL may actually be a specific self hosted URL
+async function getAppIdFromUrl(ctx) {
+  // the "appId" component of the URL can be the id or the custom url
   let possibleAppUrl = `/${encodeURI(ctx.params.appId).toLowerCase()}`
-  const apps = await getDeployedApps()
-  if (apps[possibleAppUrl] && apps[possibleAppUrl].appId) {
-    return apps[possibleAppUrl].appId
-  } else {
-    return ctx.params.appId
-  }
+
+  // search prod apps for a url that matches, exclude dev where id is always used
+  const apps = await getAllApps({ dev: false })
+  const app = apps.filter(
+    a => a.url && a.url.toLowerCase() === possibleAppUrl
+  )[0]
+
+  const appId = app && app.appId ? app.appId : ctx.params.appId
+  updateAppId(appId)
+  return appId
 }
 
 exports.serveBuilder = async function (ctx) {
@@ -75,12 +81,9 @@ exports.uploadFile = async function (ctx) {
 }
 
 exports.serveApp = async function (ctx) {
-  let appId = ctx.params.appId
-  if (env.SELF_HOSTED) {
-    appId = await checkForSelfHostedURL(ctx)
-  }
+  let appId = await getAppIdFromUrl(ctx)
   const App = require("./templates/BudibaseApp.svelte").default
-  const db = new CouchDB(appId, { skip_setup: true })
+  const db = getAppDB({ skip_setup: true })
   const appInfo = await db.get(DocumentTypes.APP_METADATA)
 
   const { head, html, css } = App.render({
@@ -104,3 +107,51 @@ exports.serveClientLibrary = async function (ctx) {
     root: join(NODE_MODULES_PATH, "@budibase", "client", "dist"),
   })
 }
+
+exports.getSignedUploadURL = async function (ctx) {
+  const database = getAppDB()
+
+  // Ensure datasource is valid
+  let datasource
+  try {
+    const { datasourceId } = ctx.params
+    datasource = await database.get(datasourceId)
+    if (!datasource) {
+      ctx.throw(400, "The specified datasource could not be found")
+    }
+  } catch (error) {
+    ctx.throw(400, "The specified datasource could not be found")
+  }
+
+  // Ensure we aren't using a custom endpoint
+  if (datasource?.config?.endpoint) {
+    ctx.throw(400, "S3 datasources with custom endpoints are not supported")
+  }
+
+  // Determine type of datasource and generate signed URL
+  let signedUrl
+  let publicUrl
+  if (datasource.source === "S3") {
+    const { bucket, key } = ctx.request.body || {}
+    if (!bucket || !key) {
+      ctx.throw(400, "bucket and key values are required")
+      return
+    }
+    try {
+      const s3 = new AWS.S3({
+        region: AWS_REGION,
+        accessKeyId: datasource?.config?.accessKeyId,
+        secretAccessKey: datasource?.config?.secretAccessKey,
+        apiVersion: "2006-03-01",
+        signatureVersion: "v4",
+      })
+      const params = { Bucket: bucket, Key: key }
+      signedUrl = s3.getSignedUrl("putObject", params)
+      publicUrl = `https://${bucket}.s3.${AWS_REGION}.amazonaws.com/${key}`
+    } catch (error) {
+      ctx.throw(400, error)
+    }
+  }
+
+  ctx.body = { signedUrl, publicUrl }
+}

package/src/api/controllers/table/bulkFormula.js

@@ -0,0 +1,173 @@
+const { FieldTypes, FormulaTypes } = require("../../../constants")
+const { getAllInternalTables, clearColumns } = require("./utils")
+const { doesContainStrings } = require("@budibase/string-templates")
+const { cloneDeep } = require("lodash/fp")
+const { isEqual, uniq } = require("lodash")
+const { updateAllFormulasInTable } = require("../row/staticFormula")
+const { getAppDB } = require("@budibase/backend-core/context")
+
+function isStaticFormula(column) {
+  return (
+    column.type === FieldTypes.FORMULA &&
+    column.formulaType === FormulaTypes.STATIC
+  )
+}
+
+/**
+ * This retrieves the formula columns from a table schema that use a specified column name
+ * in the formula.
+ */
+function getFormulaThatUseColumn(table, columnNames) {
+  let formula = []
+  columnNames = Array.isArray(columnNames) ? columnNames : [columnNames]
+  for (let column of Object.values(table.schema)) {
+    // not a static formula, or doesn't contain a relationship
+    if (!isStaticFormula(column)) {
+      continue
+    }
+    if (!doesContainStrings(column.formula, columnNames)) {
+      continue
+    }
+    formula.push(column.name)
+  }
+  return formula
+}
+
+/**
+ * This functions checks for when a related table, column or related column is deleted, if any
+ * tables need to have the formula column removed.
+ */
+async function checkIfFormulaNeedsCleared(table, { oldTable, deletion }) {
+  // start by retrieving all tables, remove the current table from the list
+  const tables = (await getAllInternalTables()).filter(
+    tbl => tbl._id !== table._id
+  )
+  const schemaToUse = oldTable ? oldTable.schema : table.schema
+  let removedColumns = Object.values(schemaToUse).filter(
+    column => deletion || !table.schema[column.name]
+  )
+  // remove any formula columns that used related columns
+  for (let removed of removedColumns) {
+    let tableToUse = table
+    // if relationship, get the related table
+    if (removed.type === FieldTypes.LINK) {
+      tableToUse = tables.find(table => table._id === removed.tableId)
+    }
+    const columnsToDelete = getFormulaThatUseColumn(tableToUse, removed.name)
+    if (columnsToDelete.length > 0) {
+      await clearColumns(table, columnsToDelete)
+    }
+    // need a special case, where a column has been removed from this table, but was used
+    // in a different, related tables formula
+    if (!table.relatedFormula) {
+      continue
+    }
+    for (let relatedTableId of table.relatedFormula) {
+      const relatedColumns = Object.values(table.schema).filter(
+        column => column.tableId === relatedTableId
+      )
+      const relatedTable = tables.find(table => table._id === relatedTableId)
+      // look to see if the column was used in a relationship formula,
+      // relationships won't be used for this
+      if (relatedTable && relatedColumns && removed.type !== FieldTypes.LINK) {
+        let relatedFormulaToRemove = []
+        for (let column of relatedColumns) {
+          relatedFormulaToRemove = relatedFormulaToRemove.concat(
+            getFormulaThatUseColumn(relatedTable, [
+              column.fieldName,
+              removed.name,
+            ])
+          )
+        }
+        if (relatedFormulaToRemove.length > 0) {
+          await clearColumns(relatedTable, uniq(relatedFormulaToRemove))
+        }
+      }
+    }
+  }
+}
+
+/**
+ * This function adds a note to related tables that they are
+ * used in a static formula - so that the link controller
+ * can manage hydrating related rows formula fields. This is
+ * specifically only for static formula.
+ */
+async function updateRelatedFormulaLinksOnTables(
+  table,
+  { deletion } = { deletion: false }
+) {
+  const db = getAppDB()
+  // start by retrieving all tables, remove the current table from the list
+  const tables = (await getAllInternalTables()).filter(
+    tbl => tbl._id !== table._id
+  )
+  // clone the tables, so we can compare at end
+  const initialTables = cloneDeep(tables)
+  // first find the related column names
+  const relatedColumns = Object.values(table.schema).filter(
+    col => col.type === FieldTypes.LINK
+  )
+  // we start by removing the formula field from all tables
+  for (let otherTable of tables) {
+    if (!otherTable.relatedFormula) {
+      continue
+    }
+    const index = otherTable.relatedFormula.indexOf(table._id)
+    if (index !== -1) {
+      otherTable.relatedFormula.splice(index, 1)
+    }
+  }
+  // if deleting, just remove the table IDs, don't try add
+  if (!deletion) {
+    for (let relatedCol of relatedColumns) {
+      let columns = getFormulaThatUseColumn(table, relatedCol.name)
+      if (!columns || columns.length === 0) {
+        continue
+      }
+      const relatedTable = tables.find(
+        related => related._id === relatedCol.tableId
+      )
+      // check if the table is already in the list of related formula, if it isn't, then add it
+      if (
+        relatedTable &&
+        (!relatedTable.relatedFormula ||
+          !relatedTable.relatedFormula.includes(table._id))
+      ) {
+        relatedTable.relatedFormula = relatedTable.relatedFormula
+          ? [...relatedTable.relatedFormula, table._id]
+          : [table._id]
+      }
+    }
+  }
+  // now we just need to compare all the tables and see if any need saved
+  for (let initial of initialTables) {
+    const found = tables.find(tbl => initial._id === tbl._id)
+    if (found && !isEqual(initial, found)) {
+      await db.put(found)
+    }
+  }
+}
+
+async function checkIfFormulaUpdated(table, { oldTable }) {
+  // look to see if any formula values have changed
+  const shouldUpdate = Object.values(table.schema).find(
+    column =>
+      isStaticFormula(column) &&
+      (!oldTable ||
+        !oldTable.schema[column.name] ||
+        !isEqual(oldTable.schema[column.name], column))
+  )
+  // if a static formula column has updated, then need to run the update
+  if (shouldUpdate != null) {
+    await updateAllFormulasInTable(table)
+  }
+}
+
+exports.runStaticFormulaChecks = async (table, { oldTable, deletion }) => {
+  await updateRelatedFormulaLinksOnTables(table, { deletion })
+  await checkIfFormulaNeedsCleared(table, { oldTable, deletion })
+  if (!deletion) {
+    await checkIfFormulaUpdated(table, { oldTable })
+  }
+}

package/src/api/controllers/table/external.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../../../db")
 const {
   buildExternalTableId,
   breakExternalTableId,
@@ -19,6 +18,7 @@ const { makeExternalQuery } = require("../../../integrations/base/utils")
 const { cloneDeep } = require("lodash/fp")
 const csvParser = require("../../../utilities/csvParser")
 const { handleRequest } = require("../row/external")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 async function makeTableRequest(
   datasource,
@@ -159,7 +159,6 @@ function isRelationshipSetup(column) {
 }
 
 exports.save = async function (ctx) {
-  const appId = ctx.appId
   const table = ctx.request.body
   // can't do this right now
   delete table.dataImport
@@ -176,14 +175,14 @@ exports.save = async function (ctx) {
 
   let oldTable
   if (ctx.request.body && ctx.request.body._id) {
-    oldTable = await getTable(appId, ctx.request.body._id)
+    oldTable = await getTable(ctx.request.body._id)
   }
 
   if (hasTypeChanged(tableToSave, oldTable)) {
     ctx.throw(400, "A column type has changed.")
   }
 
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const datasource = await db.get(datasourceId)
   const oldTables = cloneDeep(datasource.entities)
   const tables = datasource.entities
@@ -267,14 +266,13 @@ exports.save = async function (ctx) {
 }
 
 exports.destroy = async function (ctx) {
-  const appId = ctx.appId
-  const tableToDelete = await getTable(appId, ctx.params.tableId)
+  const tableToDelete = await getTable(ctx.params.tableId)
   if (!tableToDelete || !tableToDelete.created) {
     ctx.throw(400, "Cannot delete tables which weren't created in Budibase.")
   }
   const datasourceId = getDatasourceId(tableToDelete)
 
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const datasource = await db.get(datasourceId)
   const tables = datasource.entities
 
@@ -290,8 +288,7 @@ exports.destroy = async function (ctx) {
 }
 
 exports.bulkImport = async function (ctx) {
-  const appId = ctx.appId
-  const table = await getTable(appId, ctx.params.tableId)
+  const table = await getTable(ctx.params.tableId)
   const { dataImport } = ctx.request.body
   if (!dataImport || !dataImport.schema || !dataImport.csvString) {
     ctx.throw(400, "Provided data import information is invalid.")
@@ -300,7 +297,7 @@ exports.bulkImport = async function (ctx) {
     ...dataImport,
     existingTable: table,
   })
-  await handleRequest(appId, DataSourceOperation.BULK_CREATE, table._id, {
+  await handleRequest(DataSourceOperation.BULK_CREATE, table._id, {
     rows,
   })
   return table

package/src/api/controllers/table/index.js

@@ -1,14 +1,10 @@
-const CouchDB = require("../../../db")
 const internal = require("./internal")
 const external = require("./external")
 const csvParser = require("../../../utilities/csvParser")
-const { isExternalTable } = require("../../../integrations/utils")
-const {
-  getTableParams,
-  getDatasourceParams,
-  BudibaseInternalDB,
-} = require("../../../db/utils")
-const { getTable } = require("./utils")
+const { isExternalTable, isSQL } = require("../../../integrations/utils")
+const { getDatasourceParams } = require("../../../db/utils")
+const { getAppDB } = require("@budibase/backend-core/context")
+const { getTable, getAllInternalTables } = require("./utils")
 
 function pickApi({ tableId, table }) {
   if (table && !tableId) {
@@ -24,19 +20,9 @@ function pickApi({ tableId, table }) {
 
 // covers both internal and external
 exports.fetch = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
 
-  const internalTables = await db.allDocs(
-    getTableParams(null, {
-      include_docs: true,
-    })
-  )
-
-  const internal = internalTables.rows.map(row => ({
-    ...row.doc,
-    type: "internal",
-    sourceId: BudibaseInternalDB._id,
-  }))
+  const internal = await getAllInternalTables()
 
   const externalTables = await db.allDocs(
     getDatasourceParams("plus", {
@@ -44,12 +30,18 @@ exports.fetch = async function (ctx) {
     })
   )
 
-  const external = externalTables.rows.flatMap(row => {
-    return Object.values(row.doc.entities || {}).map(entity => ({
-      ...entity,
-      type: "external",
-      sourceId: row.doc._id,
-    }))
+  const external = externalTables.rows.flatMap(tableDoc => {
+    let entities = tableDoc.doc.entities
+    if (entities) {
+      return Object.values(entities).map(entity => ({
+        ...entity,
+        type: "external",
+        sourceId: tableDoc.doc._id,
+        sql: isSQL(tableDoc.doc),
+      }))
+    } else {
+      return []
+    }
   })
 
   ctx.body = [...internal, ...external]
@@ -57,7 +49,7 @@ exports.fetch = async function (ctx) {
 
 exports.find = async function (ctx) {
   const tableId = ctx.params.id
-  ctx.body = await getTable(ctx.appId, tableId)
+  ctx.body = await getTable(tableId)
 }
 
 exports.save = async function (ctx) {
@@ -96,7 +88,7 @@ exports.validateCSVSchema = async function (ctx) {
   const { csvString, schema = {}, tableId } = ctx.request.body
   let existingTable
   if (tableId) {
-    existingTable = await getTable(ctx.appId, tableId)
+    existingTable = await getTable(tableId)
   }
   let result = await csvParser.parse(csvString, schema)
   if (existingTable) {

package/src/api/controllers/table/internal.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../../../db")
 const linkRows = require("../../../db/linkedRows")
 const { getRowParams, generateTableID } = require("../../../db/utils")
 const { FieldTypes } = require("../../../constants")
@@ -8,10 +7,14 @@ const {
   getTable,
   handleDataImport,
 } = require("./utils")
+const usageQuota = require("../../../utilities/usageQuota")
+const { getAppDB } = require("@budibase/backend-core/context")
+const env = require("../../../environment")
+const { cleanupAttachments } = require("../../../utilities/rowProcessor")
+const { runStaticFormulaChecks } = require("./bulkFormula")
 
 exports.save = async function (ctx) {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const { dataImport, ...rest } = ctx.request.body
   let tableToSave = {
     type: "table",
@@ -33,8 +36,7 @@ exports.save = async function (ctx) {
   // saving a table is a complex operation, involving many different steps, this
   // has been broken out into a utility to make it more obvious/easier to manipulate
   const tableSaveFunctions = new TableSaveFunctions({
-    db,
-    ctx,
+    user: ctx.user,
     oldTable,
     dataImport,
   })
@@ -79,7 +81,6 @@ exports.save = async function (ctx) {
   // update linked rows
   try {
     const linkResp = await linkRows.updateLinks({
-      appId,
       eventType: oldTable
         ? linkRows.EventType.TABLE_UPDATED
         : linkRows.EventType.TABLE_SAVE,
@@ -103,13 +104,13 @@ exports.save = async function (ctx) {
   tableToSave._rev = result.rev
 
   tableToSave = await tableSaveFunctions.after(tableToSave)
-
+  // has to run after, make sure it has _id
+  await runStaticFormulaChecks(tableToSave, { oldTable })
   return tableToSave
 }
 
 exports.destroy = async function (ctx) {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const tableToDelete = await db.get(ctx.params.tableId)
 
   // Delete all rows for that table
@@ -119,10 +120,10 @@ exports.destroy = async function (ctx) {
     })
   )
   await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
+  await usageQuota.update(usageQuota.Properties.ROW, -rows.rows.length)
 
   // update linked rows
   await linkRows.updateLinks({
-    appId,
     eventType: linkRows.EventType.TABLE_DELETE,
     table: tableToDelete,
   })
@@ -131,21 +132,25 @@ exports.destroy = async function (ctx) {
   await db.remove(tableToDelete)
 
   // remove table search index
-  const currentIndexes = await db.getIndexes()
-  const existingIndex = currentIndexes.indexes.find(
-    existing => existing.name === `search:${ctx.params.tableId}`
-  )
-  if (existingIndex) {
-    await db.deleteIndex(existingIndex)
+  if (!env.isTest()) {
+    const currentIndexes = await db.getIndexes()
+    const existingIndex = currentIndexes.indexes.find(
+      existing => existing.name === `search:${ctx.params.tableId}`
+    )
+    if (existingIndex) {
+      await db.deleteIndex(existingIndex)
+    }
   }
 
+  // has to run after, make sure it has _id
+  await runStaticFormulaChecks(tableToDelete, { deletion: true })
+  await cleanupAttachments(tableToDelete, { rows })
   return tableToDelete
 }
 
 exports.bulkImport = async function (ctx) {
-  const appId = ctx.appId
-  const table = await getTable(appId, ctx.params.tableId)
+  const table = await getTable(ctx.params.tableId)
   const { dataImport } = ctx.request.body
-  await handleDataImport(appId, ctx.user, table, dataImport)
+  await handleDataImport(ctx.user, table, dataImport)
   return table
 }