@budibase/server 1.0.48 → 1.0.49-alpha.3

package/src/api/controllers/row/internal.js

@@ -1,8 +1,8 @@
 const CouchDB = require("../../../db")
 const linkRows = require("../../../db/linkedRows")
 const {
-  getRowParams,
   generateRowID,
+  getRowParams,
   DocumentTypes,
   InternalTables,
 } = require("../../../db/utils")
@@ -10,11 +10,9 @@ const userController = require("../user")
 const {
   inputProcessing,
   outputProcessing,
-  processAutoColumn,
   cleanupAttachments,
 } = require("../../../utilities/rowProcessor")
 const { FieldTypes } = require("../../../constants")
-const { isEqual } = require("lodash")
 const { validate, findRow } = require("./utils")
 const { fullSearch, paginatedSearch } = require("./internalSearch")
 const { getGlobalUsersFromMetadata } = require("../../../utilities/global")
@@ -27,6 +25,7 @@ const {
   getFromMemoryDoc,
 } = require("../view/utils")
 const { cloneDeep } = require("lodash/fp")
+const { finaliseRow, updateRelatedFormula } = require("./staticFormula")
 
 const CALCULATION_TYPES = {
   SUM: "sum",
@@ -34,51 +33,6 @@ const CALCULATION_TYPES = {
   STATS: "stats",
 }
 
-async function storeResponse(ctx, db, row, oldTable, table) {
-  row.type = "row"
-  // don't worry about rev, tables handle rev/lastID updates
-  // if another row has been written since processing this will
-  // handle the auto ID clash
-  if (!isEqual(oldTable, table)) {
-    try {
-      await db.put(table)
-    } catch (err) {
-      if (err.status === 409) {
-        const updatedTable = await db.get(table._id)
-        let response = processAutoColumn(null, updatedTable, row, {
-          reprocessing: true,
-        })
-        await db.put(response.table)
-        row = response.row
-      } else {
-        throw err
-      }
-    }
-  }
-  const response = await db.put(row)
-  row._rev = response.rev
-  // process the row before return, to include relationships
-  row = await outputProcessing(ctx, table, row, { squash: false })
-  return { row, table }
-}
-
-// doesn't do the outputProcessing
-async function getRawTableData(ctx, db, tableId) {
-  let rows
-  if (tableId === InternalTables.USER_METADATA) {
-    await userController.fetchMetadata(ctx)
-    rows = ctx.body
-  } else {
-    const response = await db.allDocs(
-      getRowParams(tableId, null, {
-        include_docs: true,
-      })
-    )
-    rows = response.rows.map(row => row.doc)
-  }
-  return rows
-}
-
 async function getView(db, viewName) {
   let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
   let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
@@ -105,6 +59,22 @@ async function getView(db, viewName) {
   return viewInfo
 }
 
+async function getRawTableData(ctx, db, tableId) {
+  let rows
+  if (tableId === InternalTables.USER_METADATA) {
+    await userController.fetchMetadata(ctx)
+    rows = ctx.body
+  } else {
+    const response = await db.allDocs(
+      getRowParams(tableId, null, {
+        include_docs: true,
+      })
+    )
+    rows = response.rows.map(row => row.doc)
+  }
+  return rows
+}
+
 exports.patch = async ctx => {
   const appId = ctx.appId
   const db = new CouchDB(appId)
@@ -162,7 +132,10 @@ exports.patch = async ctx => {
     return { row: ctx.body, table }
   }
 
-  return storeResponse(ctx, db, row, dbTable, table)
+  return finaliseRow(ctx.appId, table, row, {
+    oldTable: dbTable,
+    updateFormula: true,
+  })
 }
 
 exports.save = async function (ctx) {
@@ -196,7 +169,10 @@ exports.save = async function (ctx) {
     table,
   })
 
-  return storeResponse(ctx, db, row, dbTable, table)
+  return finaliseRow(ctx.appId, table, row, {
+    oldTable: dbTable,
+    updateFormula: true,
+  })
 }
 
 exports.fetchView = async ctx => {
@@ -302,6 +278,8 @@ exports.destroy = async function (ctx) {
   })
   // remove any attachments that were on the row from object storage
   await cleanupAttachments(appId, table, { row })
+  // remove any static formula
+  await updateRelatedFormula(appId, table, row)
 
   let response
   if (ctx.params.tableId === InternalTables.USER_METADATA) {
@@ -350,6 +328,7 @@ exports.bulkDestroy = async ctx => {
   }
   // remove any attachments that were on the rows from object storage
   await cleanupAttachments(appId, table, { rows })
+  await updateRelatedFormula(appId, table, rows)
   await Promise.all(updates)
   return { response: { ok: true }, rows }
 }

package/src/api/controllers/row/internalSearch.js

@@ -37,22 +37,30 @@ class QueryBuilder {
   }
 
   setLimit(limit) {
-    this.limit = limit
+    if (limit != null) {
+      this.limit = limit
+    }
     return this
   }
 
   setSort(sort) {
-    this.sort = sort
+    if (sort != null) {
+      this.sort = sort
+    }
     return this
   }
 
   setSortOrder(sortOrder) {
-    this.sortOrder = sortOrder
+    if (sortOrder != null) {
+      this.sortOrder = sortOrder
+    }
     return this
   }
 
   setSortType(sortType) {
-    this.sortType = sortType
+    if (sortType != null) {
+      this.sortType = sortType
+    }
     return this
   }
 

package/src/api/controllers/row/staticFormula.js

@@ -0,0 +1,157 @@
+const CouchDB = require("../../../db")
+const { getRowParams } = require("../../../db/utils")
+const {
+  outputProcessing,
+  processAutoColumn,
+  processFormulas,
+} = require("../../../utilities/rowProcessor")
+const { FieldTypes, FormulaTypes } = require("../../../constants")
+const { isEqual } = require("lodash")
+const { cloneDeep } = require("lodash/fp")
+
+/**
+ * This function runs through a list of enriched rows, looks at the rows which
+ * are related and then checks if they need the state of their formulas
+ * updated.
+ * NOTE: this will only for affect static formulas.
+ */
+exports.updateRelatedFormula = async (appId, table, enrichedRows) => {
+  const db = new CouchDB(appId)
+  // no formula to update, we're done
+  if (!table.relatedFormula) {
+    return
+  }
+  let promises = []
+  for (let enrichedRow of Array.isArray(enrichedRows)
+    ? enrichedRows
+    : [enrichedRows]) {
+    // the related rows by tableId
+    let relatedRows = {}
+    for (let [key, field] of Object.entries(enrichedRow)) {
+      const columnDefinition = table.schema[key]
+      if (columnDefinition && columnDefinition.type === FieldTypes.LINK) {
+        const relatedTableId = columnDefinition.tableId
+        if (!relatedRows[relatedTableId]) {
+          relatedRows[relatedTableId] = []
+        }
+        relatedRows[relatedTableId] = relatedRows[relatedTableId].concat(field)
+      }
+    }
+    for (let tableId of table.relatedFormula) {
+      let relatedTable
+      try {
+        // no rows to update, skip
+        if (!relatedRows[tableId] || relatedRows[tableId].length === 0) {
+          continue
+        }
+        relatedTable = await db.get(tableId)
+      } catch (err) {
+        // no error scenario, table doesn't seem to exist anymore, ignore
+      }
+      for (let column of Object.values(relatedTable.schema)) {
+        // needs updated in related rows
+        if (
+          column.type === FieldTypes.FORMULA &&
+          column.formulaType === FormulaTypes.STATIC
+        ) {
+          // re-enrich rows for all the related, don't update the related formula for them
+          promises = promises.concat(
+            relatedRows[tableId].map(related =>
+              exports.finaliseRow(appId, relatedTable, related, {
+                updateFormula: false,
+              })
+            )
+          )
+        }
+      }
+    }
+  }
+  await Promise.all(promises)
+}
+
+exports.updateAllFormulasInTable = async (appId, table) => {
+  const db = new CouchDB(appId)
+  // start by getting the raw rows (which will be written back to DB after update)
+  let rows = (
+    await db.allDocs(
+      getRowParams(table._id, null, {
+        include_docs: true,
+      })
+    )
+  ).rows.map(row => row.doc)
+  // now enrich the rows, note the clone so that we have the base state of the
+  // rows so that we don't write any of the enriched information back
+  let enrichedRows = await outputProcessing({ appId }, table, cloneDeep(rows), {
+    squash: false,
+  })
+  const updatedRows = []
+  for (let row of rows) {
+    // find the enriched row, if found process the formulas
+    const enrichedRow = enrichedRows.find(enriched => enriched._id === row._id)
+    if (enrichedRow) {
+      const processed = processFormulas(table, cloneDeep(row), {
+        dynamic: false,
+        contextRows: enrichedRow,
+      })
+      // values have changed, need to add to bulk docs to update
+      if (!isEqual(processed, row)) {
+        updatedRows.push(processed)
+      }
+    }
+  }
+  await db.bulkDocs(updatedRows)
+}
+
+/**
+ * This function runs at the end of the save/patch functions of the row controller, all this
+ * really does is enrich the row, handle any static formula processing, then return the enriched
+ * row. The reason we need to return the enriched row is that the automation row created trigger
+ * expects the row to be totally enriched/contain all relationships.
+ */
+exports.finaliseRow = async (
+  appId,
+  table,
+  row,
+  { oldTable, updateFormula } = { updateFormula: true }
+) => {
+  const db = new CouchDB(appId)
+  row.type = "row"
+  // process the row before return, to include relationships
+  let enrichedRow = await outputProcessing({ appId }, table, cloneDeep(row), {
+    squash: false,
+  })
+  // use enriched row to generate formulas for saving, specifically only use as context
+  row = processFormulas(table, row, {
+    dynamic: false,
+    contextRows: enrichedRow,
+  })
+
+  // don't worry about rev, tables handle rev/lastID updates
+  // if another row has been written since processing this will
+  // handle the auto ID clash
+  if (oldTable && !isEqual(oldTable, table)) {
+    try {
+      await db.put(table)
+    } catch (err) {
+      if (err.status === 409) {
+        const updatedTable = await db.get(table._id)
+        let response = processAutoColumn(null, updatedTable, row, {
+          reprocessing: true,
+        })
+        await db.put(response.table)
+        row = response.row
+      } else {
+        throw err
+      }
+    }
+  }
+  const response = await db.put(row)
+  // for response, calculate the formulas for the enriched row
+  enrichedRow._rev = response.rev
+  enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false })
+  // this updates the related formulas in other rows based on the relations to this row
+  if (updateFormula) {
+    await exports.updateRelatedFormula(appId, table, enrichedRow)
+  }
+  return { row: enrichedRow, table }
+}

package/src/api/controllers/row/utils.js

@@ -58,22 +58,16 @@ exports.validate = async ({ appId, tableId, row, table }) => {
     let res
 
     // Validate.js doesn't seem to handle array
-    if (type === FieldTypes.ARRAY) {
-      const hasValues =
-        Array.isArray(row[fieldName]) && row[fieldName].length > 0
-
-      // Check values are valid if values are specified
-      if (hasValues) {
+    if (type === FieldTypes.ARRAY && row[fieldName]) {
+      if (row[fieldName].length) {
         row[fieldName].map(val => {
           if (!constraints.inclusion.includes(val)) {
-            errors[fieldName] = "Value not in list"
+            errors[fieldName] = "Field not in list"
           }
         })
-      }
-
-      // Check for required constraint
-      if (constraints.presence === true && !hasValues) {
-        errors[fieldName] = "Required field"
+      } else if (constraints.presence && row[fieldName].length === 0) {
+        // non required MultiSelect creates an empty array, which should not throw errors
+        errors[fieldName] = [`${fieldName} is required`]
       }
     } else if (type === FieldTypes.JSON && typeof row[fieldName] === "string") {
       // this should only happen if there is an error

package/src/api/controllers/static/index.js

@@ -5,7 +5,7 @@ const { resolve, join } = require("../../../utilities/centralPath")
 const uuid = require("uuid")
 const { ObjectStoreBuckets } = require("../../../constants")
 const { processString } = require("@budibase/string-templates")
-const { getDeployedApps } = require("../../../utilities/workerRequests")
+const { getAllApps } = require("@budibase/backend-core/db")
 const CouchDB = require("../../../db")
 const {
   loadHandlebarsFile,
@@ -17,6 +17,8 @@ const { clientLibraryPath } = require("../../../utilities")
 const { upload } = require("../../../utilities/fileSystem")
 const { attachmentsRelativeURL } = require("../../../utilities")
 const { DocumentTypes } = require("../../../db/utils")
+const AWS = require("aws-sdk")
+const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1"
 
 async function prepareUpload({ s3Key, bucket, metadata, file }) {
   const response = await upload({
@@ -37,12 +39,18 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) {
   }
 }
 
-async function checkForSelfHostedURL(ctx) {
-  // the "appId" component of the URL may actually be a specific self hosted URL
+async function getAppIdFromUrl(ctx) {
+  // the "appId" component of the URL can be the id or the custom url
   let possibleAppUrl = `/${encodeURI(ctx.params.appId).toLowerCase()}`
-  const apps = await getDeployedApps()
-  if (apps[possibleAppUrl] && apps[possibleAppUrl].appId) {
-    return apps[possibleAppUrl].appId
+
+  // search prod apps for a url that matches, exclude dev where id is always used
+  const apps = await getAllApps(CouchDB, { dev: false })
+  const app = apps.filter(
+    a => a.url && a.url.toLowerCase() === possibleAppUrl
+  )[0]
+
+  if (app && app.appId) {
+    return app.appId
   } else {
     return ctx.params.appId
   }
@@ -75,10 +83,7 @@ exports.uploadFile = async function (ctx) {
 }
 
 exports.serveApp = async function (ctx) {
-  let appId = ctx.params.appId
-  if (env.SELF_HOSTED) {
-    appId = await checkForSelfHostedURL(ctx)
-  }
+  let appId = await getAppIdFromUrl(ctx)
   const App = require("./templates/BudibaseApp.svelte").default
   const db = new CouchDB(appId, { skip_setup: true })
   const appInfo = await db.get(DocumentTypes.APP_METADATA)
@@ -104,3 +109,51 @@ exports.serveClientLibrary = async function (ctx) {
     root: join(NODE_MODULES_PATH, "@budibase", "client", "dist"),
   })
 }
+
+exports.getSignedUploadURL = async function (ctx) {
+  const database = new CouchDB(ctx.appId)
+
+  // Ensure datasource is valid
+  let datasource
+  try {
+    const { datasourceId } = ctx.params
+    datasource = await database.get(datasourceId)
+    if (!datasource) {
+      ctx.throw(400, "The specified datasource could not be found")
+    }
+  } catch (error) {
+    ctx.throw(400, "The specified datasource could not be found")
+  }
+
+  // Ensure we aren't using a custom endpoint
+  if (datasource?.config?.endpoint) {
+    ctx.throw(400, "S3 datasources with custom endpoints are not supported")
+  }
+
+  // Determine type of datasource and generate signed URL
+  let signedUrl
+  let publicUrl
+  if (datasource.source === "S3") {
+    const { bucket, key } = ctx.request.body || {}
+    if (!bucket || !key) {
+      ctx.throw(400, "bucket and key values are required")
+      return
+    }
+    try {
+      const s3 = new AWS.S3({
+        region: AWS_REGION,
+        accessKeyId: datasource?.config?.accessKeyId,
+        secretAccessKey: datasource?.config?.secretAccessKey,
+        apiVersion: "2006-03-01",
+        signatureVersion: "v4",
+      })
+      const params = { Bucket: bucket, Key: key }
+      signedUrl = s3.getSignedUrl("putObject", params)
+      publicUrl = `https://${bucket}.s3.${AWS_REGION}.amazonaws.com/${key}`
+    } catch (error) {
+      ctx.throw(400, error)
+    }
+  }
+
+  ctx.body = { signedUrl, publicUrl }
+}

package/src/api/controllers/table/bulkFormula.js

@@ -0,0 +1,183 @@
+const CouchDB = require("../../../db")
+const { FieldTypes, FormulaTypes } = require("../../../constants")
+const { getAllInternalTables, clearColumns } = require("./utils")
+const { doesContainStrings } = require("@budibase/string-templates")
+const { cloneDeep } = require("lodash/fp")
+const { isEqual, uniq } = require("lodash")
+const { updateAllFormulasInTable } = require("../row/staticFormula")
+
+function isStaticFormula(column) {
+  return (
+    column.type === FieldTypes.FORMULA &&
+    column.formulaType === FormulaTypes.STATIC
+  )
+}
+
+/**
+ * This retrieves the formula columns from a table schema that use a specified column name
+ * in the formula.
+ */
+function getFormulaThatUseColumn(table, columnNames) {
+  let formula = []
+  columnNames = Array.isArray(columnNames) ? columnNames : [columnNames]
+  for (let column of Object.values(table.schema)) {
+    // not a static formula, or doesn't contain a relationship
+    if (!isStaticFormula(column)) {
+      continue
+    }
+    if (!doesContainStrings(column.formula, columnNames)) {
+      continue
+    }
+    formula.push(column.name)
+  }
+  return formula
+}
+
+/**
+ * This functions checks for when a related table, column or related column is deleted, if any
+ * tables need to have the formula column removed.
+ */
+async function checkIfFormulaNeedsCleared(
+  appId,
+  table,
+  { oldTable, deletion }
+) {
+  const db = new CouchDB(appId)
+  // start by retrieving all tables, remove the current table from the list
+  const tables = (await getAllInternalTables(appId)).filter(
+    tbl => tbl._id !== table._id
+  )
+  const schemaToUse = oldTable ? oldTable.schema : table.schema
+  let removedColumns = Object.values(schemaToUse).filter(
+    column => deletion || !table.schema[column.name]
+  )
+  // remove any formula columns that used related columns
+  for (let removed of removedColumns) {
+    let tableToUse = table
+    // if relationship, get the related table
+    if (removed.type === FieldTypes.LINK) {
+      tableToUse = tables.find(table => table._id === removed.tableId)
+    }
+    const columnsToDelete = getFormulaThatUseColumn(tableToUse, removed.name)
+    if (columnsToDelete.length > 0) {
+      await clearColumns(db, table, columnsToDelete)
+    }
+    // need a special case, where a column has been removed from this table, but was used
+    // in a different, related tables formula
+    if (!table.relatedFormula) {
+      continue
+    }
+    for (let relatedTableId of table.relatedFormula) {
+      const relatedColumns = Object.values(table.schema).filter(
+        column => column.tableId === relatedTableId
+      )
+      const relatedTable = tables.find(table => table._id === relatedTableId)
+      // look to see if the column was used in a relationship formula,
+      // relationships won't be used for this
+      if (relatedTable && relatedColumns && removed.type !== FieldTypes.LINK) {
+        let relatedFormulaToRemove = []
+        for (let column of relatedColumns) {
+          relatedFormulaToRemove = relatedFormulaToRemove.concat(
+            getFormulaThatUseColumn(relatedTable, [
+              column.fieldName,
+              removed.name,
+            ])
+          )
+        }
+        if (relatedFormulaToRemove.length > 0) {
+          await clearColumns(db, relatedTable, uniq(relatedFormulaToRemove))
+        }
+      }
+    }
+  }
+}
+
+/**
+ * This function adds a note to related tables that they are
+ * used in a static formula - so that the link controller
+ * can manage hydrating related rows formula fields. This is
+ * specifically only for static formula.
+ */
+async function updateRelatedFormulaLinksOnTables(
+  appId,
+  table,
+  { deletion } = { deletion: false }
+) {
+  const db = new CouchDB(appId)
+  // start by retrieving all tables, remove the current table from the list
+  const tables = (await getAllInternalTables(appId)).filter(
+    tbl => tbl._id !== table._id
+  )
+  // clone the tables, so we can compare at end
+  const initialTables = cloneDeep(tables)
+  // first find the related column names
+  const relatedColumns = Object.values(table.schema).filter(
+    col => col.type === FieldTypes.LINK
+  )
+  // we start by removing the formula field from all tables
+  for (let otherTable of tables) {
+    if (!otherTable.relatedFormula) {
+      continue
+    }
+    const index = otherTable.relatedFormula.indexOf(table._id)
+    if (index !== -1) {
+      otherTable.relatedFormula.splice(index, 1)
+    }
+  }
+  // if deleting, just remove the table IDs, don't try add
+  if (!deletion) {
+    for (let relatedCol of relatedColumns) {
+      let columns = getFormulaThatUseColumn(table, relatedCol.name)
+      if (!columns || columns.length === 0) {
+        continue
+      }
+      const relatedTable = tables.find(
+        related => related._id === relatedCol.tableId
+      )
+      // check if the table is already in the list of related formula, if it isn't, then add it
+      if (
+        relatedTable &&
+        (!relatedTable.relatedFormula ||
+          !relatedTable.relatedFormula.includes(table._id))
+      ) {
+        relatedTable.relatedFormula = relatedTable.relatedFormula
+          ? [...relatedTable.relatedFormula, table._id]
+          : [table._id]
+      }
+    }
+  }
+  // now we just need to compare all the tables and see if any need saved
+  for (let initial of initialTables) {
+    const found = tables.find(tbl => initial._id === tbl._id)
+    if (found && !isEqual(initial, found)) {
+      await db.put(found)
+    }
+  }
+}
+
+async function checkIfFormulaUpdated(appId, table, { oldTable }) {
+  // look to see if any formula values have changed
+  const shouldUpdate = Object.values(table.schema).find(
+    column =>
+      isStaticFormula(column) &&
+      (!oldTable ||
+        !oldTable.schema[column.name] ||
+        !isEqual(oldTable.schema[column.name], column))
+  )
+  // if a static formula column has updated, then need to run the update
+  if (shouldUpdate != null) {
+    await updateAllFormulasInTable(appId, table)
+  }
+}
+
+exports.runStaticFormulaChecks = async (
+  appId,
+  table,
+  { oldTable, deletion }
+) => {
+  await updateRelatedFormulaLinksOnTables(appId, table, { deletion })
+  await checkIfFormulaNeedsCleared(appId, table, { oldTable, deletion })
+  if (!deletion) {
+    await checkIfFormulaUpdated(appId, table, { oldTable })
+  }
+}