@budibase/server 1.0.49 → 1.0.50-alpha.0

package/src/api/controllers/row/internal.js

@@ -1,8 +1,7 @@
-const CouchDB = require("../../../db")
 const linkRows = require("../../../db/linkedRows")
 const {
-  getRowParams,
   generateRowID,
+  getRowParams,
   DocumentTypes,
   InternalTables,
 } = require("../../../db/utils")
@@ -10,11 +9,9 @@ const userController = require("../user")
 const {
   inputProcessing,
   outputProcessing,
-  processAutoColumn,
   cleanupAttachments,
 } = require("../../../utilities/rowProcessor")
 const { FieldTypes } = require("../../../constants")
-const { isEqual } = require("lodash")
 const { validate, findRow } = require("./utils")
 const { fullSearch, paginatedSearch } = require("./internalSearch")
 const { getGlobalUsersFromMetadata } = require("../../../utilities/global")
@@ -27,6 +24,8 @@ const {
   getFromMemoryDoc,
 } = require("../view/utils")
 const { cloneDeep } = require("lodash/fp")
+const { getAppDB } = require("@budibase/backend-core/context")
+const { finaliseRow, updateRelatedFormula } = require("./staticFormula")
 
 const CALCULATION_TYPES = {
   SUM: "sum",
@@ -34,51 +33,6 @@ const CALCULATION_TYPES = {
   STATS: "stats",
 }
 
-async function storeResponse(ctx, db, row, oldTable, table) {
-  row.type = "row"
-  // don't worry about rev, tables handle rev/lastID updates
-  // if another row has been written since processing this will
-  // handle the auto ID clash
-  if (!isEqual(oldTable, table)) {
-    try {
-      await db.put(table)
-    } catch (err) {
-      if (err.status === 409) {
-        const updatedTable = await db.get(table._id)
-        let response = processAutoColumn(null, updatedTable, row, {
-          reprocessing: true,
-        })
-        await db.put(response.table)
-        row = response.row
-      } else {
-        throw err
-      }
-    }
-  }
-  const response = await db.put(row)
-  row._rev = response.rev
-  // process the row before return, to include relationships
-  row = await outputProcessing(ctx, table, row, { squash: false })
-  return { row, table }
-}
-
-// doesn't do the outputProcessing
-async function getRawTableData(ctx, db, tableId) {
-  let rows
-  if (tableId === InternalTables.USER_METADATA) {
-    await userController.fetchMetadata(ctx)
-    rows = ctx.body
-  } else {
-    const response = await db.allDocs(
-      getRowParams(tableId, null, {
-        include_docs: true,
-      })
-    )
-    rows = response.rows.map(row => row.doc)
-  }
-  return rows
-}
-
 async function getView(db, viewName) {
   let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
   let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
@@ -105,9 +59,24 @@ async function getView(db, viewName) {
   return viewInfo
 }
 
+async function getRawTableData(ctx, db, tableId) {
+  let rows
+  if (tableId === InternalTables.USER_METADATA) {
+    await userController.fetchMetadata(ctx)
+    rows = ctx.body
+  } else {
+    const response = await db.allDocs(
+      getRowParams(tableId, null, {
+        include_docs: true,
+      })
+    )
+    rows = response.rows.map(row => row.doc)
+  }
+  return rows
+}
+
 exports.patch = async ctx => {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const inputs = ctx.request.body
   const tableId = inputs.tableId
   const isUserTable = tableId === InternalTables.USER_METADATA
@@ -146,14 +115,13 @@ exports.patch = async ctx => {
 
   // returned row is cleaned and prepared for writing to DB
   row = await linkRows.updateLinks({
-    appId,
     eventType: linkRows.EventType.ROW_UPDATE,
     row,
     tableId: row.tableId,
     table,
   })
   // check if any attachments removed
-  await cleanupAttachments(appId, table, { oldRow, row })
+  await cleanupAttachments(table, { oldRow, row })
 
   if (isUserTable) {
     // the row has been updated, need to put it into the ctx
@@ -162,12 +130,14 @@ exports.patch = async ctx => {
     return { row: ctx.body, table }
   }
 
-  return storeResponse(ctx, db, row, dbTable, table)
+  return finaliseRow(table, row, {
+    oldTable: dbTable,
+    updateFormula: true,
+  })
 }
 
 exports.save = async function (ctx) {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   let inputs = ctx.request.body
   inputs.tableId = ctx.params.tableId
 
@@ -189,18 +159,19 @@ exports.save = async function (ctx) {
 
   // make sure link rows are up to date
   row = await linkRows.updateLinks({
-    appId,
     eventType: linkRows.EventType.ROW_SAVE,
     row,
     tableId: row.tableId,
     table,
   })
 
-  return storeResponse(ctx, db, row, dbTable, table)
+  return finaliseRow(table, row, {
+    oldTable: dbTable,
+    updateFormula: true,
+  })
 }
 
 exports.fetchView = async ctx => {
-  const appId = ctx.appId
   const viewName = ctx.params.viewName
 
   // if this is a table view being looked for just transfer to that
@@ -209,7 +180,7 @@ exports.fetchView = async ctx => {
     return exports.fetch(ctx)
   }
 
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const { calculation, group, field } = ctx.query
   const viewInfo = await getView(db, viewName)
   let response
@@ -236,7 +207,7 @@ exports.fetchView = async ctx => {
         schema: {},
       }
     }
-    rows = await outputProcessing(ctx, table, response.rows)
+    rows = await outputProcessing(table, response.rows)
   }
 
   if (calculation === CALCULATION_TYPES.STATS) {
@@ -263,27 +234,24 @@ exports.fetchView = async ctx => {
 }
 
 exports.fetch = async ctx => {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
 
   const tableId = ctx.params.tableId
   let table = await db.get(tableId)
   let rows = await getRawTableData(ctx, db, tableId)
-  return outputProcessing(ctx, table, rows)
+  return outputProcessing(table, rows)
 }
 
 exports.find = async ctx => {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const table = await db.get(ctx.params.tableId)
-  let row = await findRow(ctx, db, ctx.params.tableId, ctx.params.rowId)
-  row = await outputProcessing(ctx, table, row)
+  let row = await findRow(ctx, ctx.params.tableId, ctx.params.rowId)
+  row = await outputProcessing(table, row)
   return row
 }
 
 exports.destroy = async function (ctx) {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const { _id, _rev } = ctx.request.body
   let row = await db.get(_id)
 
@@ -292,16 +260,17 @@ exports.destroy = async function (ctx) {
   }
   const table = await db.get(row.tableId)
   // update the row to include full relationships before deleting them
-  row = await outputProcessing(ctx, table, row, { squash: false })
+  row = await outputProcessing(table, row, { squash: false })
   // now remove the relationships
   await linkRows.updateLinks({
-    appId,
     eventType: linkRows.EventType.ROW_DELETE,
     row,
     tableId: row.tableId,
   })
   // remove any attachments that were on the row from object storage
-  await cleanupAttachments(appId, table, { row })
+  await cleanupAttachments(table, { row })
+  // remove any static formula
+  await updateRelatedFormula(table, row)
 
   let response
   if (ctx.params.tableId === InternalTables.USER_METADATA) {
@@ -317,20 +286,18 @@ exports.destroy = async function (ctx) {
 }
 
 exports.bulkDestroy = async ctx => {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const tableId = ctx.params.tableId
   const table = await db.get(tableId)
   let { rows } = ctx.request.body
 
   // before carrying out any updates, make sure the rows are ready to be returned
   // they need to be the full rows (including previous relationships) for automations
-  rows = await outputProcessing(ctx, table, rows, { squash: false })
+  rows = await outputProcessing(table, rows, { squash: false })
 
   // remove the relationships first
   let updates = rows.map(row =>
     linkRows.updateLinks({
-      appId,
       eventType: linkRows.EventType.ROW_DELETE,
       row,
       tableId: row.tableId,
@@ -349,7 +316,8 @@ exports.bulkDestroy = async ctx => {
     await db.bulkDocs(rows.map(row => ({ ...row, _deleted: true })))
   }
   // remove any attachments that were on the rows from object storage
-  await cleanupAttachments(appId, table, { rows })
+  await cleanupAttachments(table, { rows })
+  await updateRelatedFormula(table, rows)
   await Promise.all(updates)
   return { response: { ok: true }, rows }
 }
@@ -360,28 +328,27 @@ exports.search = async ctx => {
     return { rows: await exports.fetch(ctx) }
   }
 
-  const appId = ctx.appId
   const { tableId } = ctx.params
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const { paginate, query, ...params } = ctx.request.body
   params.version = ctx.version
   params.tableId = tableId
 
   let response
   if (paginate) {
-    response = await paginatedSearch(appId, query, params)
+    response = await paginatedSearch(query, params)
   } else {
-    response = await fullSearch(appId, query, params)
+    response = await fullSearch(query, params)
   }
 
   // Enrich search results with relationships
   if (response.rows && response.rows.length) {
     // enrich with global users if from users table
     if (tableId === InternalTables.USER_METADATA) {
-      response.rows = await getGlobalUsersFromMetadata(appId, response.rows)
+      response.rows = await getGlobalUsersFromMetadata(response.rows)
     }
     const table = await db.get(tableId)
-    response.rows = await outputProcessing(ctx, table, response.rows)
+    response.rows = await outputProcessing(table, response.rows)
   }
 
   return response
@@ -389,25 +356,22 @@ exports.search = async ctx => {
 
 exports.validate = async ctx => {
   return validate({
-    appId: ctx.appId,
     tableId: ctx.params.tableId,
     row: ctx.request.body,
   })
 }
 
 exports.fetchEnrichedRow = async ctx => {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const tableId = ctx.params.tableId
   const rowId = ctx.params.rowId
   // need table to work out where links go in row
   let [table, row] = await Promise.all([
     db.get(tableId),
-    findRow(ctx, db, tableId, rowId),
+    findRow(ctx, tableId, rowId),
   ])
   // get the link docs
   const linkVals = await linkRows.getLinkDocuments({
-    appId,
     tableId,
     rowId,
   })
@@ -434,7 +398,7 @@ exports.fetchEnrichedRow = async ctx => {
   for (let [tableId, rows] of Object.entries(groups)) {
     // need to include the IDs in these rows for any links they may have
     linkedRows = linkedRows.concat(
-      await outputProcessing(ctx, tables[tableId], rows)
+      await outputProcessing(tables[tableId], rows)
     )
   }
 

package/src/api/controllers/row/internalSearch.js

@@ -1,14 +1,14 @@
 const { SearchIndexes } = require("../../../db/utils")
 const fetch = require("node-fetch")
 const { getCouchUrl } = require("@budibase/backend-core/db")
+const { getAppId } = require("@budibase/backend-core/context")
 
 /**
  * Class to build lucene query URLs.
  * Optionally takes a base lucene query object.
  */
 class QueryBuilder {
-  constructor(appId, base) {
-    this.appId = appId
+  constructor(base) {
     this.query = {
       string: {},
       fuzzy: {},
@@ -37,22 +37,30 @@ class QueryBuilder {
   }
 
   setLimit(limit) {
-    this.limit = limit
+    if (limit != null) {
+      this.limit = limit
+    }
     return this
   }
 
   setSort(sort) {
-    this.sort = sort
+    if (sort != null) {
+      this.sort = sort
+    }
     return this
   }
 
   setSortOrder(sortOrder) {
-    this.sortOrder = sortOrder
+    if (sortOrder != null) {
+      this.sortOrder = sortOrder
+    }
     return this
   }
 
   setSortType(sortType) {
-    this.sortType = sortType
+    if (sortType != null) {
+      this.sortType = sortType
+    }
     return this
   }
 
@@ -233,7 +241,8 @@ class QueryBuilder {
   }
 
   async run() {
-    const url = `${getCouchUrl()}/${this.appId}/_design/database/_search/${
+    const appId = getAppId()
+    const url = `${getCouchUrl()}/${appId}/_design/database/_search/${
       SearchIndexes.ROWS
     }`
     const body = this.buildSearchBody()
@@ -270,7 +279,6 @@ const runQuery = async (url, body) => {
  * Gets round the fixed limit of 200 results from a query by fetching as many
  * pages as required and concatenating the results. This recursively operates
  * until enough results have been found.
- * @param appId {string} The app ID to search
  * @param query {object} The JSON query structure
  * @param params {object} The search params including:
  *   tableId {string} The table ID to search
@@ -283,7 +291,7 @@ const runQuery = async (url, body) => {
  *   rows {array|null} Current results in the recursive search
  * @returns {Promise<*[]|*>}
  */
-const recursiveSearch = async (appId, query, params) => {
+const recursiveSearch = async (query, params) => {
   const bookmark = params.bookmark
   const rows = params.rows || []
   if (rows.length >= params.limit) {
@@ -293,7 +301,7 @@ const recursiveSearch = async (appId, query, params) => {
   if (rows.length > params.limit - 200) {
     pageSize = params.limit - rows.length
   }
-  const page = await new QueryBuilder(appId, query)
+  const page = await new QueryBuilder(query)
     .setVersion(params.version)
     .setTable(params.tableId)
     .setBookmark(bookmark)
@@ -313,14 +321,13 @@ const recursiveSearch = async (appId, query, params) => {
     bookmark: page.bookmark,
     rows: [...rows, ...page.rows],
   }
-  return await recursiveSearch(appId, query, newParams)
+  return await recursiveSearch(query, newParams)
 }
 
 /**
  * Performs a paginated search. A bookmark will be returned to allow the next
  * page to be fetched. There is a max limit off 200 results per page in a
  * paginated search.
- * @param appId {string} The app ID to search
  * @param query {object} The JSON query structure
  * @param params {object} The search params including:
  *   tableId {string} The table ID to search
@@ -332,13 +339,13 @@ const recursiveSearch = async (appId, query, params) => {
  *   bookmark {string} The bookmark to resume from
  * @returns {Promise<{hasNextPage: boolean, rows: *[]}>}
  */
-exports.paginatedSearch = async (appId, query, params) => {
+exports.paginatedSearch = async (query, params) => {
   let limit = params.limit
   if (limit == null || isNaN(limit) || limit < 0) {
     limit = 50
   }
   limit = Math.min(limit, 200)
-  const search = new QueryBuilder(appId, query)
+  const search = new QueryBuilder(query)
     .setVersion(params.version)
     .setTable(params.tableId)
     .setSort(params.sort)
@@ -367,7 +374,6 @@ exports.paginatedSearch = async (appId, query, params) => {
  * desired amount of results. There is a limit of 1000 results to avoid
  * heavy performance hits, and to avoid client components breaking from
  * handling too much data.
- * @param appId {string} The app ID to search
  * @param query {object} The JSON query structure
  * @param params {object} The search params including:
  *   tableId {string} The table ID to search
@@ -378,12 +384,12 @@ exports.paginatedSearch = async (appId, query, params) => {
  *   limit {number} The desired number of results
  * @returns {Promise<{rows: *}>}
  */
-exports.fullSearch = async (appId, query, params) => {
+exports.fullSearch = async (query, params) => {
   let limit = params.limit
   if (limit == null || isNaN(limit) || limit < 0) {
     limit = 1000
   }
   params.limit = Math.min(limit, 1000)
-  const rows = await recursiveSearch(appId, query, params)
+  const rows = await recursiveSearch(query, params)
   return { rows }
 }

package/src/api/controllers/row/staticFormula.js

@@ -0,0 +1,156 @@
+const { getRowParams } = require("../../../db/utils")
+const {
+  outputProcessing,
+  processAutoColumn,
+  processFormulas,
+} = require("../../../utilities/rowProcessor")
+const { FieldTypes, FormulaTypes } = require("../../../constants")
+const { isEqual } = require("lodash")
+const { cloneDeep } = require("lodash/fp")
+const { getAppDB } = require("@budibase/backend-core/context")
+
+/**
+ * This function runs through a list of enriched rows, looks at the rows which
+ * are related and then checks if they need the state of their formulas
+ * updated.
+ * NOTE: this will only for affect static formulas.
+ */
+exports.updateRelatedFormula = async (table, enrichedRows) => {
+  const db = getAppDB()
+  // no formula to update, we're done
+  if (!table.relatedFormula) {
+    return
+  }
+  let promises = []
+  for (let enrichedRow of Array.isArray(enrichedRows)
+    ? enrichedRows
+    : [enrichedRows]) {
+    // the related rows by tableId
+    let relatedRows = {}
+    for (let [key, field] of Object.entries(enrichedRow)) {
+      const columnDefinition = table.schema[key]
+      if (columnDefinition && columnDefinition.type === FieldTypes.LINK) {
+        const relatedTableId = columnDefinition.tableId
+        if (!relatedRows[relatedTableId]) {
+          relatedRows[relatedTableId] = []
+        }
+        relatedRows[relatedTableId] = relatedRows[relatedTableId].concat(field)
+      }
+    }
+    for (let tableId of table.relatedFormula) {
+      let relatedTable
+      try {
+        // no rows to update, skip
+        if (!relatedRows[tableId] || relatedRows[tableId].length === 0) {
+          continue
+        }
+        relatedTable = await db.get(tableId)
+      } catch (err) {
+        // no error scenario, table doesn't seem to exist anymore, ignore
+      }
+      for (let column of Object.values(relatedTable.schema)) {
+        // needs updated in related rows
+        if (
+          column.type === FieldTypes.FORMULA &&
+          column.formulaType === FormulaTypes.STATIC
+        ) {
+          // re-enrich rows for all the related, don't update the related formula for them
+          promises = promises.concat(
+            relatedRows[tableId].map(related =>
+              exports.finaliseRow(relatedTable, related, {
+                updateFormula: false,
+              })
+            )
+          )
+        }
+      }
+    }
+  }
+  await Promise.all(promises)
+}
+
+exports.updateAllFormulasInTable = async table => {
+  const db = getAppDB()
+  // start by getting the raw rows (which will be written back to DB after update)
+  let rows = (
+    await db.allDocs(
+      getRowParams(table._id, null, {
+        include_docs: true,
+      })
+    )
+  ).rows.map(row => row.doc)
+  // now enrich the rows, note the clone so that we have the base state of the
+  // rows so that we don't write any of the enriched information back
+  let enrichedRows = await outputProcessing(table, cloneDeep(rows), {
+    squash: false,
+  })
+  const updatedRows = []
+  for (let row of rows) {
+    // find the enriched row, if found process the formulas
+    const enrichedRow = enrichedRows.find(enriched => enriched._id === row._id)
+    if (enrichedRow) {
+      const processed = processFormulas(table, cloneDeep(row), {
+        dynamic: false,
+        contextRows: enrichedRow,
+      })
+      // values have changed, need to add to bulk docs to update
+      if (!isEqual(processed, row)) {
+        updatedRows.push(processed)
+      }
+    }
+  }
+  await db.bulkDocs(updatedRows)
+}
+
+/**
+ * This function runs at the end of the save/patch functions of the row controller, all this
+ * really does is enrich the row, handle any static formula processing, then return the enriched
+ * row. The reason we need to return the enriched row is that the automation row created trigger
+ * expects the row to be totally enriched/contain all relationships.
+ */
+exports.finaliseRow = async (
+  table,
+  row,
+  { oldTable, updateFormula } = { updateFormula: true }
+) => {
+  const db = getAppDB()
+  row.type = "row"
+  // process the row before return, to include relationships
+  let enrichedRow = await outputProcessing(table, cloneDeep(row), {
+    squash: false,
+  })
+  // use enriched row to generate formulas for saving, specifically only use as context
+  row = processFormulas(table, row, {
+    dynamic: false,
+    contextRows: enrichedRow,
+  })
+
+  // don't worry about rev, tables handle rev/lastID updates
+  // if another row has been written since processing this will
+  // handle the auto ID clash
+  if (oldTable && !isEqual(oldTable, table)) {
+    try {
+      await db.put(table)
+    } catch (err) {
+      if (err.status === 409) {
+        const updatedTable = await db.get(table._id)
+        let response = processAutoColumn(null, updatedTable, row, {
+          reprocessing: true,
+        })
+        await db.put(response.table)
+        row = response.row
+      } else {
+        throw err
+      }
+    }
+  }
+  const response = await db.put(row)
+  // for response, calculate the formulas for the enriched row
+  enrichedRow._rev = response.rev
+  enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false })
+  // this updates the related formulas in other rows based on the relations to this row
+  if (updateFormula) {
+    await exports.updateRelatedFormula(table, enrichedRow)
+  }
+  return { row: enrichedRow, table }
+}