@budibase/server 1.0.49-alpha.0 → 1.0.49-alpha.4

package/dist/utilities/rowProcessor/index.js

@@ -175,6 +175,7 @@ function processAutoColumn(user, table, row, opts = { reprocessing: false, noAut
     return { table, row };
 }
 exports.processAutoColumn = processAutoColumn;
+exports.processFormulas = processFormulas;
 /**
  * This will coerce a value to the correct types based on the type transform map
  * @param {object} row The value to coerce
@@ -218,11 +219,11 @@ exports.inputProcessing = (user = {}, table, row, opts = { noAutoRelationships:
             }
             continue;
         }
-        // specific case to delete formula values if they get saved
-        // type coercion cannot completely remove the field, so have to do it here
+        // remove any formula values, they are to be generated
         if (field.type === FieldTypes.FORMULA) {
             delete clonedRow[key];
         }
+        // otherwise coerce what is there to correct types
         else {
             clonedRow[key] = exports.coerce(value, field.type);
         }
@@ -237,24 +238,23 @@ exports.inputProcessing = (user = {}, table, row, opts = { noAutoRelationships:
 /**
  * This function enriches the input rows with anything they are supposed to contain, for example
  * link records or attachment links.
- * @param {object} ctx the request which is looking for enriched rows.
+ * @param {string} appId the app in which the request is looking for enriched rows.
  * @param {object} table the table from which these rows came from originally, this is used to determine
  * the schema of the rows and then enrich.
  * @param {object[]|object} rows the rows which are to be enriched.
  * @param {object} opts used to set some options for the output, such as disabling relationship squashing.
  * @returns {object[]|object} the enriched rows will be returned.
  */
-exports.outputProcessing = (ctx, table, rows, opts = { squash: true }) => __awaiter(void 0, void 0, void 0, function* () {
-    const appId = ctx.appId;
+exports.outputProcessing = ({ appId }, table, rows, opts = { squash: true }) => __awaiter(void 0, void 0, void 0, function* () {
     let wasArray = true;
     if (!(rows instanceof Array)) {
         rows = [rows];
         wasArray = false;
     }
     // attach any linked row information
-    let enriched = yield linkRows.attachFullLinkedDocs(ctx, table, rows);
+    let enriched = yield linkRows.attachFullLinkedDocs(appId, table, rows);
     // process formulas
-    enriched = processFormulas(table, enriched);
+    enriched = processFormulas(table, enriched, { dynamic: true });
     // update the attachments URL depending on hosting
     for (let [property, column] of Object.entries(table.schema)) {
         if (column.type === FieldTypes.ATTACHMENT) {
@@ -280,9 +280,11 @@ exports.outputProcessing = (ctx, table, rows, opts = { squash: true }) => __awai
  * @param {any} row optional - the row being removed.
  * @param {any} rows optional - if multiple rows being deleted can do this in bulk.
  * @param {any} oldRow optional - if updating a row this will determine the difference.
+ * @param {any} oldTable optional - if updating a table, can supply the old table to look for
+ * deleted attachment columns.
  * @return {Promise<void>} When all attachments have been removed this will return.
  */
-exports.cleanupAttachments = (appId, table, { row, rows, oldRow }) => __awaiter(void 0, void 0, void 0, function* () {
+exports.cleanupAttachments = (appId, table, { row, rows, oldRow, oldTable }) => __awaiter(void 0, void 0, void 0, function* () {
     if (!isProdAppID(appId)) {
         const prodAppId = getDeployedAppID(appId);
         // if prod exists, then don't allow deleting
@@ -297,12 +299,17 @@ exports.cleanupAttachments = (appId, table, { row, rows, oldRow }) => __awaiter(
             files = files.concat(row[key].map(attachment => attachment.key));
         }
     }
-    for (let [key, schema] of Object.entries(table.schema)) {
+    const schemaToUse = oldTable ? oldTable.schema : table.schema;
+    for (let [key, schema] of Object.entries(schemaToUse)) {
         if (schema.type !== FieldTypes.ATTACHMENT) {
             continue;
         }
-        // if updating, need to manage the differences
-        if (oldRow && row) {
+        // old table had this column, new table doesn't - delete it
+        if (oldTable && !table.schema[key]) {
+            rows.forEach(row => addFiles(row, key));
+        }
+        else if (oldRow && row) {
+            // if updating, need to manage the differences
             files = files.concat(getRemovedAttachmentKeys(oldRow, row, key));
         }
         else if (row) {

package/dist/utilities/rowProcessor/utils.js

@@ -1,20 +1,30 @@
 "use strict";
-const { FieldTypes } = require("../../constants");
+const { FieldTypes, FormulaTypes } = require("../../constants");
 const { processStringSync } = require("@budibase/string-templates");
 /**
  * Looks through the rows provided and finds formulas - which it then processes.
  */
-exports.processFormulas = (table, rows) => {
+exports.processFormulas = (table, rows, { dynamic, contextRows } = { dynamic: true }) => {
     const single = !Array.isArray(rows);
     if (single) {
         rows = [rows];
+        contextRows = contextRows ? [contextRows] : contextRows;
     }
     for (let [column, schema] of Object.entries(table.schema)) {
-        if (schema.type !== FieldTypes.FORMULA) {
+        const isStatic = schema.formulaType === FormulaTypes.STATIC;
+        if (schema.type !== FieldTypes.FORMULA ||
+            (dynamic && isStatic) ||
+            (!dynamic && !isStatic)) {
             continue;
         }
         // iterate through rows and process formula
-        rows = rows.map(row => (Object.assign(Object.assign({}, row), { [column]: processStringSync(schema.formula, row) })));
+        for (let i = 0; i < rows.length; i++) {
+            if (schema.formula) {
+                let row = rows[i];
+                let context = contextRows ? contextRows[i] : row;
+                rows[i] = Object.assign(Object.assign({}, row), { [column]: processStringSync(schema.formula, context) });
+            }
+        }
     }
     return single ? rows[0] : rows;
 };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.49-alpha.0",
+  "version": "1.0.49-alpha.4",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.49-alpha.0",
-    "@budibase/client": "^1.0.49-alpha.0",
-    "@budibase/string-templates": "^1.0.49-alpha.0",
+    "@budibase/backend-core": "^1.0.49-alpha.4",
+    "@budibase/client": "^1.0.49-alpha.4",
+    "@budibase/string-templates": "^1.0.49-alpha.4",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -168,5 +168,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "27d295ae310158a0a240fe62c45362ad03f82115"
+  "gitHead": "c991f284e3be5257091f234bdde5abaf1d53d2be"
 }

package/src/api/controllers/application.js

@@ -75,7 +75,7 @@ function getUserRoleId(ctx) {
     : ctx.user.role._id
 }
 
-async function getAppUrl(ctx) {
+exports.getAppUrl = ctx => {
   // construct the url
   let url
   if (ctx.request.body.url) {
@@ -219,7 +219,7 @@ exports.create = async ctx => {
   const apps = await getAllApps(CouchDB, { dev: true })
   const name = ctx.request.body.name
   checkAppName(ctx, apps, name)
-  const url = await getAppUrl(ctx)
+  const url = exports.getAppUrl(ctx)
   checkAppUrl(ctx, apps, url)
 
   const { useTemplate, templateKey, templateString } = ctx.request.body
@@ -286,7 +286,7 @@ exports.update = async ctx => {
   if (name) {
     checkAppName(ctx, apps, name, ctx.params.appId)
   }
-  const url = await getAppUrl(ctx)
+  const url = await exports.getAppUrl(ctx)
   if (url) {
     checkAppUrl(ctx, apps, url, ctx.params.appId)
     ctx.request.body.url = url

package/src/api/controllers/migrations.js

@@ -0,0 +1,13 @@
+const { migrate, MIGRATIONS } = require("../../migrations")
+
+exports.migrate = async ctx => {
+  const options = ctx.request.body
+  // don't await as can take a while, just return
+  migrate(options)
+  ctx.status = 200
+}
+
+exports.fetchDefinitions = async ctx => {
+  ctx.body = MIGRATIONS
+  ctx.status = 200
+}

package/src/api/controllers/row/internal.js

@@ -1,8 +1,8 @@
 const CouchDB = require("../../../db")
 const linkRows = require("../../../db/linkedRows")
 const {
-  getRowParams,
   generateRowID,
+  getRowParams,
   DocumentTypes,
   InternalTables,
 } = require("../../../db/utils")
@@ -10,11 +10,9 @@ const userController = require("../user")
 const {
   inputProcessing,
   outputProcessing,
-  processAutoColumn,
   cleanupAttachments,
 } = require("../../../utilities/rowProcessor")
 const { FieldTypes } = require("../../../constants")
-const { isEqual } = require("lodash")
 const { validate, findRow } = require("./utils")
 const { fullSearch, paginatedSearch } = require("./internalSearch")
 const { getGlobalUsersFromMetadata } = require("../../../utilities/global")
@@ -27,6 +25,7 @@ const {
   getFromMemoryDoc,
 } = require("../view/utils")
 const { cloneDeep } = require("lodash/fp")
+const { finaliseRow, updateRelatedFormula } = require("./staticFormula")
 
 const CALCULATION_TYPES = {
   SUM: "sum",
@@ -34,51 +33,6 @@ const CALCULATION_TYPES = {
   STATS: "stats",
 }
 
-async function storeResponse(ctx, db, row, oldTable, table) {
-  row.type = "row"
-  // don't worry about rev, tables handle rev/lastID updates
-  // if another row has been written since processing this will
-  // handle the auto ID clash
-  if (!isEqual(oldTable, table)) {
-    try {
-      await db.put(table)
-    } catch (err) {
-      if (err.status === 409) {
-        const updatedTable = await db.get(table._id)
-        let response = processAutoColumn(null, updatedTable, row, {
-          reprocessing: true,
-        })
-        await db.put(response.table)
-        row = response.row
-      } else {
-        throw err
-      }
-    }
-  }
-  const response = await db.put(row)
-  row._rev = response.rev
-  // process the row before return, to include relationships
-  row = await outputProcessing(ctx, table, row, { squash: false })
-  return { row, table }
-}
-
-// doesn't do the outputProcessing
-async function getRawTableData(ctx, db, tableId) {
-  let rows
-  if (tableId === InternalTables.USER_METADATA) {
-    await userController.fetchMetadata(ctx)
-    rows = ctx.body
-  } else {
-    const response = await db.allDocs(
-      getRowParams(tableId, null, {
-        include_docs: true,
-      })
-    )
-    rows = response.rows.map(row => row.doc)
-  }
-  return rows
-}
-
 async function getView(db, viewName) {
   let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
   let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
@@ -105,6 +59,22 @@ async function getView(db, viewName) {
   return viewInfo
 }
 
+async function getRawTableData(ctx, db, tableId) {
+  let rows
+  if (tableId === InternalTables.USER_METADATA) {
+    await userController.fetchMetadata(ctx)
+    rows = ctx.body
+  } else {
+    const response = await db.allDocs(
+      getRowParams(tableId, null, {
+        include_docs: true,
+      })
+    )
+    rows = response.rows.map(row => row.doc)
+  }
+  return rows
+}
+
 exports.patch = async ctx => {
   const appId = ctx.appId
   const db = new CouchDB(appId)
@@ -162,7 +132,10 @@ exports.patch = async ctx => {
     return { row: ctx.body, table }
   }
 
-  return storeResponse(ctx, db, row, dbTable, table)
+  return finaliseRow(ctx.appId, table, row, {
+    oldTable: dbTable,
+    updateFormula: true,
+  })
 }
 
 exports.save = async function (ctx) {
@@ -196,7 +169,10 @@ exports.save = async function (ctx) {
     table,
   })
 
-  return storeResponse(ctx, db, row, dbTable, table)
+  return finaliseRow(ctx.appId, table, row, {
+    oldTable: dbTable,
+    updateFormula: true,
+  })
 }
 
 exports.fetchView = async ctx => {
@@ -302,6 +278,8 @@ exports.destroy = async function (ctx) {
   })
   // remove any attachments that were on the row from object storage
   await cleanupAttachments(appId, table, { row })
+  // remove any static formula
+  await updateRelatedFormula(appId, table, row)
 
   let response
   if (ctx.params.tableId === InternalTables.USER_METADATA) {
@@ -350,6 +328,7 @@ exports.bulkDestroy = async ctx => {
   }
   // remove any attachments that were on the rows from object storage
   await cleanupAttachments(appId, table, { rows })
+  await updateRelatedFormula(appId, table, rows)
   await Promise.all(updates)
   return { response: { ok: true }, rows }
 }

package/src/api/controllers/row/internalSearch.js

@@ -37,22 +37,30 @@ class QueryBuilder {
   }
 
   setLimit(limit) {
-    this.limit = limit
+    if (limit != null) {
+      this.limit = limit
+    }
     return this
   }
 
   setSort(sort) {
-    this.sort = sort
+    if (sort != null) {
+      this.sort = sort
+    }
     return this
   }
 
   setSortOrder(sortOrder) {
-    this.sortOrder = sortOrder
+    if (sortOrder != null) {
+      this.sortOrder = sortOrder
+    }
     return this
   }
 
   setSortType(sortType) {
-    this.sortType = sortType
+    if (sortType != null) {
+      this.sortType = sortType
+    }
     return this
   }
 

package/src/api/controllers/row/staticFormula.js

@@ -0,0 +1,157 @@
+const CouchDB = require("../../../db")
+const { getRowParams } = require("../../../db/utils")
+const {
+  outputProcessing,
+  processAutoColumn,
+  processFormulas,
+} = require("../../../utilities/rowProcessor")
+const { FieldTypes, FormulaTypes } = require("../../../constants")
+const { isEqual } = require("lodash")
+const { cloneDeep } = require("lodash/fp")
+
+/**
+ * This function runs through a list of enriched rows, looks at the rows which
+ * are related and then checks if they need the state of their formulas
+ * updated.
+ * NOTE: this will only for affect static formulas.
+ */
+exports.updateRelatedFormula = async (appId, table, enrichedRows) => {
+  const db = new CouchDB(appId)
+  // no formula to update, we're done
+  if (!table.relatedFormula) {
+    return
+  }
+  let promises = []
+  for (let enrichedRow of Array.isArray(enrichedRows)
+    ? enrichedRows
+    : [enrichedRows]) {
+    // the related rows by tableId
+    let relatedRows = {}
+    for (let [key, field] of Object.entries(enrichedRow)) {
+      const columnDefinition = table.schema[key]
+      if (columnDefinition && columnDefinition.type === FieldTypes.LINK) {
+        const relatedTableId = columnDefinition.tableId
+        if (!relatedRows[relatedTableId]) {
+          relatedRows[relatedTableId] = []
+        }
+        relatedRows[relatedTableId] = relatedRows[relatedTableId].concat(field)
+      }
+    }
+    for (let tableId of table.relatedFormula) {
+      let relatedTable
+      try {
+        // no rows to update, skip
+        if (!relatedRows[tableId] || relatedRows[tableId].length === 0) {
+          continue
+        }
+        relatedTable = await db.get(tableId)
+      } catch (err) {
+        // no error scenario, table doesn't seem to exist anymore, ignore
+      }
+      for (let column of Object.values(relatedTable.schema)) {
+        // needs updated in related rows
+        if (
+          column.type === FieldTypes.FORMULA &&
+          column.formulaType === FormulaTypes.STATIC
+        ) {
+          // re-enrich rows for all the related, don't update the related formula for them
+          promises = promises.concat(
+            relatedRows[tableId].map(related =>
+              exports.finaliseRow(appId, relatedTable, related, {
+                updateFormula: false,
+              })
+            )
+          )
+        }
+      }
+    }
+  }
+  await Promise.all(promises)
+}
+
+exports.updateAllFormulasInTable = async (appId, table) => {
+  const db = new CouchDB(appId)
+  // start by getting the raw rows (which will be written back to DB after update)
+  let rows = (
+    await db.allDocs(
+      getRowParams(table._id, null, {
+        include_docs: true,
+      })
+    )
+  ).rows.map(row => row.doc)
+  // now enrich the rows, note the clone so that we have the base state of the
+  // rows so that we don't write any of the enriched information back
+  let enrichedRows = await outputProcessing({ appId }, table, cloneDeep(rows), {
+    squash: false,
+  })
+  const updatedRows = []
+  for (let row of rows) {
+    // find the enriched row, if found process the formulas
+    const enrichedRow = enrichedRows.find(enriched => enriched._id === row._id)
+    if (enrichedRow) {
+      const processed = processFormulas(table, cloneDeep(row), {
+        dynamic: false,
+        contextRows: enrichedRow,
+      })
+      // values have changed, need to add to bulk docs to update
+      if (!isEqual(processed, row)) {
+        updatedRows.push(processed)
+      }
+    }
+  }
+  await db.bulkDocs(updatedRows)
+}
+
+/**
+ * This function runs at the end of the save/patch functions of the row controller, all this
+ * really does is enrich the row, handle any static formula processing, then return the enriched
+ * row. The reason we need to return the enriched row is that the automation row created trigger
+ * expects the row to be totally enriched/contain all relationships.
+ */
+exports.finaliseRow = async (
+  appId,
+  table,
+  row,
+  { oldTable, updateFormula } = { updateFormula: true }
+) => {
+  const db = new CouchDB(appId)
+  row.type = "row"
+  // process the row before return, to include relationships
+  let enrichedRow = await outputProcessing({ appId }, table, cloneDeep(row), {
+    squash: false,
+  })
+  // use enriched row to generate formulas for saving, specifically only use as context
+  row = processFormulas(table, row, {
+    dynamic: false,
+    contextRows: enrichedRow,
+  })
+
+  // don't worry about rev, tables handle rev/lastID updates
+  // if another row has been written since processing this will
+  // handle the auto ID clash
+  if (oldTable && !isEqual(oldTable, table)) {
+    try {
+      await db.put(table)
+    } catch (err) {
+      if (err.status === 409) {
+        const updatedTable = await db.get(table._id)
+        let response = processAutoColumn(null, updatedTable, row, {
+          reprocessing: true,
+        })
+        await db.put(response.table)
+        row = response.row
+      } else {
+        throw err
+      }
+    }
+  }
+  const response = await db.put(row)
+  // for response, calculate the formulas for the enriched row
+  enrichedRow._rev = response.rev
+  enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false })
+  // this updates the related formulas in other rows based on the relations to this row
+  if (updateFormula) {
+    await exports.updateRelatedFormula(appId, table, enrichedRow)
+  }
+  return { row: enrichedRow, table }
+}