@budibase/server 1.0.49-alpha.0 → 1.0.49-alpha.4

package/src/api/controllers/table/bulkFormula.js

@@ -0,0 +1,183 @@
+const CouchDB = require("../../../db")
+const { FieldTypes, FormulaTypes } = require("../../../constants")
+const { getAllInternalTables, clearColumns } = require("./utils")
+const { doesContainStrings } = require("@budibase/string-templates")
+const { cloneDeep } = require("lodash/fp")
+const { isEqual, uniq } = require("lodash")
+const { updateAllFormulasInTable } = require("../row/staticFormula")
+
+function isStaticFormula(column) {
+  return (
+    column.type === FieldTypes.FORMULA &&
+    column.formulaType === FormulaTypes.STATIC
+  )
+}
+
+/**
+ * This retrieves the formula columns from a table schema that use a specified column name
+ * in the formula.
+ */
+function getFormulaThatUseColumn(table, columnNames) {
+  let formula = []
+  columnNames = Array.isArray(columnNames) ? columnNames : [columnNames]
+  for (let column of Object.values(table.schema)) {
+    // not a static formula, or doesn't contain a relationship
+    if (!isStaticFormula(column)) {
+      continue
+    }
+    if (!doesContainStrings(column.formula, columnNames)) {
+      continue
+    }
+    formula.push(column.name)
+  }
+  return formula
+}
+
+/**
+ * This functions checks for when a related table, column or related column is deleted, if any
+ * tables need to have the formula column removed.
+ */
+async function checkIfFormulaNeedsCleared(
+  appId,
+  table,
+  { oldTable, deletion }
+) {
+  const db = new CouchDB(appId)
+  // start by retrieving all tables, remove the current table from the list
+  const tables = (await getAllInternalTables(appId)).filter(
+    tbl => tbl._id !== table._id
+  )
+  const schemaToUse = oldTable ? oldTable.schema : table.schema
+  let removedColumns = Object.values(schemaToUse).filter(
+    column => deletion || !table.schema[column.name]
+  )
+  // remove any formula columns that used related columns
+  for (let removed of removedColumns) {
+    let tableToUse = table
+    // if relationship, get the related table
+    if (removed.type === FieldTypes.LINK) {
+      tableToUse = tables.find(table => table._id === removed.tableId)
+    }
+    const columnsToDelete = getFormulaThatUseColumn(tableToUse, removed.name)
+    if (columnsToDelete.length > 0) {
+      await clearColumns(db, table, columnsToDelete)
+    }
+    // need a special case, where a column has been removed from this table, but was used
+    // in a different, related tables formula
+    if (!table.relatedFormula) {
+      continue
+    }
+    for (let relatedTableId of table.relatedFormula) {
+      const relatedColumns = Object.values(table.schema).filter(
+        column => column.tableId === relatedTableId
+      )
+      const relatedTable = tables.find(table => table._id === relatedTableId)
+      // look to see if the column was used in a relationship formula,
+      // relationships won't be used for this
+      if (relatedTable && relatedColumns && removed.type !== FieldTypes.LINK) {
+        let relatedFormulaToRemove = []
+        for (let column of relatedColumns) {
+          relatedFormulaToRemove = relatedFormulaToRemove.concat(
+            getFormulaThatUseColumn(relatedTable, [
+              column.fieldName,
+              removed.name,
+            ])
+          )
+        }
+        if (relatedFormulaToRemove.length > 0) {
+          await clearColumns(db, relatedTable, uniq(relatedFormulaToRemove))
+        }
+      }
+    }
+  }
+}
+
+/**
+ * This function adds a note to related tables that they are
+ * used in a static formula - so that the link controller
+ * can manage hydrating related rows formula fields. This is
+ * specifically only for static formula.
+ */
+async function updateRelatedFormulaLinksOnTables(
+  appId,
+  table,
+  { deletion } = { deletion: false }
+) {
+  const db = new CouchDB(appId)
+  // start by retrieving all tables, remove the current table from the list
+  const tables = (await getAllInternalTables(appId)).filter(
+    tbl => tbl._id !== table._id
+  )
+  // clone the tables, so we can compare at end
+  const initialTables = cloneDeep(tables)
+  // first find the related column names
+  const relatedColumns = Object.values(table.schema).filter(
+    col => col.type === FieldTypes.LINK
+  )
+  // we start by removing the formula field from all tables
+  for (let otherTable of tables) {
+    if (!otherTable.relatedFormula) {
+      continue
+    }
+    const index = otherTable.relatedFormula.indexOf(table._id)
+    if (index !== -1) {
+      otherTable.relatedFormula.splice(index, 1)
+    }
+  }
+  // if deleting, just remove the table IDs, don't try add
+  if (!deletion) {
+    for (let relatedCol of relatedColumns) {
+      let columns = getFormulaThatUseColumn(table, relatedCol.name)
+      if (!columns || columns.length === 0) {
+        continue
+      }
+      const relatedTable = tables.find(
+        related => related._id === relatedCol.tableId
+      )
+      // check if the table is already in the list of related formula, if it isn't, then add it
+      if (
+        relatedTable &&
+        (!relatedTable.relatedFormula ||
+          !relatedTable.relatedFormula.includes(table._id))
+      ) {
+        relatedTable.relatedFormula = relatedTable.relatedFormula
+          ? [...relatedTable.relatedFormula, table._id]
+          : [table._id]
+      }
+    }
+  }
+  // now we just need to compare all the tables and see if any need saved
+  for (let initial of initialTables) {
+    const found = tables.find(tbl => initial._id === tbl._id)
+    if (found && !isEqual(initial, found)) {
+      await db.put(found)
+    }
+  }
+}
+
+async function checkIfFormulaUpdated(appId, table, { oldTable }) {
+  // look to see if any formula values have changed
+  const shouldUpdate = Object.values(table.schema).find(
+    column =>
+      isStaticFormula(column) &&
+      (!oldTable ||
+        !oldTable.schema[column.name] ||
+        !isEqual(oldTable.schema[column.name], column))
+  )
+  // if a static formula column has updated, then need to run the update
+  if (shouldUpdate != null) {
+    await updateAllFormulasInTable(appId, table)
+  }
+}
+
+exports.runStaticFormulaChecks = async (
+  appId,
+  table,
+  { oldTable, deletion }
+) => {
+  await updateRelatedFormulaLinksOnTables(appId, table, { deletion })
+  await checkIfFormulaNeedsCleared(appId, table, { oldTable, deletion })
+  if (!deletion) {
+    await checkIfFormulaUpdated(appId, table, { oldTable })
+  }
+}

package/src/api/controllers/table/index.js

@@ -3,12 +3,8 @@ const internal = require("./internal")
 const external = require("./external")
 const csvParser = require("../../../utilities/csvParser")
 const { isExternalTable, isSQL } = require("../../../integrations/utils")
-const {
-  getTableParams,
-  getDatasourceParams,
-  BudibaseInternalDB,
-} = require("../../../db/utils")
-const { getTable } = require("./utils")
+const { getDatasourceParams } = require("../../../db/utils")
+const { getTable, getAllInternalTables } = require("./utils")
 
 function pickApi({ tableId, table }) {
   if (table && !tableId) {
@@ -26,17 +22,7 @@ function pickApi({ tableId, table }) {
 exports.fetch = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
-  const internalTables = await db.allDocs(
-    getTableParams(null, {
-      include_docs: true,
-    })
-  )
-
-  const internal = internalTables.rows.map(tableDoc => ({
-    ...tableDoc.doc,
-    type: "internal",
-    sourceId: BudibaseInternalDB._id,
-  }))
+  const internal = await getAllInternalTables(ctx.appId)
 
   const externalTables = await db.allDocs(
     getDatasourceParams("plus", {

package/src/api/controllers/table/internal.js

@@ -9,6 +9,8 @@ const {
   handleDataImport,
 } = require("./utils")
 const usageQuota = require("../../../utilities/usageQuota")
+const { cleanupAttachments } = require("../../../utilities/rowProcessor")
+const { runStaticFormulaChecks } = require("./bulkFormula")
 
 exports.save = async function (ctx) {
   const appId = ctx.appId
@@ -104,7 +106,8 @@ exports.save = async function (ctx) {
   tableToSave._rev = result.rev
 
   tableToSave = await tableSaveFunctions.after(tableToSave)
-
+  // has to run after, make sure it has _id
+  await runStaticFormulaChecks(appId, tableToSave, { oldTable })
   return tableToSave
 }
 
@@ -141,6 +144,9 @@ exports.destroy = async function (ctx) {
     await db.deleteIndex(existingIndex)
   }
 
+  // has to run after, make sure it has _id
+  await runStaticFormulaChecks(appId, tableToDelete, { deletion: true })
+  await cleanupAttachments(appId, tableToDelete, { rows })
   return tableToDelete
 }
 

package/src/api/controllers/table/utils.js

@@ -4,10 +4,15 @@ const {
   getRowParams,
   generateRowID,
   InternalTables,
+  getTableParams,
+  BudibaseInternalDB,
 } = require("../../../db/utils")
-const { isEqual } = require("lodash/fp")
+const { isEqual } = require("lodash")
 const { AutoFieldSubTypes, FieldTypes } = require("../../../constants")
-const { inputProcessing } = require("../../../utilities/rowProcessor")
+const {
+  inputProcessing,
+  cleanupAttachments,
+} = require("../../../utilities/rowProcessor")
 const {
   USERS_TABLE_SCHEMA,
   SwitchableTypes,
@@ -21,8 +26,24 @@ const {
 const { getViews, saveView } = require("../view/utils")
 const viewTemplate = require("../view/viewBuilder")
 const usageQuota = require("../../../utilities/usageQuota")
+const { cloneDeep } = require("lodash/fp")
+
+exports.clearColumns = async (appId, table, columnNames) => {
+  const db = new CouchDB(appId)
+  const rows = await db.allDocs(
+    getRowParams(table._id, null, {
+      include_docs: true,
+    })
+  )
+  return db.bulkDocs(
+    rows.rows.map(({ doc }) => {
+      columnNames.forEach(colName => delete doc[colName])
+      return doc
+    })
+  )
+}
 
-exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
+exports.checkForColumnUpdates = async (appId, db, oldTable, updatedTable) => {
   let updatedRows = []
   const rename = updatedTable._rename
   let deletedColumns = []
@@ -39,16 +60,20 @@ exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
         include_docs: true,
       })
     )
-    updatedRows = rows.rows.map(({ doc }) => {
+    const rawRows = rows.rows.map(({ doc }) => doc)
+    updatedRows = rawRows.map(row => {
+      row = cloneDeep(row)
       if (rename) {
-        doc[rename.updated] = doc[rename.old]
-        delete doc[rename.old]
+        row[rename.updated] = row[rename.old]
+        delete row[rename.old]
       } else if (deletedColumns.length !== 0) {
-        deletedColumns.forEach(colName => delete doc[colName])
+        deletedColumns.forEach(colName => delete row[colName])
       }
-      return doc
+      return row
     })
 
+    // cleanup any attachments from object storage for deleted attachment columns
+    await cleanupAttachments(appId, updatedTable, { oldTable, rows: rawRows })
     // Update views
     await exports.checkForViewUpdates(db, updatedTable, rename, deletedColumns)
     delete updatedTable._rename
@@ -209,6 +234,7 @@ class TableSaveFunctions {
   // when confirmed valid
   async mid(table) {
     let response = await exports.checkForColumnUpdates(
+      this.appId,
       this.db,
       this.oldTable,
       table
@@ -234,6 +260,20 @@ class TableSaveFunctions {
   }
 }
 
+exports.getAllInternalTables = async appId => {
+  const db = new CouchDB(appId)
+  const internalTables = await db.allDocs(
+    getTableParams(null, {
+      include_docs: true,
+    })
+  )
+  return internalTables.rows.map(tableDoc => ({
+    ...tableDoc.doc,
+    type: "internal",
+    sourceId: BudibaseInternalDB._id,
+  }))
+}
+
 exports.getAllExternalTables = async (appId, datasourceId) => {
   const db = new CouchDB(appId)
   const datasource = await db.get(datasourceId)

package/src/api/routes/index.js

@@ -24,6 +24,7 @@ const backupRoutes = require("./backup")
 const metadataRoutes = require("./metadata")
 const devRoutes = require("./dev")
 const cloudRoutes = require("./cloud")
+const migrationRoutes = require("./migrations")
 
 exports.mainRoutes = [
   authRoutes,
@@ -53,6 +54,7 @@ exports.mainRoutes = [
   // this could be breaking as koa may recognise other routes as this
   tableRoutes,
   rowRoutes,
+  migrationRoutes,
 ]
 
 exports.staticRoutes = staticRoutes

package/src/api/routes/migrations.js

@@ -0,0 +1,14 @@
+const Router = require("@koa/router")
+const migrationsController = require("../controllers/migrations")
+const router = Router()
+const { internalApi } = require("@budibase/backend-core/auth")
+
+router
+  .post("/api/migrations/run", internalApi, migrationsController.migrate)
+  .get(
+    "/api/migrations/definitions",
+    internalApi,
+    migrationsController.fetchDefinitions
+  )
+
+module.exports = router

package/src/app.ts

@@ -1,7 +1,7 @@
 // need to load environment first
 import { ExtendableContext } from "koa"
 
-const env = require("./environment")
+import * as env from "./environment"
 const CouchDB = require("./db")
 require("@budibase/backend-core").init(CouchDB)
 const Koa = require("koa")
@@ -16,6 +16,7 @@ const Sentry = require("@sentry/node")
 const fileSystem = require("./utilities/fileSystem")
 const bullboard = require("./automations/bullboard")
 const redis = require("./utilities/redis")
+import * as migrations from "./migrations"
 
 const app = new Koa()
 
@@ -84,13 +85,25 @@ module.exports = server.listen(env.PORT || 0, async () => {
   await automations.init()
 })
 
-process.on("uncaughtException", err => {
-  console.error(err)
+const shutdown = () => {
   server.close()
   server.destroy()
+}
+
+process.on("uncaughtException", err => {
+  console.error(err)
+  shutdown()
 })
 
 process.on("SIGTERM", () => {
-  server.close()
-  server.destroy()
+  shutdown()
 })
+
+// run migrations on startup if not done via http
+// not recommended in a clustered environment
+if (!env.HTTP_MIGRATIONS) {
+  migrations.migrate().catch(err => {
+    console.error("Error performing migrations. Exiting.\n", err)
+    shutdown()
+  })
+}

package/src/constants/index.js

@@ -61,6 +61,11 @@ exports.RelationshipTypes = {
   MANY_TO_MANY: "many-to-many",
 }
 
+exports.FormulaTypes = {
+  STATIC: "static",
+  DYNAMIC: "dynamic",
+}
+
 exports.AuthTypes = {
   APP: "app",
   BUILDER: "builder",

package/src/db/linkedRows/index.js

@@ -72,7 +72,7 @@ async function getLinksForRows(appId, rows) {
   )
 }
 
-async function getFullLinkedDocs(ctx, appId, links) {
+async function getFullLinkedDocs(appId, links) {
   // create DBs
   const db = new CouchDB(appId)
   const linkedRowIds = links.map(link => link.id)
@@ -146,13 +146,12 @@ exports.updateLinks = async function (args) {
 /**
  * Given a table and a list of rows this will retrieve all of the attached docs and enrich them into the row.
  * This is required for formula fields, this may only be utilised internally (for now).
- * @param {object} ctx The request which is looking for rows.
+ * @param {string} appId The ID of the app which this request is in the context of.
  * @param {object} table The table from which the rows originated.
  * @param {array<object>} rows The rows which are to be enriched.
  * @return {Promise<*>} returns the rows with all of the enriched relationships on it.
  */
-exports.attachFullLinkedDocs = async (ctx, table, rows) => {
-  const appId = ctx.appId
+exports.attachFullLinkedDocs = async (appId, table, rows) => {
   const linkedTableIds = getLinkedTableIDs(table)
   if (linkedTableIds.length === 0) {
     return rows
@@ -166,7 +165,7 @@ exports.attachFullLinkedDocs = async (ctx, table, rows) => {
   // clear any existing links that could be dupe'd
   rows = clearRelationshipFields(table, rows)
   // now get the docs and combine into the rows
-  let linked = await getFullLinkedDocs(ctx, appId, links)
+  let linked = await getFullLinkedDocs(appId, links)
   const linkedTables = []
   for (let row of rows) {
     for (let link of links.filter(link => link.thisId === row._id)) {

package/src/definitions/common.ts

@@ -17,6 +17,8 @@ export interface FieldSchema {
   autocolumn?: boolean
   throughFrom?: string
   throughTo?: string
+  formula?: string
+  formulaType?: string
   main?: boolean
   meta?: {
     toTable: string
@@ -46,6 +48,7 @@ export interface Table extends Base {
   schema: TableSchema
   primaryDisplay?: string
   sourceId?: string
+  relatedFormula?: string[]
   constrained?: string[]
 }
 

package/src/environment.js

@@ -43,6 +43,7 @@ module.exports = {
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
+  HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS,
   // environment
   NODE_ENV: process.env.NODE_ENV,
   JEST_WORKER_ID: process.env.JEST_WORKER_ID,

package/src/middleware/usageQuota.js

@@ -5,7 +5,6 @@ const {
   isExternalTable,
   isRowId: isExternalRowId,
 } = require("../integrations/utils")
-const migration = require("../migrations/usageQuotas")
 
 // currently only counting new writes and deletes
 const METHOD_MAP = {
@@ -74,7 +73,6 @@ module.exports = async (ctx, next) => {
     usage = files.map(file => file.size).reduce((total, size) => total + size)
   }
   try {
-    await migration.run()
     await performRequest(ctx, next, property, usage)
   } catch (err) {
     ctx.throw(400, err)

package/src/migrations/functions/appUrls.ts

@@ -0,0 +1,34 @@
+const { DocumentTypes } = require("@budibase/backend-core/db")
+import { getAppUrl } from "../../api/controllers/application"
+
+/**
+ * Date:
+ * January 2022
+ *
+ * Description:
+ * Add the url to the app metadata if it doesn't exist
+ */
+export const run = async (appDb: any) => {
+  let metadata
+  try {
+    metadata = await appDb.get(DocumentTypes.APP_METADATA)
+  } catch (e) {
+    // sometimes the metadata document doesn't exist
+    // exit early instead of failing the migration
+    console.error("Error retrieving app metadata. Skipping", e)
+    return
+  }
+
+  if (!metadata.url) {
+    const context = {
+      request: {
+        body: {
+          name: metadata.name,
+        },
+      },
+    }
+    metadata.url = getAppUrl(context)
+    console.log(`Adding url to app: ${metadata.url}`)
+    await appDb.put(metadata)
+  }
+}

package/src/migrations/functions/quotas1.ts

@@ -0,0 +1,20 @@
+import { runQuotaMigration } from "./usageQuotas"
+import * as syncApps from "./usageQuotas/syncApps"
+import * as syncRows from "./usageQuotas/syncRows"
+
+/**
+ * Date:
+ * January 2022
+ *
+ * Description:
+ * Synchronise the app and row quotas to the state of the db after it was
+ * discovered that the quota resets were still in place and the row quotas
+ * weren't being decremented correctly.
+ */
+
+export const run = async () => {
+  await runQuotaMigration(async () => {
+    await syncApps.run()
+    await syncRows.run()
+  })
+}

package/src/migrations/functions/tests/appUrls.spec.js

@@ -0,0 +1,29 @@
+const { DocumentTypes } = require("@budibase/backend-core/db")
+const env = require("../../../environment")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+
+const migration = require("../appUrls")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+  const CouchDB = config.getCouch()
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    const app = await config.createApp("testApp")
+    const appDb = new CouchDB(app.appId)
+    let metadata = await appDb.get(DocumentTypes.APP_METADATA)
+    delete metadata.url
+    await appDb.put(metadata)
+    
+    await migration.run(appDb)
+
+    metadata = await appDb.get(DocumentTypes.APP_METADATA)
+    expect(metadata.url).toEqual("/testapp")
+  })
+})

package/src/migrations/{tests/usageQuotas/index.spec.js → functions/tests/quotas1.spec.js}

@@ -4,10 +4,10 @@ const TestConfig = require("../../../tests/utilities/TestConfiguration")
 const syncApps = jest.fn()
 const syncRows = jest.fn()
 
-jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) )
-jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) )
+jest.mock("../usageQuotas/syncApps", () => ({ run: syncApps }) )
+jest.mock("../usageQuotas/syncRows", () => ({ run: syncRows }) )
 
-const migrations = require("../../usageQuotas")
+const migration = require("../quotas1")
 
 describe("run", () => {
   let config = new TestConfig(false)
@@ -19,8 +19,8 @@ describe("run", () => {
 
   afterAll(config.end)  
 
-  it("runs the required migrations", async () => {
-    await migrations.run()
+  it("runs ", async () => {
+    await migration.run()
     expect(syncApps).toHaveBeenCalledTimes(1)
     expect(syncRows).toHaveBeenCalledTimes(1)
   })

package/src/migrations/functions/tests/userEmailViewCasing.spec.js

@@ -0,0 +1,25 @@
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+
+// mock email view creation
+const coreDb = require("@budibase/backend-core/db")
+const createUserEmailView = jest.fn()
+coreDb.createUserEmailView = createUserEmailView
+
+const migration = require("../userEmailViewCasing")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+  const globalDb = getGlobalDB()
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    await migration.run(globalDb)
+    expect(createUserEmailView).toHaveBeenCalledTimes(1)
+  })
+})

package/src/migrations/functions/usageQuotas/index.ts

@@ -0,0 +1,8 @@
+const { useQuotas } = require("../../../utilities/usageQuota")
+
+export const runQuotaMigration = async (migration: Function) => {
+  if (!useQuotas()) {
+    return
+  }
+  await migration()
+}

package/src/migrations/{usageQuotas/syncApps.js → functions/usageQuotas/syncApps.ts}

@@ -1,9 +1,9 @@
 const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
 const { getAllApps } = require("@budibase/backend-core/db")
-const CouchDB = require("../../db")
-const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
+import CouchDB from "../../../db"
+import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
 
-exports.run = async () => {
+export const run = async () => {
   const db = getGlobalDB()
   // get app count
   const devApps = await getAllApps(CouchDB, { dev: true })