@budibase/server 1.0.49-alpha.0 → 1.0.49-alpha.12

package/src/api/controllers/table/utils.js

@@ -1,13 +1,17 @@
-const CouchDB = require("../../../db")
 const csvParser = require("../../../utilities/csvParser")
 const {
   getRowParams,
   generateRowID,
   InternalTables,
+  getTableParams,
+  BudibaseInternalDB,
 } = require("../../../db/utils")
-const { isEqual } = require("lodash/fp")
+const { isEqual } = require("lodash")
 const { AutoFieldSubTypes, FieldTypes } = require("../../../constants")
-const { inputProcessing } = require("../../../utilities/rowProcessor")
+const {
+  inputProcessing,
+  cleanupAttachments,
+} = require("../../../utilities/rowProcessor")
 const {
   USERS_TABLE_SCHEMA,
   SwitchableTypes,
@@ -21,8 +25,26 @@ const {
 const { getViews, saveView } = require("../view/utils")
 const viewTemplate = require("../view/viewBuilder")
 const usageQuota = require("../../../utilities/usageQuota")
+const { getAppDB } = require("@budibase/backend-core/context")
+const { cloneDeep } = require("lodash/fp")
+
+exports.clearColumns = async (table, columnNames) => {
+  const db = getAppDB()
+  const rows = await db.allDocs(
+    getRowParams(table._id, null, {
+      include_docs: true,
+    })
+  )
+  return db.bulkDocs(
+    rows.rows.map(({ doc }) => {
+      columnNames.forEach(colName => delete doc[colName])
+      return doc
+    })
+  )
+}
 
-exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
+exports.checkForColumnUpdates = async (oldTable, updatedTable) => {
+  const db = getAppDB()
   let updatedRows = []
   const rename = updatedTable._rename
   let deletedColumns = []
@@ -39,18 +61,22 @@ exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
         include_docs: true,
       })
     )
-    updatedRows = rows.rows.map(({ doc }) => {
+    const rawRows = rows.rows.map(({ doc }) => doc)
+    updatedRows = rawRows.map(row => {
+      row = cloneDeep(row)
       if (rename) {
-        doc[rename.updated] = doc[rename.old]
-        delete doc[rename.old]
+        row[rename.updated] = row[rename.old]
+        delete row[rename.old]
       } else if (deletedColumns.length !== 0) {
-        deletedColumns.forEach(colName => delete doc[colName])
+        deletedColumns.forEach(colName => delete row[colName])
       }
-      return doc
+      return row
     })
 
+    // cleanup any attachments from object storage for deleted attachment columns
+    await cleanupAttachments(updatedTable, { oldTable, rows: rawRows })
     // Update views
-    await exports.checkForViewUpdates(db, updatedTable, rename, deletedColumns)
+    await exports.checkForViewUpdates(updatedTable, rename, deletedColumns)
     delete updatedTable._rename
   }
   return { rows: updatedRows, table: updatedTable }
@@ -77,12 +103,12 @@ exports.makeSureTableUpToDate = (table, tableToSave) => {
   return tableToSave
 }
 
-exports.handleDataImport = async (appId, user, table, dataImport) => {
+exports.handleDataImport = async (user, table, dataImport) => {
   if (!dataImport || !dataImport.csvString) {
     return table
   }
 
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   // Populate the table with rows imported from CSV in a bulk update
   const data = await csvParser.transform({
     ...dataImport,
@@ -127,8 +153,8 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
   return table
 }
 
-exports.handleSearchIndexes = async (appId, table) => {
-  const db = new CouchDB(appId)
+exports.handleSearchIndexes = async table => {
+  const db = getAppDB()
   // create relevant search indexes
   if (table.indexes && table.indexes.length > 0) {
     const currentIndexes = await db.getIndexes()
@@ -185,12 +211,9 @@ exports.checkStaticTables = table => {
 }
 
 class TableSaveFunctions {
-  constructor({ db, ctx, oldTable, dataImport }) {
-    this.db = db
-    this.ctx = ctx
-    if (this.ctx && this.ctx.user) {
-      this.appId = this.ctx.appId
-    }
+  constructor({ user, oldTable, dataImport }) {
+    this.db = getAppDB()
+    this.user = user
     this.oldTable = oldTable
     this.dataImport = dataImport
     // any rows that need updated
@@ -208,24 +231,15 @@ class TableSaveFunctions {
 
   // when confirmed valid
   async mid(table) {
-    let response = await exports.checkForColumnUpdates(
-      this.db,
-      this.oldTable,
-      table
-    )
+    let response = await exports.checkForColumnUpdates(this.oldTable, table)
     this.rows = this.rows.concat(response.rows)
     return table
   }
 
   // after saving
   async after(table) {
-    table = await exports.handleSearchIndexes(this.appId, table)
-    table = await exports.handleDataImport(
-      this.appId,
-      this.ctx.user,
-      table,
-      this.dataImport
-    )
+    table = await exports.handleSearchIndexes(table)
+    table = await exports.handleDataImport(this.user, table, this.dataImport)
     return table
   }
 
@@ -234,8 +248,22 @@ class TableSaveFunctions {
   }
 }
 
-exports.getAllExternalTables = async (appId, datasourceId) => {
-  const db = new CouchDB(appId)
+exports.getAllInternalTables = async () => {
+  const db = getAppDB()
+  const internalTables = await db.allDocs(
+    getTableParams(null, {
+      include_docs: true,
+    })
+  )
+  return internalTables.rows.map(tableDoc => ({
+    ...tableDoc.doc,
+    type: "internal",
+    sourceId: BudibaseInternalDB._id,
+  }))
+}
+
+exports.getAllExternalTables = async datasourceId => {
+  const db = getAppDB()
   const datasource = await db.get(datasourceId)
   if (!datasource || !datasource.entities) {
     throw "Datasource is not configured fully."
@@ -243,25 +271,25 @@ exports.getAllExternalTables = async (appId, datasourceId) => {
   return datasource.entities
 }
 
-exports.getExternalTable = async (appId, datasourceId, tableName) => {
-  const entities = await exports.getAllExternalTables(appId, datasourceId)
+exports.getExternalTable = async (datasourceId, tableName) => {
+  const entities = await exports.getAllExternalTables(datasourceId)
   return entities[tableName]
 }
 
-exports.getTable = async (appId, tableId) => {
-  const db = new CouchDB(appId)
+exports.getTable = async tableId => {
+  const db = getAppDB()
   if (isExternalTable(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
     const datasource = await db.get(datasourceId)
-    const table = await exports.getExternalTable(appId, datasourceId, tableName)
+    const table = await exports.getExternalTable(datasourceId, tableName)
     return { ...table, sql: isSQL(datasource) }
   } else {
     return db.get(tableId)
   }
 }
 
-exports.checkForViewUpdates = async (db, table, rename, deletedColumns) => {
-  const views = await getViews(db)
+exports.checkForViewUpdates = async (table, rename, deletedColumns) => {
+  const views = await getViews()
   const tableViews = views.filter(view => view.meta.tableId === table._id)
 
   // Check each table view to see if impacted by this table action
@@ -323,7 +351,7 @@ exports.checkForViewUpdates = async (db, table, rename, deletedColumns) => {
     // Update view if required
     if (needsUpdated) {
       const newViewTemplate = viewTemplate(view.meta)
-      await saveView(db, null, view.name, newViewTemplate)
+      await saveView(null, view.name, newViewTemplate)
       if (!newViewTemplate.meta.schema) {
         newViewTemplate.meta.schema = table.schema
       }

package/src/api/controllers/user.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../../db")
 const {
   generateUserMetadataID,
   getUserMetadataParams,
@@ -11,12 +10,14 @@ const { isEqual } = require("lodash")
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const {
   getDevelopmentAppID,
-  getDeployedAppIDs,
+  getProdAppIDs,
+  dbExists,
 } = require("@budibase/backend-core/db")
-const { doesDatabaseExist } = require("../../utilities")
 const { UserStatus } = require("@budibase/backend-core/constants")
+const { getAppDB } = require("@budibase/backend-core/context")
 
-async function rawMetadata(db) {
+async function rawMetadata() {
+  const db = getAppDB()
   return (
     await db.allDocs(
       getUserMetadataParams(null, {
@@ -54,13 +55,10 @@ function combineMetadataAndUser(user, metadata) {
   return null
 }
 
-exports.syncGlobalUsers = async appId => {
+exports.syncGlobalUsers = async () => {
   // sync user metadata
-  const db = new CouchDB(appId)
-  const [users, metadata] = await Promise.all([
-    getGlobalUsers(appId),
-    rawMetadata(db),
-  ])
+  const db = getAppDB()
+  const [users, metadata] = await Promise.all([getGlobalUsers(), rawMetadata()])
   const toWrite = []
   for (let user of users) {
     const combined = await combineMetadataAndUser(user, metadata)
@@ -94,7 +92,7 @@ exports.syncUser = async function (ctx) {
   let prodAppIds
   // if they are a builder then get all production app IDs
   if ((user.builder && user.builder.global) || deleting) {
-    prodAppIds = await getDeployedAppIDs(CouchDB)
+    prodAppIds = await getProdAppIDs()
   } else {
     prodAppIds = Object.entries(roles)
       .filter(entry => entry[1] !== BUILTIN_ROLE_IDS.PUBLIC)
@@ -104,10 +102,10 @@ exports.syncUser = async function (ctx) {
     const roleId = roles[prodAppId]
     const devAppId = getDevelopmentAppID(prodAppId)
     for (let appId of [prodAppId, devAppId]) {
-      if (!(await doesDatabaseExist(appId))) {
+      if (!(await dbExists(appId))) {
         continue
       }
-      const db = new CouchDB(appId)
+      const db = getAppDB()
       const metadataId = generateUserMetadataID(userId)
       let metadata
       try {
@@ -143,8 +141,8 @@ exports.syncUser = async function (ctx) {
 }
 
 exports.fetchMetadata = async function (ctx) {
-  const database = new CouchDB(ctx.appId)
-  const global = await getGlobalUsers(ctx.appId)
+  const database = getAppDB()
+  const global = await getGlobalUsers()
   const metadata = await rawMetadata(database)
   const users = []
   for (let user of global) {
@@ -173,8 +171,7 @@ exports.updateSelfMetadata = async function (ctx) {
 }
 
 exports.updateMetadata = async function (ctx) {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const user = ctx.request.body
   // this isn't applicable to the user
   delete user.roles
@@ -186,7 +183,7 @@ exports.updateMetadata = async function (ctx) {
 }
 
 exports.destroyMetadata = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   try {
     const dbUser = await db.get(ctx.params.id)
     await db.remove(dbUser._id, dbUser._rev)
@@ -209,7 +206,7 @@ exports.setFlag = async function (ctx) {
     ctx.throw(400, "Must supply a 'flag' field in request body.")
   }
   const flagDocId = generateUserFlagID(userId)
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   let doc
   try {
     doc = await db.get(flagDocId)
@@ -224,7 +221,7 @@ exports.setFlag = async function (ctx) {
 exports.getFlags = async function (ctx) {
   const userId = ctx.user._id
   const docId = generateUserFlagID(userId)
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   let doc
   try {
     doc = await db.get(docId)

package/src/api/controllers/view/index.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../../../db")
 const viewTemplate = require("./viewBuilder")
 const { apiFileReturn } = require("../../../utilities/fileSystem")
 const exporters = require("./exporters")
@@ -6,14 +5,14 @@ const { saveView, getView, getViews, deleteView } = require("./utils")
 const { fetchView } = require("../row")
 const { getTable } = require("../table/utils")
 const { FieldTypes } = require("../../../constants")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 exports.fetch = async ctx => {
-  const db = new CouchDB(ctx.appId)
-  ctx.body = await getViews(db)
+  ctx.body = await getViews()
 }
 
 exports.save = async ctx => {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const { originalName, ...viewToSave } = ctx.request.body
   const view = viewTemplate(viewToSave)
 
@@ -21,7 +20,7 @@ exports.save = async ctx => {
     ctx.throw(400, "Cannot create view without a name")
   }
 
-  await saveView(db, originalName, viewToSave.name, view)
+  await saveView(originalName, viewToSave.name, view)
 
   // add views to table document
   const table = await db.get(ctx.request.body.tableId)
@@ -42,9 +41,9 @@ exports.save = async ctx => {
 }
 
 exports.destroy = async ctx => {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const viewName = decodeURI(ctx.params.viewName)
-  const view = await deleteView(db, viewName)
+  const view = await deleteView(viewName)
   const table = await db.get(view.meta.tableId)
   delete table.views[viewName]
   await db.put(table)
@@ -53,9 +52,8 @@ exports.destroy = async ctx => {
 }
 
 exports.exportView = async ctx => {
-  const db = new CouchDB(ctx.appId)
   const viewName = decodeURI(ctx.query.view)
-  const view = await getView(db, viewName)
+  const view = await getView(viewName)
 
   const format = ctx.query.format
   if (!format || !Object.values(exporters.ExportFormats).includes(format)) {
@@ -83,7 +81,7 @@ exports.exportView = async ctx => {
   let schema = view && view.meta && view.meta.schema
   if (!schema) {
     const tableId = ctx.params.tableId || view.meta.tableId
-    const table = await getTable(ctx.appId, tableId)
+    const table = await getTable(tableId)
     schema = table.schema
   }
 

package/src/api/controllers/view/utils.js

@@ -6,8 +6,10 @@ const {
   SEPARATOR,
 } = require("../../../db/utils")
 const env = require("../../../environment")
+const { getAppDB } = require("@budibase/backend-core/context")
 
-exports.getView = async (db, viewName) => {
+exports.getView = async viewName => {
+  const db = getAppDB()
   if (env.SELF_HOSTED) {
     const designDoc = await db.get("_design/database")
     return designDoc.views[viewName]
@@ -22,7 +24,8 @@ exports.getView = async (db, viewName) => {
   }
 }
 
-exports.getViews = async db => {
+exports.getViews = async () => {
+  const db = getAppDB()
   const response = []
   if (env.SELF_HOSTED) {
     const designDoc = await db.get("_design/database")
@@ -54,7 +57,8 @@ exports.getViews = async db => {
   return response
 }
 
-exports.saveView = async (db, originalName, viewName, viewTemplate) => {
+exports.saveView = async (originalName, viewName, viewTemplate) => {
+  const db = getAppDB()
   if (env.SELF_HOSTED) {
     const designDoc = await db.get("_design/database")
     designDoc.views = {
@@ -91,7 +95,8 @@ exports.saveView = async (db, originalName, viewName, viewTemplate) => {
   }
 }
 
-exports.deleteView = async (db, viewName) => {
+exports.deleteView = async viewName => {
+  const db = getAppDB()
   if (env.SELF_HOSTED) {
     const designDoc = await db.get("_design/database")
     const view = designDoc.views[viewName]

package/src/api/controllers/webhook.js

@@ -1,9 +1,9 @@
-const CouchDB = require("../../db")
 const { generateWebhookID, getWebhookParams } = require("../../db/utils")
 const toJsonSchema = require("to-json-schema")
 const validate = require("jsonschema").validate
 const triggers = require("../../automations/triggers")
-const { getDeployedAppID } = require("@budibase/backend-core/db")
+const { getProdAppID } = require("@budibase/backend-core/db")
+const { getAppDB, updateAppId } = require("@budibase/backend-core/context")
 
 const AUTOMATION_DESCRIPTION = "Generated from Webhook Schema"
 
@@ -23,7 +23,7 @@ exports.WebhookType = {
 }
 
 exports.fetch = async ctx => {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const response = await db.allDocs(
     getWebhookParams(null, {
       include_docs: true,
@@ -33,7 +33,7 @@ exports.fetch = async ctx => {
 }
 
 exports.save = async ctx => {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const webhook = ctx.request.body
   webhook.appId = ctx.appId
 
@@ -52,12 +52,13 @@ exports.save = async ctx => {
 }
 
 exports.destroy = async ctx => {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   ctx.body = await db.remove(ctx.params.id, ctx.params.rev)
 }
 
 exports.buildSchema = async ctx => {
-  const db = new CouchDB(ctx.params.instance)
+  updateAppId(ctx.params.instance)
+  const db = getAppDB()
   const webhook = await db.get(ctx.params.id)
   webhook.bodySchema = toJsonSchema(ctx.request.body)
   // update the automation outputs
@@ -81,9 +82,10 @@ exports.buildSchema = async ctx => {
 }
 
 exports.trigger = async ctx => {
-  const prodAppId = getDeployedAppID(ctx.params.instance)
+  const prodAppId = getProdAppID(ctx.params.instance)
+  updateAppId(prodAppId)
   try {
-    const db = new CouchDB(prodAppId)
+    const db = getAppDB()
     const webhook = await db.get(ctx.params.id)
     // validate against the schema
     if (webhook.bodySchema) {

package/src/api/routes/index.js

@@ -24,6 +24,7 @@ const backupRoutes = require("./backup")
 const metadataRoutes = require("./metadata")
 const devRoutes = require("./dev")
 const cloudRoutes = require("./cloud")
+const migrationRoutes = require("./migrations")
 
 exports.mainRoutes = [
   authRoutes,
@@ -53,6 +54,7 @@ exports.mainRoutes = [
   // this could be breaking as koa may recognise other routes as this
   tableRoutes,
   rowRoutes,
+  migrationRoutes,
 ]
 
 exports.staticRoutes = staticRoutes

package/src/api/routes/migrations.js

@@ -0,0 +1,14 @@
+const Router = require("@koa/router")
+const migrationsController = require("../controllers/migrations")
+const router = Router()
+const { internalApi } = require("@budibase/backend-core/auth")
+
+router
+  .post("/api/migrations/run", internalApi, migrationsController.migrate)
+  .get(
+    "/api/migrations/definitions",
+    internalApi,
+    migrationsController.fetchDefinitions
+  )
+
+module.exports = router

package/src/api/routes/tests/automation.spec.js

@@ -145,6 +145,7 @@ describe("/automations", () => {
       let table = await config.createTable()
       automation.definition.trigger.inputs.tableId = table._id
       automation.definition.steps[0].inputs.row.tableId = table._id
+      automation.appId = config.appId
       automation = await config.createAutomation(automation)
       await setup.delay(500)
       const res = await testAutomation(config, automation)

package/src/api/routes/tests/misc.spec.js

@@ -82,7 +82,6 @@ describe("run misc tests", () => {
         dataImport.schema[col] = { type: "string" }
       }
       await tableUtils.handleDataImport(
-        config.getAppId(),
         { userId: "test" },
         table,
         dataImport

package/src/api/routes/tests/routing.spec.js

@@ -1,10 +1,15 @@
 const setup = require("./utilities")
 const { basicScreen } = setup.structures
-const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
+const { checkBuilderEndpoint, runInProd } = require("./utilities/TestFunctions")
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
+const { doInAppContext } = require("@budibase/backend-core/context")
 
 const route = "/test"
 
+// there are checks which are disabled in test env,
+// these checks need to be enabled for this test
+
+
 describe("/routing", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
@@ -26,20 +31,24 @@ describe("/routing", () => {
 
   describe("fetch", () => {
     it("prevents a public user from accessing development app", async () => {
-      await request
-        .get(`/api/routing/client`)
-        .set(config.publicHeaders({ prodApp: false }))
-        .expect(302)
+      await runInProd(() => {
+        return request
+          .get(`/api/routing/client`)
+          .set(config.publicHeaders({ prodApp: false }))
+          .expect(302)
+      })
     })
 
     it("prevents a non builder from accessing development app", async () => {
-      await request
-        .get(`/api/routing/client`)
-        .set(await config.roleHeaders({
-          roleId: BUILTIN_ROLE_IDS.BASIC,
-          prodApp: false
-        }))
-        .expect(302)
+      await runInProd(async () => {
+        return request
+          .get(`/api/routing/client`)
+          .set(await config.roleHeaders({
+            roleId: BUILTIN_ROLE_IDS.BASIC,
+            prodApp: false
+          }))
+          .expect(302)
+      })
     })
     it("returns the correct routing for basic user", async () => {
       const res = await request

package/src/api/routes/tests/row.spec.js

@@ -1,6 +1,7 @@
 const { outputProcessing } = require("../../../utilities/rowProcessor")
 const setup = require("./utilities")
 const { basicRow } = setup.structures
+const { doInAppContext } = require("@budibase/backend-core/context")
 
 // mock the fetch for the search system
 jest.mock("node-fetch")
@@ -387,10 +388,12 @@ describe("/rows", () => {
       })
       // the environment needs configured for this
       await setup.switchToSelfHosted(async () => {
-        const enriched = await outputProcessing({ appId: config.getAppId() }, table, [row])
-        expect(enriched[0].attachment[0].url).toBe(
-          `/prod-budi-app-assets/${config.getAppId()}/attachments/test/thing.csv`
-        )
+        doInAppContext(config.getAppId(), async () => {
+          const enriched = await outputProcessing(table, [row])
+          expect(enriched[0].attachment[0].url).toBe(
+            `/prod-budi-app-assets/${config.getAppId()}/attachments/test/thing.csv`
+          )
+        })
       })
     })
   })