@budibase/server 1.0.49-alpha.6 → 1.0.49

package/src/api/controllers/table/utils.js

@@ -1,17 +1,13 @@
+const CouchDB = require("../../../db")
 const csvParser = require("../../../utilities/csvParser")
 const {
   getRowParams,
   generateRowID,
   InternalTables,
-  getTableParams,
-  BudibaseInternalDB,
 } = require("../../../db/utils")
-const { isEqual } = require("lodash")
+const { isEqual } = require("lodash/fp")
 const { AutoFieldSubTypes, FieldTypes } = require("../../../constants")
-const {
-  inputProcessing,
-  cleanupAttachments,
-} = require("../../../utilities/rowProcessor")
+const { inputProcessing } = require("../../../utilities/rowProcessor")
 const {
   USERS_TABLE_SCHEMA,
   SwitchableTypes,
@@ -20,31 +16,11 @@ const {
 const {
   isExternalTable,
   breakExternalTableId,
-  isSQL,
 } = require("../../../integrations/utils")
 const { getViews, saveView } = require("../view/utils")
 const viewTemplate = require("../view/viewBuilder")
-const usageQuota = require("../../../utilities/usageQuota")
-const { getAppDB } = require("@budibase/backend-core/context")
-const { cloneDeep } = require("lodash/fp")
-
-exports.clearColumns = async (table, columnNames) => {
-  const db = getAppDB()
-  const rows = await db.allDocs(
-    getRowParams(table._id, null, {
-      include_docs: true,
-    })
-  )
-  return db.bulkDocs(
-    rows.rows.map(({ doc }) => {
-      columnNames.forEach(colName => delete doc[colName])
-      return doc
-    })
-  )
-}
 
-exports.checkForColumnUpdates = async (oldTable, updatedTable) => {
-  const db = getAppDB()
+exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
   let updatedRows = []
   const rename = updatedTable._rename
   let deletedColumns = []
@@ -61,22 +37,18 @@ exports.checkForColumnUpdates = async (oldTable, updatedTable) => {
         include_docs: true,
       })
     )
-    const rawRows = rows.rows.map(({ doc }) => doc)
-    updatedRows = rawRows.map(row => {
-      row = cloneDeep(row)
+    updatedRows = rows.rows.map(({ doc }) => {
       if (rename) {
-        row[rename.updated] = row[rename.old]
-        delete row[rename.old]
+        doc[rename.updated] = doc[rename.old]
+        delete doc[rename.old]
       } else if (deletedColumns.length !== 0) {
-        deletedColumns.forEach(colName => delete row[colName])
+        deletedColumns.forEach(colName => delete doc[colName])
       }
-      return row
+      return doc
     })
 
-    // cleanup any attachments from object storage for deleted attachment columns
-    await cleanupAttachments(updatedTable, { oldTable, rows: rawRows })
     // Update views
-    await exports.checkForViewUpdates(updatedTable, rename, deletedColumns)
+    await exports.checkForViewUpdates(db, updatedTable, rename, deletedColumns)
     delete updatedTable._rename
   }
   return { rows: updatedRows, table: updatedTable }
@@ -103,12 +75,12 @@ exports.makeSureTableUpToDate = (table, tableToSave) => {
   return tableToSave
 }
 
-exports.handleDataImport = async (user, table, dataImport) => {
+exports.handleDataImport = async (appId, user, table, dataImport) => {
   if (!dataImport || !dataImport.csvString) {
     return table
   }
 
-  const db = getAppDB()
+  const db = new CouchDB(appId)
   // Populate the table with rows imported from CSV in a bulk update
   const data = await csvParser.transform({
     ...dataImport,
@@ -143,18 +115,14 @@ exports.handleDataImport = async (user, table, dataImport) => {
     finalData.push(row)
   }
 
-  await usageQuota.update(usageQuota.Properties.ROW, finalData.length, {
-    dryRun: true,
-  })
   await db.bulkDocs(finalData)
-  await usageQuota.update(usageQuota.Properties.ROW, finalData.length)
   let response = await db.put(table)
   table._rev = response._rev
   return table
 }
 
-exports.handleSearchIndexes = async table => {
-  const db = getAppDB()
+exports.handleSearchIndexes = async (appId, table) => {
+  const db = new CouchDB(appId)
   // create relevant search indexes
   if (table.indexes && table.indexes.length > 0) {
     const currentIndexes = await db.getIndexes()
@@ -211,9 +179,12 @@ exports.checkStaticTables = table => {
 }
 
 class TableSaveFunctions {
-  constructor({ user, oldTable, dataImport }) {
-    this.db = getAppDB()
-    this.user = user
+  constructor({ db, ctx, oldTable, dataImport }) {
+    this.db = db
+    this.ctx = ctx
+    if (this.ctx && this.ctx.user) {
+      this.appId = this.ctx.appId
+    }
     this.oldTable = oldTable
     this.dataImport = dataImport
     // any rows that need updated
@@ -231,15 +202,24 @@ class TableSaveFunctions {
 
   // when confirmed valid
   async mid(table) {
-    let response = await exports.checkForColumnUpdates(this.oldTable, table)
+    let response = await exports.checkForColumnUpdates(
+      this.db,
+      this.oldTable,
+      table
+    )
     this.rows = this.rows.concat(response.rows)
     return table
   }
 
   // after saving
   async after(table) {
-    table = await exports.handleSearchIndexes(table)
-    table = await exports.handleDataImport(this.user, table, this.dataImport)
+    table = await exports.handleSearchIndexes(this.appId, table)
+    table = await exports.handleDataImport(
+      this.appId,
+      this.ctx.user,
+      table,
+      this.dataImport
+    )
     return table
   }
 
@@ -248,22 +228,8 @@ class TableSaveFunctions {
   }
 }
 
-exports.getAllInternalTables = async () => {
-  const db = getAppDB()
-  const internalTables = await db.allDocs(
-    getTableParams(null, {
-      include_docs: true,
-    })
-  )
-  return internalTables.rows.map(tableDoc => ({
-    ...tableDoc.doc,
-    type: "internal",
-    sourceId: BudibaseInternalDB._id,
-  }))
-}
-
-exports.getAllExternalTables = async datasourceId => {
-  const db = getAppDB()
+exports.getAllExternalTables = async (appId, datasourceId) => {
+  const db = new CouchDB(appId)
   const datasource = await db.get(datasourceId)
   if (!datasource || !datasource.entities) {
     throw "Datasource is not configured fully."
@@ -271,25 +237,23 @@ exports.getAllExternalTables = async datasourceId => {
   return datasource.entities
 }
 
-exports.getExternalTable = async (datasourceId, tableName) => {
-  const entities = await exports.getAllExternalTables(datasourceId)
+exports.getExternalTable = async (appId, datasourceId, tableName) => {
+  const entities = await exports.getAllExternalTables(appId, datasourceId)
   return entities[tableName]
 }
 
-exports.getTable = async tableId => {
-  const db = getAppDB()
+exports.getTable = async (appId, tableId) => {
+  const db = new CouchDB(appId)
   if (isExternalTable(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
-    const datasource = await db.get(datasourceId)
-    const table = await exports.getExternalTable(datasourceId, tableName)
-    return { ...table, sql: isSQL(datasource) }
+    return exports.getExternalTable(appId, datasourceId, tableName)
   } else {
     return db.get(tableId)
   }
 }
 
-exports.checkForViewUpdates = async (table, rename, deletedColumns) => {
-  const views = await getViews()
+exports.checkForViewUpdates = async (db, table, rename, deletedColumns) => {
+  const views = await getViews(db)
   const tableViews = views.filter(view => view.meta.tableId === table._id)
 
   // Check each table view to see if impacted by this table action
@@ -351,7 +315,7 @@ exports.checkForViewUpdates = async (table, rename, deletedColumns) => {
     // Update view if required
     if (needsUpdated) {
       const newViewTemplate = viewTemplate(view.meta)
-      await saveView(null, view.name, newViewTemplate)
+      await saveView(db, null, view.name, newViewTemplate)
       if (!newViewTemplate.meta.schema) {
         newViewTemplate.meta.schema = table.schema
       }

package/src/api/controllers/user.js

@@ -1,3 +1,4 @@
+const CouchDB = require("../../db")
 const {
   generateUserMetadataID,
   getUserMetadataParams,
@@ -10,14 +11,12 @@ const { isEqual } = require("lodash")
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const {
   getDevelopmentAppID,
-  getProdAppIDs,
-  dbExists,
+  getDeployedAppIDs,
 } = require("@budibase/backend-core/db")
+const { doesDatabaseExist } = require("../../utilities")
 const { UserStatus } = require("@budibase/backend-core/constants")
-const { getAppDB } = require("@budibase/backend-core/context")
 
-async function rawMetadata() {
-  const db = getAppDB()
+async function rawMetadata(db) {
   return (
     await db.allDocs(
       getUserMetadataParams(null, {
@@ -55,10 +54,13 @@ function combineMetadataAndUser(user, metadata) {
   return null
 }
 
-exports.syncGlobalUsers = async () => {
+exports.syncGlobalUsers = async appId => {
   // sync user metadata
-  const db = getAppDB()
-  const [users, metadata] = await Promise.all([getGlobalUsers(), rawMetadata()])
+  const db = new CouchDB(appId)
+  const [users, metadata] = await Promise.all([
+    getGlobalUsers(appId),
+    rawMetadata(db),
+  ])
   const toWrite = []
   for (let user of users) {
     const combined = await combineMetadataAndUser(user, metadata)
@@ -92,7 +94,7 @@ exports.syncUser = async function (ctx) {
   let prodAppIds
   // if they are a builder then get all production app IDs
   if ((user.builder && user.builder.global) || deleting) {
-    prodAppIds = await getProdAppIDs()
+    prodAppIds = await getDeployedAppIDs(CouchDB)
   } else {
     prodAppIds = Object.entries(roles)
       .filter(entry => entry[1] !== BUILTIN_ROLE_IDS.PUBLIC)
@@ -102,10 +104,10 @@ exports.syncUser = async function (ctx) {
     const roleId = roles[prodAppId]
     const devAppId = getDevelopmentAppID(prodAppId)
     for (let appId of [prodAppId, devAppId]) {
-      if (!(await dbExists(appId))) {
+      if (!(await doesDatabaseExist(appId))) {
         continue
       }
-      const db = getAppDB()
+      const db = new CouchDB(appId)
       const metadataId = generateUserMetadataID(userId)
       let metadata
       try {
@@ -141,8 +143,8 @@ exports.syncUser = async function (ctx) {
 }
 
 exports.fetchMetadata = async function (ctx) {
-  const database = getAppDB()
-  const global = await getGlobalUsers()
+  const database = new CouchDB(ctx.appId)
+  const global = await getGlobalUsers(ctx.appId)
   const metadata = await rawMetadata(database)
   const users = []
   for (let user of global) {
@@ -165,13 +167,12 @@ exports.updateSelfMetadata = async function (ctx) {
   ctx.request.body._id = ctx.user._id
   // make sure no stale rev
   delete ctx.request.body._rev
-  // make sure no csrf token
-  delete ctx.request.body.csrfToken
   await exports.updateMetadata(ctx)
 }
 
 exports.updateMetadata = async function (ctx) {
-  const db = getAppDB()
+  const appId = ctx.appId
+  const db = new CouchDB(appId)
   const user = ctx.request.body
   // this isn't applicable to the user
   delete user.roles
@@ -183,7 +184,7 @@ exports.updateMetadata = async function (ctx) {
 }
 
 exports.destroyMetadata = async function (ctx) {
-  const db = getAppDB()
+  const db = new CouchDB(ctx.appId)
   try {
     const dbUser = await db.get(ctx.params.id)
     await db.remove(dbUser._id, dbUser._rev)
@@ -206,7 +207,7 @@ exports.setFlag = async function (ctx) {
     ctx.throw(400, "Must supply a 'flag' field in request body.")
   }
   const flagDocId = generateUserFlagID(userId)
-  const db = getAppDB()
+  const db = new CouchDB(ctx.appId)
   let doc
   try {
     doc = await db.get(flagDocId)
@@ -221,7 +222,7 @@ exports.setFlag = async function (ctx) {
 exports.getFlags = async function (ctx) {
   const userId = ctx.user._id
   const docId = generateUserFlagID(userId)
-  const db = getAppDB()
+  const db = new CouchDB(ctx.appId)
   let doc
   try {
     doc = await db.get(docId)

package/src/api/controllers/view/index.js

@@ -1,3 +1,4 @@
+const CouchDB = require("../../../db")
 const viewTemplate = require("./viewBuilder")
 const { apiFileReturn } = require("../../../utilities/fileSystem")
 const exporters = require("./exporters")
@@ -5,14 +6,14 @@ const { saveView, getView, getViews, deleteView } = require("./utils")
 const { fetchView } = require("../row")
 const { getTable } = require("../table/utils")
 const { FieldTypes } = require("../../../constants")
-const { getAppDB } = require("@budibase/backend-core/context")
 
 exports.fetch = async ctx => {
-  ctx.body = await getViews()
+  const db = new CouchDB(ctx.appId)
+  ctx.body = await getViews(db)
 }
 
 exports.save = async ctx => {
-  const db = getAppDB()
+  const db = new CouchDB(ctx.appId)
   const { originalName, ...viewToSave } = ctx.request.body
   const view = viewTemplate(viewToSave)
 
@@ -20,7 +21,7 @@ exports.save = async ctx => {
     ctx.throw(400, "Cannot create view without a name")
   }
 
-  await saveView(originalName, viewToSave.name, view)
+  await saveView(db, originalName, viewToSave.name, view)
 
   // add views to table document
   const table = await db.get(ctx.request.body.tableId)
@@ -41,9 +42,9 @@ exports.save = async ctx => {
 }
 
 exports.destroy = async ctx => {
-  const db = getAppDB()
+  const db = new CouchDB(ctx.appId)
   const viewName = decodeURI(ctx.params.viewName)
-  const view = await deleteView(viewName)
+  const view = await deleteView(db, viewName)
   const table = await db.get(view.meta.tableId)
   delete table.views[viewName]
   await db.put(table)
@@ -52,8 +53,9 @@ exports.destroy = async ctx => {
 }
 
 exports.exportView = async ctx => {
+  const db = new CouchDB(ctx.appId)
   const viewName = decodeURI(ctx.query.view)
-  const view = await getView(viewName)
+  const view = await getView(db, viewName)
 
   const format = ctx.query.format
   if (!format || !Object.values(exporters.ExportFormats).includes(format)) {
@@ -81,7 +83,7 @@ exports.exportView = async ctx => {
   let schema = view && view.meta && view.meta.schema
   if (!schema) {
     const tableId = ctx.params.tableId || view.meta.tableId
-    const table = await getTable(tableId)
+    const table = await getTable(ctx.appId, tableId)
     schema = table.schema
   }
 

package/src/api/controllers/view/utils.js

@@ -6,10 +6,8 @@ const {
   SEPARATOR,
 } = require("../../../db/utils")
 const env = require("../../../environment")
-const { getAppDB } = require("@budibase/backend-core/context")
 
-exports.getView = async viewName => {
-  const db = getAppDB()
+exports.getView = async (db, viewName) => {
   if (env.SELF_HOSTED) {
     const designDoc = await db.get("_design/database")
     return designDoc.views[viewName]
@@ -24,8 +22,7 @@ exports.getView = async viewName => {
   }
 }
 
-exports.getViews = async () => {
-  const db = getAppDB()
+exports.getViews = async db => {
   const response = []
   if (env.SELF_HOSTED) {
     const designDoc = await db.get("_design/database")
@@ -57,8 +54,7 @@ exports.getViews = async () => {
   return response
 }
 
-exports.saveView = async (originalName, viewName, viewTemplate) => {
-  const db = getAppDB()
+exports.saveView = async (db, originalName, viewName, viewTemplate) => {
   if (env.SELF_HOSTED) {
     const designDoc = await db.get("_design/database")
     designDoc.views = {
@@ -95,8 +91,7 @@ exports.saveView = async (originalName, viewName, viewTemplate) => {
   }
 }
 
-exports.deleteView = async viewName => {
-  const db = getAppDB()
+exports.deleteView = async (db, viewName) => {
   if (env.SELF_HOSTED) {
     const designDoc = await db.get("_design/database")
     const view = designDoc.views[viewName]

package/src/api/controllers/webhook.js

@@ -1,9 +1,9 @@
+const CouchDB = require("../../db")
 const { generateWebhookID, getWebhookParams } = require("../../db/utils")
 const toJsonSchema = require("to-json-schema")
 const validate = require("jsonschema").validate
 const triggers = require("../../automations/triggers")
-const { getProdAppID } = require("@budibase/backend-core/db")
-const { getAppDB, updateAppId } = require("@budibase/backend-core/context")
+const { getDeployedAppID } = require("@budibase/backend-core/db")
 
 const AUTOMATION_DESCRIPTION = "Generated from Webhook Schema"
 
@@ -23,7 +23,7 @@ exports.WebhookType = {
 }
 
 exports.fetch = async ctx => {
-  const db = getAppDB()
+  const db = new CouchDB(ctx.appId)
   const response = await db.allDocs(
     getWebhookParams(null, {
       include_docs: true,
@@ -33,7 +33,7 @@ exports.fetch = async ctx => {
 }
 
 exports.save = async ctx => {
-  const db = getAppDB()
+  const db = new CouchDB(ctx.appId)
   const webhook = ctx.request.body
   webhook.appId = ctx.appId
 
@@ -52,13 +52,12 @@ exports.save = async ctx => {
 }
 
 exports.destroy = async ctx => {
-  const db = getAppDB()
+  const db = new CouchDB(ctx.appId)
   ctx.body = await db.remove(ctx.params.id, ctx.params.rev)
 }
 
 exports.buildSchema = async ctx => {
-  updateAppId(ctx.params.instance)
-  const db = getAppDB()
+  const db = new CouchDB(ctx.params.instance)
   const webhook = await db.get(ctx.params.id)
   webhook.bodySchema = toJsonSchema(ctx.request.body)
   // update the automation outputs
@@ -82,10 +81,9 @@ exports.buildSchema = async ctx => {
 }
 
 exports.trigger = async ctx => {
-  const prodAppId = getProdAppID(ctx.params.instance)
-  updateAppId(prodAppId)
+  const prodAppId = getDeployedAppID(ctx.params.instance)
   try {
-    const db = getAppDB()
+    const db = new CouchDB(prodAppId)
     const webhook = await db.get(ctx.params.id)
     // validate against the schema
     if (webhook.bodySchema) {

package/src/api/routes/hosting.js

@@ -0,0 +1,13 @@
+const Router = require("@koa/router")
+const controller = require("../controllers/hosting")
+const authorized = require("../../middleware/authorized")
+const { BUILDER } = require("@budibase/backend-core/permissions")
+
+const router = Router()
+
+router
+  .get("/api/hosting/urls", authorized(BUILDER), controller.fetchUrls)
+  // this isn't risky, doesn't return anything about apps other than names and URLs
+  .get("/api/hosting/apps", controller.getDeployedApps)
+
+module.exports = router

package/src/api/routes/index.js

@@ -20,11 +20,11 @@ const integrationRoutes = require("./integration")
 const permissionRoutes = require("./permission")
 const datasourceRoutes = require("./datasource")
 const queryRoutes = require("./query")
+const hostingRoutes = require("./hosting")
 const backupRoutes = require("./backup")
 const metadataRoutes = require("./metadata")
 const devRoutes = require("./dev")
 const cloudRoutes = require("./cloud")
-const migrationRoutes = require("./migrations")
 
 exports.mainRoutes = [
   authRoutes,
@@ -46,6 +46,7 @@ exports.mainRoutes = [
   permissionRoutes,
   datasourceRoutes,
   queryRoutes,
+  hostingRoutes,
   backupRoutes,
   metadataRoutes,
   devRoutes,
@@ -54,7 +55,6 @@ exports.mainRoutes = [
   // this could be breaking as koa may recognise other routes as this
   tableRoutes,
   rowRoutes,
-  migrationRoutes,
 ]
 
 exports.staticRoutes = staticRoutes

package/src/api/routes/static.js

@@ -46,10 +46,5 @@ router
   )
   // TODO: this likely needs to be secured in some way
   .get("/:appId/:path*", controller.serveApp)
-  .post(
-    "/api/attachments/:datasourceId/url",
-    authorized(PermissionTypes.TABLE, PermissionLevels.READ),
-    controller.getSignedUploadURL
-  )
 
 module.exports = router

package/src/api/routes/tests/application.spec.js

@@ -53,8 +53,8 @@ describe("/applications", () => {
 
   describe("fetch", () => {
     it("lists all applications", async () => {
-      await config.createApp("app1")
-      await config.createApp("app2")
+      await config.createApp(request, "app1")
+      await config.createApp(request, "app2")
 
       const res = await request
         .get(`/api/applications?status=${AppStatus.DEV}`)

package/src/api/routes/tests/auth.spec.js

@@ -13,9 +13,10 @@ describe("/authenticate", () => {
 
   describe("fetch self", () => {
     it("should be able to fetch self", async () => {
+      const headers = await config.login()
       const res = await request
         .get(`/api/self`)
-        .set(config.defaultHeaders())
+        .set(headers)
         .expect("Content-Type", /json/)
         .expect(200)
       expect(res.body._id).toEqual(generateUserMetadataID("us_uuid1"))

package/src/api/routes/tests/automation.spec.js

@@ -145,7 +145,6 @@ describe("/automations", () => {
       let table = await config.createTable()
       automation.definition.trigger.inputs.tableId = table._id
       automation.definition.steps[0].inputs.row.tableId = table._id
-      automation.appId = config.appId
       automation = await config.createAutomation(automation)
       await setup.delay(500)
       const res = await testAutomation(config, automation)

package/src/api/routes/tests/hosting.spec.js

@@ -0,0 +1,36 @@
+// mock out node fetch for this
+jest.mock("node-fetch")
+
+const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
+const setup = require("./utilities")
+
+describe("/hosting", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+  let app
+
+  afterAll(setup.afterAll)
+
+  beforeEach(async () => {
+    app = await config.init()
+  })
+
+  describe("fetchUrls", () => {
+    it("should be able to fetch current app URLs", async () => {
+      const res = await request
+        .get(`/api/hosting/urls`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      expect(res.body.app).toEqual(`http://localhost:10000/app`)
+    })
+
+    it("should apply authorization to endpoint", async () => {
+      await checkBuilderEndpoint({
+        config,
+        method: "GET",
+        url: `/api/hosting/urls`,
+      })
+    })
+  })
+})

package/src/api/routes/tests/misc.spec.js

@@ -82,6 +82,7 @@ describe("run misc tests", () => {
         dataImport.schema[col] = { type: "string" }
       }
       await tableUtils.handleDataImport(
+        config.getAppId(),
         { userId: "test" },
         table,
         dataImport