@budibase/server 1.0.49 → 1.0.50-alpha.3

package/src/api/controllers/table/utils.js

@@ -1,13 +1,17 @@
-const CouchDB = require("../../../db")
 const csvParser = require("../../../utilities/csvParser")
 const {
   getRowParams,
   generateRowID,
   InternalTables,
+  getTableParams,
+  BudibaseInternalDB,
 } = require("../../../db/utils")
-const { isEqual } = require("lodash/fp")
+const { isEqual } = require("lodash")
 const { AutoFieldSubTypes, FieldTypes } = require("../../../constants")
-const { inputProcessing } = require("../../../utilities/rowProcessor")
+const {
+  inputProcessing,
+  cleanupAttachments,
+} = require("../../../utilities/rowProcessor")
 const {
   USERS_TABLE_SCHEMA,
   SwitchableTypes,
@@ -16,11 +20,31 @@ const {
 const {
   isExternalTable,
   breakExternalTableId,
+  isSQL,
 } = require("../../../integrations/utils")
 const { getViews, saveView } = require("../view/utils")
 const viewTemplate = require("../view/viewBuilder")
+const usageQuota = require("../../../utilities/usageQuota")
+const { getAppDB } = require("@budibase/backend-core/context")
+const { cloneDeep } = require("lodash/fp")
+
+exports.clearColumns = async (table, columnNames) => {
+  const db = getAppDB()
+  const rows = await db.allDocs(
+    getRowParams(table._id, null, {
+      include_docs: true,
+    })
+  )
+  return db.bulkDocs(
+    rows.rows.map(({ doc }) => {
+      columnNames.forEach(colName => delete doc[colName])
+      return doc
+    })
+  )
+}
 
-exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
+exports.checkForColumnUpdates = async (oldTable, updatedTable) => {
+  const db = getAppDB()
   let updatedRows = []
   const rename = updatedTable._rename
   let deletedColumns = []
@@ -37,18 +61,22 @@ exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
         include_docs: true,
       })
     )
-    updatedRows = rows.rows.map(({ doc }) => {
+    const rawRows = rows.rows.map(({ doc }) => doc)
+    updatedRows = rawRows.map(row => {
+      row = cloneDeep(row)
       if (rename) {
-        doc[rename.updated] = doc[rename.old]
-        delete doc[rename.old]
+        row[rename.updated] = row[rename.old]
+        delete row[rename.old]
       } else if (deletedColumns.length !== 0) {
-        deletedColumns.forEach(colName => delete doc[colName])
+        deletedColumns.forEach(colName => delete row[colName])
       }
-      return doc
+      return row
     })
 
+    // cleanup any attachments from object storage for deleted attachment columns
+    await cleanupAttachments(updatedTable, { oldTable, rows: rawRows })
     // Update views
-    await exports.checkForViewUpdates(db, updatedTable, rename, deletedColumns)
+    await exports.checkForViewUpdates(updatedTable, rename, deletedColumns)
     delete updatedTable._rename
   }
   return { rows: updatedRows, table: updatedTable }
@@ -75,12 +103,12 @@ exports.makeSureTableUpToDate = (table, tableToSave) => {
   return tableToSave
 }
 
-exports.handleDataImport = async (appId, user, table, dataImport) => {
+exports.handleDataImport = async (user, table, dataImport) => {
   if (!dataImport || !dataImport.csvString) {
     return table
   }
 
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   // Populate the table with rows imported from CSV in a bulk update
   const data = await csvParser.transform({
     ...dataImport,
@@ -115,14 +143,18 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
     finalData.push(row)
   }
 
+  await usageQuota.update(usageQuota.Properties.ROW, finalData.length, {
+    dryRun: true,
+  })
   await db.bulkDocs(finalData)
+  await usageQuota.update(usageQuota.Properties.ROW, finalData.length)
   let response = await db.put(table)
   table._rev = response._rev
   return table
 }
 
-exports.handleSearchIndexes = async (appId, table) => {
-  const db = new CouchDB(appId)
+exports.handleSearchIndexes = async table => {
+  const db = getAppDB()
   // create relevant search indexes
   if (table.indexes && table.indexes.length > 0) {
     const currentIndexes = await db.getIndexes()
@@ -179,12 +211,9 @@ exports.checkStaticTables = table => {
 }
 
 class TableSaveFunctions {
-  constructor({ db, ctx, oldTable, dataImport }) {
-    this.db = db
-    this.ctx = ctx
-    if (this.ctx && this.ctx.user) {
-      this.appId = this.ctx.appId
-    }
+  constructor({ user, oldTable, dataImport }) {
+    this.db = getAppDB()
+    this.user = user
     this.oldTable = oldTable
     this.dataImport = dataImport
     // any rows that need updated
@@ -202,24 +231,15 @@ class TableSaveFunctions {
 
   // when confirmed valid
   async mid(table) {
-    let response = await exports.checkForColumnUpdates(
-      this.db,
-      this.oldTable,
-      table
-    )
+    let response = await exports.checkForColumnUpdates(this.oldTable, table)
     this.rows = this.rows.concat(response.rows)
     return table
   }
 
   // after saving
   async after(table) {
-    table = await exports.handleSearchIndexes(this.appId, table)
-    table = await exports.handleDataImport(
-      this.appId,
-      this.ctx.user,
-      table,
-      this.dataImport
-    )
+    table = await exports.handleSearchIndexes(table)
+    table = await exports.handleDataImport(this.user, table, this.dataImport)
     return table
   }
 
@@ -228,8 +248,22 @@ class TableSaveFunctions {
   }
 }
 
-exports.getAllExternalTables = async (appId, datasourceId) => {
-  const db = new CouchDB(appId)
+exports.getAllInternalTables = async () => {
+  const db = getAppDB()
+  const internalTables = await db.allDocs(
+    getTableParams(null, {
+      include_docs: true,
+    })
+  )
+  return internalTables.rows.map(tableDoc => ({
+    ...tableDoc.doc,
+    type: "internal",
+    sourceId: BudibaseInternalDB._id,
+  }))
+}
+
+exports.getAllExternalTables = async datasourceId => {
+  const db = getAppDB()
   const datasource = await db.get(datasourceId)
   if (!datasource || !datasource.entities) {
     throw "Datasource is not configured fully."
@@ -237,23 +271,25 @@ exports.getAllExternalTables = async (appId, datasourceId) => {
   return datasource.entities
 }
 
-exports.getExternalTable = async (appId, datasourceId, tableName) => {
-  const entities = await exports.getAllExternalTables(appId, datasourceId)
+exports.getExternalTable = async (datasourceId, tableName) => {
+  const entities = await exports.getAllExternalTables(datasourceId)
   return entities[tableName]
 }
 
-exports.getTable = async (appId, tableId) => {
-  const db = new CouchDB(appId)
+exports.getTable = async tableId => {
+  const db = getAppDB()
   if (isExternalTable(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
-    return exports.getExternalTable(appId, datasourceId, tableName)
+    const datasource = await db.get(datasourceId)
+    const table = await exports.getExternalTable(datasourceId, tableName)
+    return { ...table, sql: isSQL(datasource) }
   } else {
     return db.get(tableId)
   }
 }
 
-exports.checkForViewUpdates = async (db, table, rename, deletedColumns) => {
-  const views = await getViews(db)
+exports.checkForViewUpdates = async (table, rename, deletedColumns) => {
+  const views = await getViews()
   const tableViews = views.filter(view => view.meta.tableId === table._id)
 
   // Check each table view to see if impacted by this table action
@@ -315,7 +351,7 @@ exports.checkForViewUpdates = async (db, table, rename, deletedColumns) => {
     // Update view if required
     if (needsUpdated) {
       const newViewTemplate = viewTemplate(view.meta)
-      await saveView(db, null, view.name, newViewTemplate)
+      await saveView(null, view.name, newViewTemplate)
       if (!newViewTemplate.meta.schema) {
         newViewTemplate.meta.schema = table.schema
       }

package/src/api/controllers/user.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../../db")
 const {
   generateUserMetadataID,
   getUserMetadataParams,
@@ -11,12 +10,14 @@ const { isEqual } = require("lodash")
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const {
   getDevelopmentAppID,
-  getDeployedAppIDs,
+  getProdAppIDs,
+  dbExists,
 } = require("@budibase/backend-core/db")
-const { doesDatabaseExist } = require("../../utilities")
 const { UserStatus } = require("@budibase/backend-core/constants")
+const { getAppDB, doInAppContext } = require("@budibase/backend-core/context")
 
-async function rawMetadata(db) {
+async function rawMetadata() {
+  const db = getAppDB()
   return (
     await db.allDocs(
       getUserMetadataParams(null, {
@@ -54,13 +55,10 @@ function combineMetadataAndUser(user, metadata) {
   return null
 }
 
-exports.syncGlobalUsers = async appId => {
+exports.syncGlobalUsers = async () => {
   // sync user metadata
-  const db = new CouchDB(appId)
-  const [users, metadata] = await Promise.all([
-    getGlobalUsers(appId),
-    rawMetadata(db),
-  ])
+  const db = getAppDB()
+  const [users, metadata] = await Promise.all([getGlobalUsers(), rawMetadata()])
   const toWrite = []
   for (let user of users) {
     const combined = await combineMetadataAndUser(user, metadata)
@@ -94,7 +92,7 @@ exports.syncUser = async function (ctx) {
   let prodAppIds
   // if they are a builder then get all production app IDs
   if ((user.builder && user.builder.global) || deleting) {
-    prodAppIds = await getDeployedAppIDs(CouchDB)
+    prodAppIds = await getProdAppIDs()
   } else {
     prodAppIds = Object.entries(roles)
       .filter(entry => entry[1] !== BUILTIN_ROLE_IDS.PUBLIC)
@@ -104,37 +102,39 @@ exports.syncUser = async function (ctx) {
     const roleId = roles[prodAppId]
     const devAppId = getDevelopmentAppID(prodAppId)
     for (let appId of [prodAppId, devAppId]) {
-      if (!(await doesDatabaseExist(appId))) {
+      if (!(await dbExists(appId))) {
         continue
       }
-      const db = new CouchDB(appId)
-      const metadataId = generateUserMetadataID(userId)
-      let metadata
-      try {
-        metadata = await db.get(metadataId)
-      } catch (err) {
-        if (deleting) {
-          continue
+      await doInAppContext(appId, async () => {
+        const db = getAppDB()
+        const metadataId = generateUserMetadataID(userId)
+        let metadata
+        try {
+          metadata = await db.get(metadataId)
+        } catch (err) {
+          if (deleting) {
+            return
+          }
+          metadata = {
+            tableId: InternalTables.USER_METADATA,
+          }
         }
-        metadata = {
-          tableId: InternalTables.USER_METADATA,
+        // assign the roleId for the metadata doc
+        if (roleId) {
+          metadata.roleId = roleId
         }
-      }
-      // assign the roleId for the metadata doc
-      if (roleId) {
-        metadata.roleId = roleId
-      }
-      let combined = !deleting
-        ? combineMetadataAndUser(user, metadata)
-        : {
-            ...metadata,
-            status: UserStatus.INACTIVE,
-            metadata: BUILTIN_ROLE_IDS.PUBLIC,
-          }
-      // if its null then there was no updates required
-      if (combined) {
-        await db.put(combined)
-      }
+        let combined = !deleting
+          ? combineMetadataAndUser(user, metadata)
+          : {
+              ...metadata,
+              status: UserStatus.INACTIVE,
+              metadata: BUILTIN_ROLE_IDS.PUBLIC,
+            }
+        // if its null then there was no updates required
+        if (combined) {
+          await db.put(combined)
+        }
+      })
     }
   }
   ctx.body = {
@@ -143,8 +143,8 @@ exports.syncUser = async function (ctx) {
 }
 
 exports.fetchMetadata = async function (ctx) {
-  const database = new CouchDB(ctx.appId)
-  const global = await getGlobalUsers(ctx.appId)
+  const database = getAppDB()
+  const global = await getGlobalUsers()
   const metadata = await rawMetadata(database)
   const users = []
   for (let user of global) {
@@ -167,12 +167,13 @@ exports.updateSelfMetadata = async function (ctx) {
   ctx.request.body._id = ctx.user._id
   // make sure no stale rev
   delete ctx.request.body._rev
+  // make sure no csrf token
+  delete ctx.request.body.csrfToken
   await exports.updateMetadata(ctx)
 }
 
 exports.updateMetadata = async function (ctx) {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const user = ctx.request.body
   // this isn't applicable to the user
   delete user.roles
@@ -184,7 +185,7 @@ exports.updateMetadata = async function (ctx) {
 }
 
 exports.destroyMetadata = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   try {
     const dbUser = await db.get(ctx.params.id)
     await db.remove(dbUser._id, dbUser._rev)
@@ -207,7 +208,7 @@ exports.setFlag = async function (ctx) {
     ctx.throw(400, "Must supply a 'flag' field in request body.")
   }
   const flagDocId = generateUserFlagID(userId)
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   let doc
   try {
     doc = await db.get(flagDocId)
@@ -222,7 +223,7 @@ exports.setFlag = async function (ctx) {
 exports.getFlags = async function (ctx) {
   const userId = ctx.user._id
   const docId = generateUserFlagID(userId)
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   let doc
   try {
     doc = await db.get(docId)

package/src/api/controllers/view/index.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../../../db")
 const viewTemplate = require("./viewBuilder")
 const { apiFileReturn } = require("../../../utilities/fileSystem")
 const exporters = require("./exporters")
@@ -6,14 +5,14 @@ const { saveView, getView, getViews, deleteView } = require("./utils")
 const { fetchView } = require("../row")
 const { getTable } = require("../table/utils")
 const { FieldTypes } = require("../../../constants")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 exports.fetch = async ctx => {
-  const db = new CouchDB(ctx.appId)
-  ctx.body = await getViews(db)
+  ctx.body = await getViews()
 }
 
 exports.save = async ctx => {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const { originalName, ...viewToSave } = ctx.request.body
   const view = viewTemplate(viewToSave)
 
@@ -21,7 +20,7 @@ exports.save = async ctx => {
     ctx.throw(400, "Cannot create view without a name")
   }
 
-  await saveView(db, originalName, viewToSave.name, view)
+  await saveView(originalName, viewToSave.name, view)
 
   // add views to table document
   const table = await db.get(ctx.request.body.tableId)
@@ -42,9 +41,9 @@ exports.save = async ctx => {
 }
 
 exports.destroy = async ctx => {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const viewName = decodeURI(ctx.params.viewName)
-  const view = await deleteView(db, viewName)
+  const view = await deleteView(viewName)
   const table = await db.get(view.meta.tableId)
   delete table.views[viewName]
   await db.put(table)
@@ -53,9 +52,8 @@ exports.destroy = async ctx => {
 }
 
 exports.exportView = async ctx => {
-  const db = new CouchDB(ctx.appId)
   const viewName = decodeURI(ctx.query.view)
-  const view = await getView(db, viewName)
+  const view = await getView(viewName)
 
   const format = ctx.query.format
   if (!format || !Object.values(exporters.ExportFormats).includes(format)) {
@@ -83,7 +81,7 @@ exports.exportView = async ctx => {
   let schema = view && view.meta && view.meta.schema
   if (!schema) {
     const tableId = ctx.params.tableId || view.meta.tableId
-    const table = await getTable(ctx.appId, tableId)
+    const table = await getTable(tableId)
     schema = table.schema
   }
 

package/src/api/controllers/view/utils.js

@@ -6,8 +6,10 @@ const {
   SEPARATOR,
 } = require("../../../db/utils")
 const env = require("../../../environment")
+const { getAppDB } = require("@budibase/backend-core/context")
 
-exports.getView = async (db, viewName) => {
+exports.getView = async viewName => {
+  const db = getAppDB()
   if (env.SELF_HOSTED) {
     const designDoc = await db.get("_design/database")
     return designDoc.views[viewName]
@@ -22,7 +24,8 @@ exports.getView = async (db, viewName) => {
   }
 }
 
-exports.getViews = async db => {
+exports.getViews = async () => {
+  const db = getAppDB()
   const response = []
   if (env.SELF_HOSTED) {
     const designDoc = await db.get("_design/database")
@@ -54,7 +57,8 @@ exports.getViews = async db => {
   return response
 }
 
-exports.saveView = async (db, originalName, viewName, viewTemplate) => {
+exports.saveView = async (originalName, viewName, viewTemplate) => {
+  const db = getAppDB()
   if (env.SELF_HOSTED) {
     const designDoc = await db.get("_design/database")
     designDoc.views = {
@@ -91,7 +95,8 @@ exports.saveView = async (db, originalName, viewName, viewTemplate) => {
   }
 }
 
-exports.deleteView = async (db, viewName) => {
+exports.deleteView = async viewName => {
+  const db = getAppDB()
   if (env.SELF_HOSTED) {
     const designDoc = await db.get("_design/database")
     const view = designDoc.views[viewName]

package/src/api/controllers/webhook.js

@@ -1,9 +1,9 @@
-const CouchDB = require("../../db")
 const { generateWebhookID, getWebhookParams } = require("../../db/utils")
 const toJsonSchema = require("to-json-schema")
 const validate = require("jsonschema").validate
 const triggers = require("../../automations/triggers")
-const { getDeployedAppID } = require("@budibase/backend-core/db")
+const { getProdAppID } = require("@budibase/backend-core/db")
+const { getAppDB, updateAppId } = require("@budibase/backend-core/context")
 
 const AUTOMATION_DESCRIPTION = "Generated from Webhook Schema"
 
@@ -23,7 +23,7 @@ exports.WebhookType = {
 }
 
 exports.fetch = async ctx => {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const response = await db.allDocs(
     getWebhookParams(null, {
       include_docs: true,
@@ -33,7 +33,7 @@ exports.fetch = async ctx => {
 }
 
 exports.save = async ctx => {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const webhook = ctx.request.body
   webhook.appId = ctx.appId
 
@@ -52,12 +52,13 @@ exports.save = async ctx => {
 }
 
 exports.destroy = async ctx => {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   ctx.body = await db.remove(ctx.params.id, ctx.params.rev)
 }
 
 exports.buildSchema = async ctx => {
-  const db = new CouchDB(ctx.params.instance)
+  updateAppId(ctx.params.instance)
+  const db = getAppDB()
   const webhook = await db.get(ctx.params.id)
   webhook.bodySchema = toJsonSchema(ctx.request.body)
   // update the automation outputs
@@ -81,9 +82,10 @@ exports.buildSchema = async ctx => {
 }
 
 exports.trigger = async ctx => {
-  const prodAppId = getDeployedAppID(ctx.params.instance)
+  const prodAppId = getProdAppID(ctx.params.instance)
+  updateAppId(prodAppId)
   try {
-    const db = new CouchDB(prodAppId)
+    const db = getAppDB()
     const webhook = await db.get(ctx.params.id)
     // validate against the schema
     if (webhook.bodySchema) {

package/src/api/routes/index.js

@@ -20,11 +20,11 @@ const integrationRoutes = require("./integration")
 const permissionRoutes = require("./permission")
 const datasourceRoutes = require("./datasource")
 const queryRoutes = require("./query")
-const hostingRoutes = require("./hosting")
 const backupRoutes = require("./backup")
 const metadataRoutes = require("./metadata")
 const devRoutes = require("./dev")
 const cloudRoutes = require("./cloud")
+const migrationRoutes = require("./migrations")
 
 exports.mainRoutes = [
   authRoutes,
@@ -46,7 +46,6 @@ exports.mainRoutes = [
   permissionRoutes,
   datasourceRoutes,
   queryRoutes,
-  hostingRoutes,
   backupRoutes,
   metadataRoutes,
   devRoutes,
@@ -55,6 +54,7 @@ exports.mainRoutes = [
   // this could be breaking as koa may recognise other routes as this
   tableRoutes,
   rowRoutes,
+  migrationRoutes,
 ]
 
 exports.staticRoutes = staticRoutes

package/src/api/routes/migrations.js

@@ -0,0 +1,14 @@
+const Router = require("@koa/router")
+const migrationsController = require("../controllers/migrations")
+const router = Router()
+const { internalApi } = require("@budibase/backend-core/auth")
+
+router
+  .post("/api/migrations/run", internalApi, migrationsController.migrate)
+  .get(
+    "/api/migrations/definitions",
+    internalApi,
+    migrationsController.fetchDefinitions
+  )
+
+module.exports = router

package/src/api/routes/static.js

@@ -46,5 +46,10 @@ router
   )
   // TODO: this likely needs to be secured in some way
   .get("/:appId/:path*", controller.serveApp)
+  .post(
+    "/api/attachments/:datasourceId/url",
+    authorized(PermissionTypes.TABLE, PermissionLevels.READ),
+    controller.getSignedUploadURL
+  )
 
 module.exports = router

package/src/api/routes/tests/application.spec.js

@@ -53,8 +53,8 @@ describe("/applications", () => {
 
   describe("fetch", () => {
     it("lists all applications", async () => {
-      await config.createApp(request, "app1")
-      await config.createApp(request, "app2")
+      await config.createApp("app1")
+      await config.createApp("app2")
 
       const res = await request
         .get(`/api/applications?status=${AppStatus.DEV}`)

package/src/api/routes/tests/auth.spec.js

@@ -13,10 +13,9 @@ describe("/authenticate", () => {
 
   describe("fetch self", () => {
     it("should be able to fetch self", async () => {
-      const headers = await config.login()
       const res = await request
         .get(`/api/self`)
-        .set(headers)
+        .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
       expect(res.body._id).toEqual(generateUserMetadataID("us_uuid1"))

package/src/api/routes/tests/automation.spec.js

@@ -145,6 +145,7 @@ describe("/automations", () => {
       let table = await config.createTable()
       automation.definition.trigger.inputs.tableId = table._id
       automation.definition.steps[0].inputs.row.tableId = table._id
+      automation.appId = config.appId
       automation = await config.createAutomation(automation)
       await setup.delay(500)
       const res = await testAutomation(config, automation)

package/src/api/routes/tests/misc.spec.js

@@ -82,7 +82,6 @@ describe("run misc tests", () => {
         dataImport.schema[col] = { type: "string" }
       }
       await tableUtils.handleDataImport(
-        config.getAppId(),
         { userId: "test" },
         table,
         dataImport