@budibase/server 1.0.49-alpha.9 → 1.0.49

package/src/api/controllers/static/index.js

@@ -5,7 +5,8 @@ const { resolve, join } = require("../../../utilities/centralPath")
 const uuid = require("uuid")
 const { ObjectStoreBuckets } = require("../../../constants")
 const { processString } = require("@budibase/string-templates")
-const { getAllApps } = require("@budibase/backend-core/db")
+const { getDeployedApps } = require("../../../utilities/workerRequests")
+const CouchDB = require("../../../db")
 const {
   loadHandlebarsFile,
   NODE_MODULES_PATH,
@@ -16,9 +17,6 @@ const { clientLibraryPath } = require("../../../utilities")
 const { upload } = require("../../../utilities/fileSystem")
 const { attachmentsRelativeURL } = require("../../../utilities")
 const { DocumentTypes } = require("../../../db/utils")
-const { getAppDB } = require("@budibase/backend-core/context")
-const AWS = require("aws-sdk")
-const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1"
 
 async function prepareUpload({ s3Key, bucket, metadata, file }) {
   const response = await upload({
@@ -39,18 +37,12 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) {
   }
 }
 
-async function getAppIdFromUrl(ctx) {
-  // the "appId" component of the URL can be the id or the custom url
+async function checkForSelfHostedURL(ctx) {
+  // the "appId" component of the URL may actually be a specific self hosted URL
   let possibleAppUrl = `/${encodeURI(ctx.params.appId).toLowerCase()}`
-
-  // search prod apps for a url that matches, exclude dev where id is always used
-  const apps = await getAllApps({ dev: false })
-  const app = apps.filter(
-    a => a.url && a.url.toLowerCase() === possibleAppUrl
-  )[0]
-
-  if (app && app.appId) {
-    return app.appId
+  const apps = await getDeployedApps()
+  if (apps[possibleAppUrl] && apps[possibleAppUrl].appId) {
+    return apps[possibleAppUrl].appId
   } else {
     return ctx.params.appId
   }
@@ -83,9 +75,12 @@ exports.uploadFile = async function (ctx) {
 }
 
 exports.serveApp = async function (ctx) {
-  let appId = await getAppIdFromUrl(ctx)
+  let appId = ctx.params.appId
+  if (env.SELF_HOSTED) {
+    appId = await checkForSelfHostedURL(ctx)
+  }
   const App = require("./templates/BudibaseApp.svelte").default
-  const db = getAppDB({ skip_setup: true })
+  const db = new CouchDB(appId, { skip_setup: true })
   const appInfo = await db.get(DocumentTypes.APP_METADATA)
 
   const { head, html, css } = App.render({
@@ -109,51 +104,3 @@ exports.serveClientLibrary = async function (ctx) {
     root: join(NODE_MODULES_PATH, "@budibase", "client", "dist"),
   })
 }
-
-exports.getSignedUploadURL = async function (ctx) {
-  const database = getAppDB()
-
-  // Ensure datasource is valid
-  let datasource
-  try {
-    const { datasourceId } = ctx.params
-    datasource = await database.get(datasourceId)
-    if (!datasource) {
-      ctx.throw(400, "The specified datasource could not be found")
-    }
-  } catch (error) {
-    ctx.throw(400, "The specified datasource could not be found")
-  }
-
-  // Ensure we aren't using a custom endpoint
-  if (datasource?.config?.endpoint) {
-    ctx.throw(400, "S3 datasources with custom endpoints are not supported")
-  }
-
-  // Determine type of datasource and generate signed URL
-  let signedUrl
-  let publicUrl
-  if (datasource.source === "S3") {
-    const { bucket, key } = ctx.request.body || {}
-    if (!bucket || !key) {
-      ctx.throw(400, "bucket and key values are required")
-      return
-    }
-    try {
-      const s3 = new AWS.S3({
-        region: AWS_REGION,
-        accessKeyId: datasource?.config?.accessKeyId,
-        secretAccessKey: datasource?.config?.secretAccessKey,
-        apiVersion: "2006-03-01",
-        signatureVersion: "v4",
-      })
-      const params = { Bucket: bucket, Key: key }
-      signedUrl = s3.getSignedUrl("putObject", params)
-      publicUrl = `https://${bucket}.s3.${AWS_REGION}.amazonaws.com/${key}`
-    } catch (error) {
-      ctx.throw(400, error)
-    }
-  }
-
-  ctx.body = { signedUrl, publicUrl }
-}

package/src/api/controllers/table/external.js

@@ -1,3 +1,4 @@
+const CouchDB = require("../../../db")
 const {
   buildExternalTableId,
   breakExternalTableId,
@@ -18,7 +19,6 @@ const { makeExternalQuery } = require("../../../integrations/base/utils")
 const { cloneDeep } = require("lodash/fp")
 const csvParser = require("../../../utilities/csvParser")
 const { handleRequest } = require("../row/external")
-const { getAppDB } = require("@budibase/backend-core/context")
 
 async function makeTableRequest(
   datasource,
@@ -159,6 +159,7 @@ function isRelationshipSetup(column) {
 }
 
 exports.save = async function (ctx) {
+  const appId = ctx.appId
   const table = ctx.request.body
   // can't do this right now
   delete table.dataImport
@@ -175,14 +176,14 @@ exports.save = async function (ctx) {
 
   let oldTable
   if (ctx.request.body && ctx.request.body._id) {
-    oldTable = await getTable(ctx.request.body._id)
+    oldTable = await getTable(appId, ctx.request.body._id)
   }
 
   if (hasTypeChanged(tableToSave, oldTable)) {
     ctx.throw(400, "A column type has changed.")
   }
 
-  const db = getAppDB()
+  const db = new CouchDB(appId)
   const datasource = await db.get(datasourceId)
   const oldTables = cloneDeep(datasource.entities)
   const tables = datasource.entities
@@ -266,13 +267,14 @@ exports.save = async function (ctx) {
 }
 
 exports.destroy = async function (ctx) {
-  const tableToDelete = await getTable(ctx.params.tableId)
+  const appId = ctx.appId
+  const tableToDelete = await getTable(appId, ctx.params.tableId)
   if (!tableToDelete || !tableToDelete.created) {
     ctx.throw(400, "Cannot delete tables which weren't created in Budibase.")
   }
   const datasourceId = getDatasourceId(tableToDelete)
 
-  const db = getAppDB()
+  const db = new CouchDB(appId)
   const datasource = await db.get(datasourceId)
   const tables = datasource.entities
 
@@ -288,7 +290,8 @@ exports.destroy = async function (ctx) {
 }
 
 exports.bulkImport = async function (ctx) {
-  const table = await getTable(ctx.params.tableId)
+  const appId = ctx.appId
+  const table = await getTable(appId, ctx.params.tableId)
   const { dataImport } = ctx.request.body
   if (!dataImport || !dataImport.schema || !dataImport.csvString) {
     ctx.throw(400, "Provided data import information is invalid.")
@@ -297,7 +300,7 @@ exports.bulkImport = async function (ctx) {
     ...dataImport,
     existingTable: table,
   })
-  await handleRequest(DataSourceOperation.BULK_CREATE, table._id, {
+  await handleRequest(appId, DataSourceOperation.BULK_CREATE, table._id, {
     rows,
   })
   return table

package/src/api/controllers/table/index.js

@@ -1,10 +1,14 @@
+const CouchDB = require("../../../db")
 const internal = require("./internal")
 const external = require("./external")
 const csvParser = require("../../../utilities/csvParser")
-const { isExternalTable, isSQL } = require("../../../integrations/utils")
-const { getDatasourceParams } = require("../../../db/utils")
-const { getAppDB } = require("@budibase/backend-core/context")
-const { getTable, getAllInternalTables } = require("./utils")
+const { isExternalTable } = require("../../../integrations/utils")
+const {
+  getTableParams,
+  getDatasourceParams,
+  BudibaseInternalDB,
+} = require("../../../db/utils")
+const { getTable } = require("./utils")
 
 function pickApi({ tableId, table }) {
   if (table && !tableId) {
@@ -20,9 +24,19 @@ function pickApi({ tableId, table }) {
 
 // covers both internal and external
 exports.fetch = async function (ctx) {
-  const db = getAppDB()
+  const db = new CouchDB(ctx.appId)
 
-  const internal = await getAllInternalTables()
+  const internalTables = await db.allDocs(
+    getTableParams(null, {
+      include_docs: true,
+    })
+  )
+
+  const internal = internalTables.rows.map(row => ({
+    ...row.doc,
+    type: "internal",
+    sourceId: BudibaseInternalDB._id,
+  }))
 
   const externalTables = await db.allDocs(
     getDatasourceParams("plus", {
@@ -30,18 +44,12 @@ exports.fetch = async function (ctx) {
     })
   )
 
-  const external = externalTables.rows.flatMap(tableDoc => {
-    let entities = tableDoc.doc.entities
-    if (entities) {
-      return Object.values(entities).map(entity => ({
-        ...entity,
-        type: "external",
-        sourceId: tableDoc.doc._id,
-        sql: isSQL(tableDoc.doc),
-      }))
-    } else {
-      return []
-    }
+  const external = externalTables.rows.flatMap(row => {
+    return Object.values(row.doc.entities || {}).map(entity => ({
+      ...entity,
+      type: "external",
+      sourceId: row.doc._id,
+    }))
   })
 
   ctx.body = [...internal, ...external]
@@ -49,7 +57,7 @@ exports.fetch = async function (ctx) {
 
 exports.find = async function (ctx) {
   const tableId = ctx.params.id
-  ctx.body = await getTable(tableId)
+  ctx.body = await getTable(ctx.appId, tableId)
 }
 
 exports.save = async function (ctx) {
@@ -88,7 +96,7 @@ exports.validateCSVSchema = async function (ctx) {
   const { csvString, schema = {}, tableId } = ctx.request.body
   let existingTable
   if (tableId) {
-    existingTable = await getTable(tableId)
+    existingTable = await getTable(ctx.appId, tableId)
   }
   let result = await csvParser.parse(csvString, schema)
   if (existingTable) {

package/src/api/controllers/table/internal.js

@@ -1,3 +1,4 @@
+const CouchDB = require("../../../db")
 const linkRows = require("../../../db/linkedRows")
 const { getRowParams, generateTableID } = require("../../../db/utils")
 const { FieldTypes } = require("../../../constants")
@@ -7,14 +8,10 @@ const {
   getTable,
   handleDataImport,
 } = require("./utils")
-const usageQuota = require("../../../utilities/usageQuota")
-const { getAppDB } = require("@budibase/backend-core/context")
-const env = require("../../../environment")
-const { cleanupAttachments } = require("../../../utilities/rowProcessor")
-const { runStaticFormulaChecks } = require("./bulkFormula")
 
 exports.save = async function (ctx) {
-  const db = getAppDB()
+  const appId = ctx.appId
+  const db = new CouchDB(appId)
   const { dataImport, ...rest } = ctx.request.body
   let tableToSave = {
     type: "table",
@@ -36,7 +33,8 @@ exports.save = async function (ctx) {
   // saving a table is a complex operation, involving many different steps, this
   // has been broken out into a utility to make it more obvious/easier to manipulate
   const tableSaveFunctions = new TableSaveFunctions({
-    user: ctx.user,
+    db,
+    ctx,
     oldTable,
     dataImport,
   })
@@ -81,6 +79,7 @@ exports.save = async function (ctx) {
   // update linked rows
   try {
     const linkResp = await linkRows.updateLinks({
+      appId,
       eventType: oldTable
         ? linkRows.EventType.TABLE_UPDATED
         : linkRows.EventType.TABLE_SAVE,
@@ -104,13 +103,13 @@ exports.save = async function (ctx) {
   tableToSave._rev = result.rev
 
   tableToSave = await tableSaveFunctions.after(tableToSave)
-  // has to run after, make sure it has _id
-  await runStaticFormulaChecks(tableToSave, { oldTable })
+
   return tableToSave
 }
 
 exports.destroy = async function (ctx) {
-  const db = getAppDB()
+  const appId = ctx.appId
+  const db = new CouchDB(appId)
   const tableToDelete = await db.get(ctx.params.tableId)
 
   // Delete all rows for that table
@@ -120,10 +119,10 @@ exports.destroy = async function (ctx) {
     })
   )
   await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
-  await usageQuota.update(usageQuota.Properties.ROW, -rows.rows.length)
 
   // update linked rows
   await linkRows.updateLinks({
+    appId,
     eventType: linkRows.EventType.TABLE_DELETE,
     table: tableToDelete,
   })
@@ -132,25 +131,21 @@ exports.destroy = async function (ctx) {
   await db.remove(tableToDelete)
 
   // remove table search index
-  if (!env.isTest()) {
-    const currentIndexes = await db.getIndexes()
-    const existingIndex = currentIndexes.indexes.find(
-      existing => existing.name === `search:${ctx.params.tableId}`
-    )
-    if (existingIndex) {
-      await db.deleteIndex(existingIndex)
-    }
+  const currentIndexes = await db.getIndexes()
+  const existingIndex = currentIndexes.indexes.find(
+    existing => existing.name === `search:${ctx.params.tableId}`
+  )
+  if (existingIndex) {
+    await db.deleteIndex(existingIndex)
   }
 
-  // has to run after, make sure it has _id
-  await runStaticFormulaChecks(tableToDelete, { deletion: true })
-  await cleanupAttachments(tableToDelete, { rows })
   return tableToDelete
 }
 
 exports.bulkImport = async function (ctx) {
-  const table = await getTable(ctx.params.tableId)
+  const appId = ctx.appId
+  const table = await getTable(appId, ctx.params.tableId)
   const { dataImport } = ctx.request.body
-  await handleDataImport(ctx.user, table, dataImport)
+  await handleDataImport(appId, ctx.user, table, dataImport)
   return table
 }

package/src/api/controllers/table/utils.js

@@ -1,17 +1,13 @@
+const CouchDB = require("../../../db")
 const csvParser = require("../../../utilities/csvParser")
 const {
   getRowParams,
   generateRowID,
   InternalTables,
-  getTableParams,
-  BudibaseInternalDB,
 } = require("../../../db/utils")
-const { isEqual } = require("lodash")
+const { isEqual } = require("lodash/fp")
 const { AutoFieldSubTypes, FieldTypes } = require("../../../constants")
-const {
-  inputProcessing,
-  cleanupAttachments,
-} = require("../../../utilities/rowProcessor")
+const { inputProcessing } = require("../../../utilities/rowProcessor")
 const {
   USERS_TABLE_SCHEMA,
   SwitchableTypes,
@@ -20,31 +16,11 @@ const {
 const {
   isExternalTable,
   breakExternalTableId,
-  isSQL,
 } = require("../../../integrations/utils")
 const { getViews, saveView } = require("../view/utils")
 const viewTemplate = require("../view/viewBuilder")
-const usageQuota = require("../../../utilities/usageQuota")
-const { getAppDB } = require("@budibase/backend-core/context")
-const { cloneDeep } = require("lodash/fp")
-
-exports.clearColumns = async (table, columnNames) => {
-  const db = getAppDB()
-  const rows = await db.allDocs(
-    getRowParams(table._id, null, {
-      include_docs: true,
-    })
-  )
-  return db.bulkDocs(
-    rows.rows.map(({ doc }) => {
-      columnNames.forEach(colName => delete doc[colName])
-      return doc
-    })
-  )
-}
 
-exports.checkForColumnUpdates = async (oldTable, updatedTable) => {
-  const db = getAppDB()
+exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
   let updatedRows = []
   const rename = updatedTable._rename
   let deletedColumns = []
@@ -61,22 +37,18 @@ exports.checkForColumnUpdates = async (oldTable, updatedTable) => {
         include_docs: true,
       })
     )
-    const rawRows = rows.rows.map(({ doc }) => doc)
-    updatedRows = rawRows.map(row => {
-      row = cloneDeep(row)
+    updatedRows = rows.rows.map(({ doc }) => {
       if (rename) {
-        row[rename.updated] = row[rename.old]
-        delete row[rename.old]
+        doc[rename.updated] = doc[rename.old]
+        delete doc[rename.old]
       } else if (deletedColumns.length !== 0) {
-        deletedColumns.forEach(colName => delete row[colName])
+        deletedColumns.forEach(colName => delete doc[colName])
       }
-      return row
+      return doc
     })
 
-    // cleanup any attachments from object storage for deleted attachment columns
-    await cleanupAttachments(updatedTable, { oldTable, rows: rawRows })
     // Update views
-    await exports.checkForViewUpdates(updatedTable, rename, deletedColumns)
+    await exports.checkForViewUpdates(db, updatedTable, rename, deletedColumns)
     delete updatedTable._rename
   }
   return { rows: updatedRows, table: updatedTable }
@@ -103,12 +75,12 @@ exports.makeSureTableUpToDate = (table, tableToSave) => {
   return tableToSave
 }
 
-exports.handleDataImport = async (user, table, dataImport) => {
+exports.handleDataImport = async (appId, user, table, dataImport) => {
   if (!dataImport || !dataImport.csvString) {
     return table
   }
 
-  const db = getAppDB()
+  const db = new CouchDB(appId)
   // Populate the table with rows imported from CSV in a bulk update
   const data = await csvParser.transform({
     ...dataImport,
@@ -143,18 +115,14 @@ exports.handleDataImport = async (user, table, dataImport) => {
     finalData.push(row)
   }
 
-  await usageQuota.update(usageQuota.Properties.ROW, finalData.length, {
-    dryRun: true,
-  })
   await db.bulkDocs(finalData)
-  await usageQuota.update(usageQuota.Properties.ROW, finalData.length)
   let response = await db.put(table)
   table._rev = response._rev
   return table
 }
 
-exports.handleSearchIndexes = async table => {
-  const db = getAppDB()
+exports.handleSearchIndexes = async (appId, table) => {
+  const db = new CouchDB(appId)
   // create relevant search indexes
   if (table.indexes && table.indexes.length > 0) {
     const currentIndexes = await db.getIndexes()
@@ -211,9 +179,12 @@ exports.checkStaticTables = table => {
 }
 
 class TableSaveFunctions {
-  constructor({ user, oldTable, dataImport }) {
-    this.db = getAppDB()
-    this.user = user
+  constructor({ db, ctx, oldTable, dataImport }) {
+    this.db = db
+    this.ctx = ctx
+    if (this.ctx && this.ctx.user) {
+      this.appId = this.ctx.appId
+    }
     this.oldTable = oldTable
     this.dataImport = dataImport
     // any rows that need updated
@@ -231,15 +202,24 @@ class TableSaveFunctions {
 
   // when confirmed valid
   async mid(table) {
-    let response = await exports.checkForColumnUpdates(this.oldTable, table)
+    let response = await exports.checkForColumnUpdates(
+      this.db,
+      this.oldTable,
+      table
+    )
     this.rows = this.rows.concat(response.rows)
     return table
   }
 
   // after saving
   async after(table) {
-    table = await exports.handleSearchIndexes(table)
-    table = await exports.handleDataImport(this.user, table, this.dataImport)
+    table = await exports.handleSearchIndexes(this.appId, table)
+    table = await exports.handleDataImport(
+      this.appId,
+      this.ctx.user,
+      table,
+      this.dataImport
+    )
     return table
   }
 
@@ -248,22 +228,8 @@ class TableSaveFunctions {
   }
 }
 
-exports.getAllInternalTables = async () => {
-  const db = getAppDB()
-  const internalTables = await db.allDocs(
-    getTableParams(null, {
-      include_docs: true,
-    })
-  )
-  return internalTables.rows.map(tableDoc => ({
-    ...tableDoc.doc,
-    type: "internal",
-    sourceId: BudibaseInternalDB._id,
-  }))
-}
-
-exports.getAllExternalTables = async datasourceId => {
-  const db = getAppDB()
+exports.getAllExternalTables = async (appId, datasourceId) => {
+  const db = new CouchDB(appId)
   const datasource = await db.get(datasourceId)
   if (!datasource || !datasource.entities) {
     throw "Datasource is not configured fully."
@@ -271,25 +237,23 @@ exports.getAllExternalTables = async datasourceId => {
   return datasource.entities
 }
 
-exports.getExternalTable = async (datasourceId, tableName) => {
-  const entities = await exports.getAllExternalTables(datasourceId)
+exports.getExternalTable = async (appId, datasourceId, tableName) => {
+  const entities = await exports.getAllExternalTables(appId, datasourceId)
   return entities[tableName]
 }
 
-exports.getTable = async tableId => {
-  const db = getAppDB()
+exports.getTable = async (appId, tableId) => {
+  const db = new CouchDB(appId)
   if (isExternalTable(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
-    const datasource = await db.get(datasourceId)
-    const table = await exports.getExternalTable(datasourceId, tableName)
-    return { ...table, sql: isSQL(datasource) }
+    return exports.getExternalTable(appId, datasourceId, tableName)
   } else {
     return db.get(tableId)
   }
 }
 
-exports.checkForViewUpdates = async (table, rename, deletedColumns) => {
-  const views = await getViews()
+exports.checkForViewUpdates = async (db, table, rename, deletedColumns) => {
+  const views = await getViews(db)
   const tableViews = views.filter(view => view.meta.tableId === table._id)
 
   // Check each table view to see if impacted by this table action
@@ -351,7 +315,7 @@ exports.checkForViewUpdates = async (table, rename, deletedColumns) => {
     // Update view if required
     if (needsUpdated) {
       const newViewTemplate = viewTemplate(view.meta)
-      await saveView(null, view.name, newViewTemplate)
+      await saveView(db, null, view.name, newViewTemplate)
       if (!newViewTemplate.meta.schema) {
         newViewTemplate.meta.schema = table.schema
       }