@budibase/server 1.0.49 → 1.0.50-alpha.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.49",
+  "version": "1.0.50-alpha.0",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.49",
-    "@budibase/client": "^1.0.49",
-    "@budibase/string-templates": "^1.0.49",
+    "@budibase/backend-core": "^1.0.50-alpha.0",
+    "@budibase/client": "^1.0.50-alpha.0",
+    "@budibase/string-templates": "^1.0.50-alpha.0",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -92,6 +92,8 @@
     "fix-path": "3.0.0",
     "form-data": "^4.0.0",
     "fs-extra": "8.1.0",
+    "google-auth-library": "^7.11.0",
+    "google-spreadsheet": "^3.2.0",
     "jimp": "0.16.1",
     "joi": "17.2.1",
     "js-yaml": "^4.1.0",
@@ -110,7 +112,7 @@
     "mongodb": "3.6.3",
     "mssql": "6.2.3",
     "mysql2": "^2.3.1",
-    "node-fetch": "2.6.0",
+    "node-fetch": "2.6.7",
     "open": "^8.4.0",
     "pg": "8.5.1",
     "pino-pretty": "4.0.0",
@@ -139,6 +141,7 @@
     "@jest/test-sequencer": "^24.8.0",
     "@types/apidoc": "^0.50.0",
     "@types/bull": "^3.15.1",
+    "@types/google-spreadsheet": "^3.1.5",
     "@types/jest": "^26.0.23",
     "@types/koa": "^2.13.3",
     "@types/koa-router": "^7.4.2",
@@ -165,5 +168,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "80ae9a30b112c1abf794bdad73da9257ba80d9d5"
+  "gitHead": "a076d7ccf1deab0d300ae229a1aa3f4ce92aa1ba"
 }

package/scripts/integrations/mssql/data/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/mssql/server
+FROM mcr.microsoft.com/mssql/server:2017-latest
 
 ENV ACCEPT_EULA=Y
 ENV SA_PASSWORD=Passw0rd

package/scripts/integrations/mssql/data/setup.sql

@@ -48,7 +48,20 @@ INSERT tasks
 VALUES
   ('Processing', 1);
 
-INSERT people
-  (name, age)
-VALUES
-  ('Bob', '30');
+INSERT INTO people (name, age)
+VALUES ('Bob', '30'),
+  ('Bert', '10'),
+  ('Jack', '12'),
+  ('Mike', '31'),
+  ('Dave', '44'),
+  ('Jim', '43'),
+  ('Kerry', '32'),
+  ('Julie', '12'),
+  ('Kim', '55'),
+  ('Andy', '33'),
+  ('John', '22'),
+  ('Ruth', '66'),
+  ('Robert', '88'),
+  ('Bobert', '99'),
+  ('Jan', '22'),
+  ('Megan', '11');

package/scripts/integrations/postgres/init.sql

@@ -9,12 +9,16 @@ CREATE TABLE Persons (
 );
 CREATE TABLE Tasks (
     TaskID SERIAL PRIMARY KEY,
-    PersonID INT,
+    ExecutorID INT,
+    QaID INT,
     Completed BOOLEAN,
     TaskName varchar(255),
-    CONSTRAINT fkPersons
-        FOREIGN KEY(PersonID)
-            REFERENCES Persons(PersonID)
+    CONSTRAINT fkexecutor
+        FOREIGN KEY(ExecutorID)
+            REFERENCES Persons(PersonID),
+    CONSTRAINT fkqa
+        FOREIGN KEY(QaID)
+	    REFERENCES Persons(PersonID)
 );
 CREATE TABLE Products (
     ProductID SERIAL PRIMARY KEY,
@@ -32,8 +36,9 @@ CREATE TABLE Products_Tasks (
     PRIMARY KEY (ProductID, TaskID)
 );
 INSERT INTO Persons (FirstName, LastName, Address, City) VALUES ('Mike', 'Hughes', '123 Fake Street', 'Belfast');
-INSERT INTO Tasks (PersonID, TaskName, Completed) VALUES (1, 'assembling', TRUE);
-INSERT INTO Tasks (PersonID, TaskName, Completed) VALUES (1, 'processing', FALSE);
+INSERT INTO Persons (FirstName, LastName, Address, City) Values ('John', 'Smith', '64 Updown Road', 'Dublin');
+INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (1, 2, 'assembling', TRUE);
+INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (2, 1, 'processing', FALSE);
 INSERT INTO Products (ProductName) VALUES ('Computers');
 INSERT INTO Products (ProductName) VALUES ('Laptops');
 INSERT INTO Products (ProductName) VALUES ('Chairs');

package/scripts/likeCypress.ts

@@ -30,4 +30,4 @@ process.env.ALLOW_DEV_AUTOMATIONS = "1"
 const server = require("../src/app")
 process.env.PORT = WORKER_PORT
 const worker = require("../../worker/src/index")
-process.env.PORT = MAIN_PORT
+process.env.PORT = MAIN_PORT

package/src/api/controllers/application.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../../db")
 const env = require("../../environment")
 const packageJson = require("../../../package.json")
 const {
@@ -29,14 +28,11 @@ const { processObject } = require("@budibase/string-templates")
 const {
   getAllApps,
   isDevAppID,
-  getDeployedAppID,
+  getProdAppID,
   Replication,
 } = require("@budibase/backend-core/db")
 const { USERS_TABLE_SCHEMA } = require("../../constants")
-const {
-  getDeployedApps,
-  removeAppFromUserRoles,
-} = require("../../utilities/workerRequests")
+const { removeAppFromUserRoles } = require("../../utilities/workerRequests")
 const { clientLibraryPath, stringToReadStream } = require("../../utilities")
 const { getAllLocks } = require("../../utilities/redis")
 const {
@@ -48,11 +44,17 @@ const { getTenantId, isMultiTenant } = require("@budibase/backend-core/tenancy")
 const { syncGlobalUsers } = require("./user")
 const { app: appCache } = require("@budibase/backend-core/cache")
 const { cleanupAutomations } = require("../../automations/utils")
+const {
+  getAppDB,
+  getProdAppDB,
+  updateAppId,
+} = require("@budibase/backend-core/context")
 
 const URL_REGEX_SLASH = /\/|\\/g
 
 // utility function, need to do away with this
-async function getLayouts(db) {
+async function getLayouts() {
+  const db = getAppDB()
   return (
     await db.allDocs(
       getLayoutParams(null, {
@@ -62,7 +64,8 @@ async function getLayouts(db) {
   ).rows.map(row => row.doc)
 }
 
-async function getScreens(db) {
+async function getScreens() {
+  const db = getAppDB()
   return (
     await db.allDocs(
       getScreenParams(null, {
@@ -78,37 +81,51 @@ function getUserRoleId(ctx) {
     : ctx.user.role._id
 }
 
-async function getAppUrlIfNotInUse(ctx) {
+exports.getAppUrl = ctx => {
+  // construct the url
   let url
   if (ctx.request.body.url) {
+    // if the url is provided, use that
     url = encodeURI(ctx.request.body.url)
   } else if (ctx.request.body.name) {
+    // otherwise use the name
     url = encodeURI(`${ctx.request.body.name}`)
   }
   if (url) {
     url = `/${url.replace(URL_REGEX_SLASH, "")}`.toLowerCase()
   }
-  if (!env.SELF_HOSTED) {
-    return url
+  return url
+}
+
+const checkAppUrl = (ctx, apps, url, currentAppId) => {
+  if (currentAppId) {
+    apps = apps.filter(app => app.appId !== currentAppId)
   }
-  const deployedApps = await getDeployedApps()
-  if (
-    url &&
-    deployedApps[url] != null &&
-    ctx.params != null &&
-    deployedApps[url].appId !== ctx.params.appId
-  ) {
-    ctx.throw(400, "App name/URL is already in use.")
+  if (apps.some(app => app.url === url)) {
+    ctx.throw(400, "App URL is already in use.")
+  }
+}
+
+const checkAppName = (ctx, apps, name, currentAppId) => {
+  // TODO: Replace with Joi
+  if (!name) {
+    ctx.throw(400, "Name is required")
+  }
+  if (currentAppId) {
+    apps = apps.filter(app => app.appId !== currentAppId)
+  }
+  if (apps.some(app => app.name === name)) {
+    ctx.throw(400, "App name is already in use.")
   }
-  return url
 }
 
 async function createInstance(template) {
   const tenantId = isMultiTenant() ? getTenantId() : null
   const baseAppId = generateAppID(tenantId)
   const appId = generateDevAppID(baseAppId)
+  updateAppId(appId)
 
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   await db.put({
     _id: "_design/database",
     // view collation information, read before writing any complex views:
@@ -118,9 +135,9 @@ async function createInstance(template) {
 
   // NOTE: indexes need to be created before any tables/templates
   // add view for linked rows
-  await createLinkView(appId)
-  await createRoutingView(appId)
-  await createAllSearchIndex(appId)
+  await createLinkView()
+  await createRoutingView()
+  await createAllSearchIndex()
 
   // replicate the template data to the instance DB
   // this is currently very hard to test, downloading and importing template files
@@ -146,7 +163,7 @@ async function createInstance(template) {
 exports.fetch = async ctx => {
   const dev = ctx.query && ctx.query.status === AppStatus.DEV
   const all = ctx.query && ctx.query.status === AppStatus.ALL
-  const apps = await getAllApps(CouchDB, { dev, all })
+  const apps = await getAllApps({ dev, all })
 
   // get the locks for all the dev apps
   if (dev || all) {
@@ -169,12 +186,11 @@ exports.fetch = async ctx => {
 }
 
 exports.fetchAppDefinition = async ctx => {
-  const db = new CouchDB(ctx.params.appId)
-  const layouts = await getLayouts(db)
+  const layouts = await getLayouts()
   const userRoleId = getUserRoleId(ctx)
-  const accessController = new AccessController(ctx.params.appId)
+  const accessController = new AccessController()
   const screens = await accessController.checkScreensAccess(
-    await getScreens(db),
+    await getScreens(),
     userRoleId
   )
   ctx.body = {
@@ -185,15 +201,15 @@ exports.fetchAppDefinition = async ctx => {
 }
 
 exports.fetchAppPackage = async ctx => {
-  const db = new CouchDB(ctx.params.appId)
+  const db = getAppDB()
   const application = await db.get(DocumentTypes.APP_METADATA)
-  const layouts = await getLayouts(db)
-  let screens = await getScreens(db)
+  const layouts = await getLayouts()
+  let screens = await getScreens()
 
   // Only filter screens if the user is not a builder
   if (!(ctx.user.builder && ctx.user.builder.global)) {
     const userRoleId = getUserRoleId(ctx)
-    const accessController = new AccessController(ctx.params.appId)
+    const accessController = new AccessController()
     screens = await accessController.checkScreensAccess(screens, userRoleId)
   }
 
@@ -206,6 +222,12 @@ exports.fetchAppPackage = async ctx => {
 }
 
 exports.create = async ctx => {
+  const apps = await getAllApps({ dev: true })
+  const name = ctx.request.body.name
+  checkAppName(ctx, apps, name)
+  const url = exports.getAppUrl(ctx)
+  checkAppUrl(ctx, apps, url)
+
   const { useTemplate, templateKey, templateString } = ctx.request.body
   const instanceConfig = {
     useTemplate,
@@ -218,8 +240,7 @@ exports.create = async ctx => {
   const instance = await createInstance(instanceConfig)
   const appId = instance._id
 
-  const url = await getAppUrlIfNotInUse(ctx)
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   let _rev
   try {
     // if template there will be an existing doc
@@ -235,7 +256,7 @@ exports.create = async ctx => {
     type: "app",
     version: packageJson.version,
     componentLibraries: ["@budibase/standard-components"],
-    name: ctx.request.body.name,
+    name: name,
     url: url,
     template: ctx.request.body.template,
     instance: instance,
@@ -262,15 +283,29 @@ exports.create = async ctx => {
   ctx.body = newApplication
 }
 
+// This endpoint currently operates as a PATCH rather than a PUT
+// Thus name and url fields are handled only if present
 exports.update = async ctx => {
-  const data = await updateAppPackage(ctx, ctx.request.body, ctx.params.appId)
+  const apps = await getAllApps({ dev: true })
+  // validation
+  const name = ctx.request.body.name
+  if (name) {
+    checkAppName(ctx, apps, name, ctx.params.appId)
+  }
+  const url = await exports.getAppUrl(ctx)
+  if (url) {
+    checkAppUrl(ctx, apps, url, ctx.params.appId)
+    ctx.request.body.url = url
+  }
+
+  const data = await updateAppPackage(ctx.request.body, ctx.params.appId)
   ctx.status = 200
   ctx.body = data
 }
 
 exports.updateClient = async ctx => {
   // Get current app version
-  const db = new CouchDB(ctx.params.appId)
+  const db = getAppDB()
   const application = await db.get(DocumentTypes.APP_METADATA)
   const currentVersion = application.version
 
@@ -285,14 +320,14 @@ exports.updateClient = async ctx => {
     version: packageJson.version,
     revertableVersion: currentVersion,
   }
-  const data = await updateAppPackage(ctx, appPackageUpdates, ctx.params.appId)
+  const data = await updateAppPackage(appPackageUpdates, ctx.params.appId)
   ctx.status = 200
   ctx.body = data
 }
 
 exports.revertClient = async ctx => {
   // Check app can be reverted
-  const db = new CouchDB(ctx.params.appId)
+  const db = getAppDB()
   const application = await db.get(DocumentTypes.APP_METADATA)
   if (!application.revertableVersion) {
     ctx.throw(400, "There is no version to revert to")
@@ -308,13 +343,13 @@ exports.revertClient = async ctx => {
     version: application.revertableVersion,
     revertableVersion: null,
   }
-  const data = await updateAppPackage(ctx, appPackageUpdates, ctx.params.appId)
+  const data = await updateAppPackage(appPackageUpdates, ctx.params.appId)
   ctx.status = 200
   ctx.body = data
 }
 
 exports.delete = async ctx => {
-  const db = new CouchDB(ctx.params.appId)
+  const db = getAppDB()
 
   const result = await db.destroy()
   /* istanbul ignore next */
@@ -339,10 +374,11 @@ exports.sync = async (ctx, next) => {
   }
 
   // replicate prod to dev
-  const prodAppId = getDeployedAppID(appId)
+  const prodAppId = getProdAppID(appId)
 
   try {
-    const prodDb = new CouchDB(prodAppId, { skip_setup: true })
+    // specific case, want to make sure setup is skipped
+    const prodDb = getProdAppDB({ skip_setup: true })
     const info = await prodDb.info()
     if (info.error) throw info.error
   } catch (err) {
@@ -370,7 +406,7 @@ exports.sync = async (ctx, next) => {
   }
 
   // sync the users
-  await syncGlobalUsers(appId)
+  await syncGlobalUsers()
 
   if (error) {
     ctx.throw(400, error)
@@ -381,12 +417,11 @@ exports.sync = async (ctx, next) => {
   }
 }
 
-const updateAppPackage = async (ctx, appPackage, appId) => {
-  const url = await getAppUrlIfNotInUse(ctx)
-  const db = new CouchDB(appId)
+const updateAppPackage = async (appPackage, appId) => {
+  const db = getAppDB()
   const application = await db.get(DocumentTypes.APP_METADATA)
 
-  const newAppPackage = { ...application, ...appPackage, url }
+  const newAppPackage = { ...application, ...appPackage }
   if (appPackage._rev !== application._rev) {
     newAppPackage._rev = application._rev
   }
@@ -402,7 +437,7 @@ const updateAppPackage = async (ctx, appPackage, appId) => {
 }
 
 const createEmptyAppPackage = async (ctx, app) => {
-  const db = new CouchDB(app.appId)
+  const db = getAppDB()
 
   let screensAndLayouts = []
   for (let layout of BASE_LAYOUTS) {

package/src/api/controllers/auth.js

@@ -1,11 +1,10 @@
-const CouchDB = require("../../db")
 const { outputProcessing } = require("../../utilities/rowProcessor")
 const { InternalTables } = require("../../db/utils")
 const { getFullUser } = require("../../utilities/users")
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
+const { getAppDB, getAppId } = require("@budibase/backend-core/context")
 
 exports.fetchSelf = async ctx => {
-  const appId = ctx.appId
   let userId = ctx.user.userId || ctx.user._id
   /* istanbul ignore next */
   if (!userId) {
@@ -16,16 +15,20 @@ exports.fetchSelf = async ctx => {
   const user = await getFullUser(ctx, userId)
   // this shouldn't be returned by the app self
   delete user.roles
+  // forward the csrf token from the session
+  user.csrfToken = ctx.user.csrfToken
 
-  if (appId) {
-    const db = new CouchDB(appId)
+  if (getAppId()) {
+    const db = getAppDB()
     // remove the full roles structure
     delete user.roles
     try {
       const userTable = await db.get(InternalTables.USER_METADATA)
       const metadata = await db.get(userId)
+      // make sure there is never a stale csrf token
+      delete metadata.csrfToken
       // specifically needs to make sure is enriched
-      ctx.body = await outputProcessing(ctx, userTable, {
+      ctx.body = await outputProcessing(userTable, {
         ...user,
         ...metadata,
       })

package/src/api/controllers/automation.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../../db")
 const actions = require("../../automations/actions")
 const triggers = require("../../automations/triggers")
 const { getAutomationParams, generateAutomationID } = require("../../db/utils")
@@ -10,6 +9,7 @@ const {
 const { deleteEntityMetadata } = require("../../utilities")
 const { MetadataTypes } = require("../../constants")
 const { setTestFlag, clearTestFlag } = require("../../utilities/redis")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS)
 const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
@@ -20,14 +20,9 @@ const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
  *                       *
  *************************/
 
-async function cleanupAutomationMetadata(appId, automationId) {
+async function cleanupAutomationMetadata(automationId) {
+  await deleteEntityMetadata(MetadataTypes.AUTOMATION_TEST_INPUT, automationId)
   await deleteEntityMetadata(
-    appId,
-    MetadataTypes.AUTOMATION_TEST_INPUT,
-    automationId
-  )
-  await deleteEntityMetadata(
-    appId,
     MetadataTypes.AUTOMATION_TEST_HISTORY,
     automationId
   )
@@ -58,7 +53,7 @@ function cleanAutomationInputs(automation) {
 }
 
 exports.create = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   let automation = ctx.request.body
   automation.appId = ctx.appId
 
@@ -72,7 +67,6 @@ exports.create = async function (ctx) {
   automation.type = "automation"
   automation = cleanAutomationInputs(automation)
   automation = await checkForWebhooks({
-    appId: ctx.appId,
     newAuto: automation,
   })
   const response = await db.put(automation)
@@ -89,13 +83,12 @@ exports.create = async function (ctx) {
 }
 
 exports.update = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   let automation = ctx.request.body
   automation.appId = ctx.appId
   const oldAutomation = await db.get(automation._id)
   automation = cleanAutomationInputs(automation)
   automation = await checkForWebhooks({
-    appId: ctx.appId,
     oldAuto: oldAutomation,
     newAuto: automation,
   })
@@ -131,7 +124,7 @@ exports.update = async function (ctx) {
 }
 
 exports.fetch = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const response = await db.allDocs(
     getAutomationParams(null, {
       include_docs: true,
@@ -141,20 +134,19 @@ exports.fetch = async function (ctx) {
 }
 
 exports.find = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   ctx.body = await db.get(ctx.params.id)
 }
 
 exports.destroy = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const automationId = ctx.params.id
   const oldAutomation = await db.get(automationId)
   await checkForWebhooks({
-    appId: ctx.appId,
     oldAuto: oldAutomation,
   })
   // delete metadata first
-  await cleanupAutomationMetadata(ctx.appId, automationId)
+  await cleanupAutomationMetadata(automationId)
   ctx.body = await db.remove(automationId, ctx.params.rev)
 }
 
@@ -180,12 +172,11 @@ module.exports.getDefinitionList = async function (ctx) {
  *********************/
 
 exports.trigger = async function (ctx) {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   let automation = await db.get(ctx.params.id)
   await triggers.externalTrigger(automation, {
     ...ctx.request.body,
-    appId,
+    appId: ctx.appId,
   })
   ctx.body = {
     message: `Automation ${automation._id} has been triggered.`,
@@ -205,8 +196,7 @@ function prepareTestInput(input) {
 }
 
 exports.test = async function (ctx) {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   let automation = await db.get(ctx.params.id)
   await setTestFlag(automation._id)
   const testInput = prepareTestInput(ctx.request.body)
@@ -214,7 +204,7 @@ exports.test = async function (ctx) {
     automation,
     {
       ...testInput,
-      appId,
+      appId: ctx.appId,
     },
     { getResponses: true }
   )

package/src/api/controllers/cloud.js

@@ -1,6 +1,5 @@
 const env = require("../../environment")
 const { getAllApps } = require("@budibase/backend-core/db")
-const CouchDB = require("../../db")
 const {
   exportDB,
   sendTempFile,
@@ -30,7 +29,7 @@ exports.exportApps = async ctx => {
   if (env.SELF_HOSTED || !env.MULTI_TENANCY) {
     ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.")
   }
-  const apps = await getAllApps(CouchDB, { all: true })
+  const apps = await getAllApps({ all: true })
   const globalDBString = await exportDB(getGlobalDBName(), {
     filter: doc => !doc._id.startsWith(DocumentTypes.USER),
   })
@@ -63,7 +62,7 @@ async function hasBeenImported() {
   if (!env.SELF_HOSTED || env.MULTI_TENANCY) {
     return true
   }
-  const apps = await getAllApps(CouchDB, { all: true })
+  const apps = await getAllApps({ all: true })
   return apps.length !== 0
 }
 

package/src/api/controllers/component.js

@@ -1,15 +1,14 @@
-const CouchDB = require("../../db")
 const { DocumentTypes } = require("../../db/utils")
 const { getComponentLibraryManifest } = require("../../utilities/fileSystem")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 exports.fetchAppComponentDefinitions = async function (ctx) {
-  const appId = ctx.params.appId || ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const app = await db.get(DocumentTypes.APP_METADATA)
 
   let componentManifests = await Promise.all(
     app.componentLibraries.map(async library => {
-      let manifest = await getComponentLibraryManifest(appId, library)
+      let manifest = await getComponentLibraryManifest(library)
 
       return {
         manifest,