@budibase/server 1.0.46-alpha.4 → 1.0.46-alpha.8

package/dist/utilities/workerRequests.js

@@ -61,26 +61,6 @@ exports.sendSmtpEmail = (to, from, subject, contents, automation) => __awaiter(v
     }
     return response.json();
 });
-exports.getDeployedApps = () => __awaiter(void 0, void 0, void 0, function* () {
-    try {
-        const response = yield fetch(checkSlashesInUrl(env.WORKER_URL + `/api/apps`), request(null, {
-            method: "GET",
-        }));
-        const json = yield response.json();
-        const apps = {};
-        for (let [key, value] of Object.entries(json)) {
-            if (value.url) {
-                value.url = value.url.toLowerCase();
-                apps[key.toLowerCase()] = value;
-            }
-        }
-        return apps;
-    }
-    catch (err) {
-        // error, cannot determine deployed apps, don't stop app creation - sort this later
-        return {};
-    }
-});
 exports.getGlobalSelf = (ctx, appId = null) => __awaiter(void 0, void 0, void 0, function* () {
     const endpoint = `/api/global/users/self`;
     const response = yield fetch(checkSlashesInUrl(env.WORKER_URL + endpoint), 

package/nodemon.json

@@ -1,5 +1,5 @@
 {
-  "watch": ["src", "../auth"],
+  "watch": ["src", "../backend-core"],
   "ext": "js,ts,json",
   "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js"],
   "exec": "ts-node src/index.ts"

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.46-alpha.4",
+  "version": "1.0.46-alpha.8",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.46-alpha.4",
-    "@budibase/client": "^1.0.46-alpha.4",
-    "@budibase/string-templates": "^1.0.46-alpha.4",
+    "@budibase/backend-core": "^1.0.46-alpha.8",
+    "@budibase/client": "^1.0.46-alpha.8",
+    "@budibase/string-templates": "^1.0.46-alpha.8",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -92,6 +92,8 @@
     "fix-path": "3.0.0",
     "form-data": "^4.0.0",
     "fs-extra": "8.1.0",
+    "google-auth-library": "^7.11.0",
+    "google-spreadsheet": "^3.2.0",
     "jimp": "0.16.1",
     "joi": "17.2.1",
     "js-yaml": "^4.1.0",
@@ -110,7 +112,7 @@
     "mongodb": "3.6.3",
     "mssql": "6.2.3",
     "mysql2": "^2.3.1",
-    "node-fetch": "2.6.0",
+    "node-fetch": "2.6.7",
     "open": "^8.4.0",
     "pg": "8.5.1",
     "pino-pretty": "4.0.0",
@@ -139,6 +141,7 @@
     "@jest/test-sequencer": "^24.8.0",
     "@types/apidoc": "^0.50.0",
     "@types/bull": "^3.15.1",
+    "@types/google-spreadsheet": "^3.1.5",
     "@types/jest": "^26.0.23",
     "@types/koa": "^2.13.3",
     "@types/koa-router": "^7.4.2",
@@ -165,5 +168,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "965525d1b1437e7c94eaff0b21f2143e3f2f0ad7"
+  "gitHead": "d69c417cf4407896917419c6f2dbb3de4ea340d2"
 }

package/src/api/controllers/application.js

@@ -33,10 +33,7 @@ const {
   Replication,
 } = require("@budibase/backend-core/db")
 const { USERS_TABLE_SCHEMA } = require("../../constants")
-const {
-  getDeployedApps,
-  removeAppFromUserRoles,
-} = require("../../utilities/workerRequests")
+const { removeAppFromUserRoles } = require("../../utilities/workerRequests")
 const { clientLibraryPath, stringToReadStream } = require("../../utilities")
 const { getAllLocks } = require("../../utilities/redis")
 const {
@@ -78,29 +75,42 @@ function getUserRoleId(ctx) {
     : ctx.user.role._id
 }
 
-async function getAppUrlIfNotInUse(ctx) {
+async function getAppUrl(ctx) {
+  // construct the url
   let url
   if (ctx.request.body.url) {
+    // if the url is provided, use that
     url = encodeURI(ctx.request.body.url)
   } else if (ctx.request.body.name) {
+    // otherwise use the name
     url = encodeURI(`${ctx.request.body.name}`)
   }
   if (url) {
     url = `/${url.replace(URL_REGEX_SLASH, "")}`.toLowerCase()
   }
-  if (!env.SELF_HOSTED) {
-    return url
+  return url
+}
+
+const checkAppUrl = (ctx, apps, url, currentAppId) => {
+  if (currentAppId) {
+    apps = apps.filter(app => app.appId !== currentAppId)
   }
-  const deployedApps = await getDeployedApps()
-  if (
-    url &&
-    deployedApps[url] != null &&
-    ctx.params != null &&
-    deployedApps[url].appId !== ctx.params.appId
-  ) {
-    ctx.throw(400, "App name/URL is already in use.")
+  if (apps.some(app => app.url === url)) {
+    ctx.throw(400, "App URL is already in use.")
+  }
+}
+
+const checkAppName = (ctx, apps, name, currentAppId) => {
+  // TODO: Replace with Joi
+  if (!name) {
+    ctx.throw(400, "Name is required")
+  }
+  if (currentAppId) {
+    apps = apps.filter(app => app.appId !== currentAppId)
+  }
+  if (apps.some(app => app.name === name)) {
+    ctx.throw(400, "App name is already in use.")
   }
-  return url
 }
 
 async function createInstance(template) {
@@ -206,6 +216,12 @@ exports.fetchAppPackage = async ctx => {
 }
 
 exports.create = async ctx => {
+  const apps = await getAllApps(CouchDB, { dev: true })
+  const name = ctx.request.body.name
+  checkAppName(ctx, apps, name)
+  const url = await getAppUrl(ctx)
+  checkAppUrl(ctx, apps, url)
+
   const { useTemplate, templateKey, templateString } = ctx.request.body
   const instanceConfig = {
     useTemplate,
@@ -218,7 +234,6 @@ exports.create = async ctx => {
   const instance = await createInstance(instanceConfig)
   const appId = instance._id
 
-  const url = await getAppUrlIfNotInUse(ctx)
   const db = new CouchDB(appId)
   let _rev
   try {
@@ -235,7 +250,7 @@ exports.create = async ctx => {
     type: "app",
     version: packageJson.version,
     componentLibraries: ["@budibase/standard-components"],
-    name: ctx.request.body.name,
+    name: name,
     url: url,
     template: ctx.request.body.template,
     instance: instance,
@@ -262,8 +277,22 @@ exports.create = async ctx => {
   ctx.body = newApplication
 }
 
+// This endpoint currently operates as a PATCH rather than a PUT
+// Thus name and url fields are handled only if present
 exports.update = async ctx => {
-  const data = await updateAppPackage(ctx, ctx.request.body, ctx.params.appId)
+  const apps = await getAllApps(CouchDB, { dev: true })
+  // validation
+  const name = ctx.request.body.name
+  if (name) {
+    checkAppName(ctx, apps, name, ctx.params.appId)
+  }
+  const url = await getAppUrl(ctx)
+  if (url) {
+    checkAppUrl(ctx, apps, url, ctx.params.appId)
+    ctx.request.body.url = url
+  }
+
+  const data = await updateAppPackage(ctx.request.body, ctx.params.appId)
   ctx.status = 200
   ctx.body = data
 }
@@ -285,7 +314,7 @@ exports.updateClient = async ctx => {
     version: packageJson.version,
     revertableVersion: currentVersion,
   }
-  const data = await updateAppPackage(ctx, appPackageUpdates, ctx.params.appId)
+  const data = await updateAppPackage(appPackageUpdates, ctx.params.appId)
   ctx.status = 200
   ctx.body = data
 }
@@ -308,7 +337,7 @@ exports.revertClient = async ctx => {
     version: application.revertableVersion,
     revertableVersion: null,
   }
-  const data = await updateAppPackage(ctx, appPackageUpdates, ctx.params.appId)
+  const data = await updateAppPackage(appPackageUpdates, ctx.params.appId)
   ctx.status = 200
   ctx.body = data
 }
@@ -381,12 +410,11 @@ exports.sync = async (ctx, next) => {
   }
 }
 
-const updateAppPackage = async (ctx, appPackage, appId) => {
-  const url = await getAppUrlIfNotInUse(ctx)
+const updateAppPackage = async (appPackage, appId) => {
   const db = new CouchDB(appId)
   const application = await db.get(DocumentTypes.APP_METADATA)
 
-  const newAppPackage = { ...application, ...appPackage, url }
+  const newAppPackage = { ...application, ...appPackage }
   if (appPackage._rev !== application._rev) {
     newAppPackage._rev = application._rev
   }

package/src/api/controllers/auth.js

@@ -16,6 +16,8 @@ exports.fetchSelf = async ctx => {
   const user = await getFullUser(ctx, userId)
   // this shouldn't be returned by the app self
   delete user.roles
+  // forward the csrf token from the session
+  user.csrfToken = ctx.user.csrfToken
 
   if (appId) {
     const db = new CouchDB(appId)
@@ -24,6 +26,8 @@ exports.fetchSelf = async ctx => {
     try {
       const userTable = await db.get(InternalTables.USER_METADATA)
       const metadata = await db.get(userId)
+      // make sure there is never a stale csrf token
+      delete metadata.csrfToken
       // specifically needs to make sure is enriched
       ctx.body = await outputProcessing(ctx, userTable, {
         ...user,

package/src/api/controllers/datasource.js

@@ -38,6 +38,13 @@ exports.fetch = async function (ctx) {
     )
   ).rows.map(row => row.doc)
 
+  for (let datasource of datasources) {
+    if (datasource.config && datasource.config.auth) {
+      // strip secrets from response so they don't show in the network request
+      delete datasource.config.auth
+    }
+  }
+
   ctx.body = [bbInternalDb, ...datasources]
 }
 
@@ -94,8 +101,13 @@ exports.update = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   const datasourceId = ctx.params.datasourceId
   let datasource = await db.get(datasourceId)
+  const auth = datasource.config.auth
   await invalidateVariables(datasource, ctx.request.body)
   datasource = { ...datasource, ...ctx.request.body }
+  if (auth && !ctx.request.body.auth) {
+    // don't strip auth config from DB
+    datasource.config.auth = auth
+  }
 
   const response = await db.put(datasource)
   datasource._rev = response.rev

package/src/api/controllers/query/index.js

@@ -141,6 +141,16 @@ async function execute(ctx, opts = { rowsOnly: false }) {
   const query = await db.get(ctx.params.queryId)
   const datasource = await db.get(query.datasourceId)
 
+  const enrichedParameters = ctx.request.body.parameters || {}
+  // make sure parameters are fully enriched with defaults
+  if (query && query.parameters) {
+    for (let parameter of query.parameters) {
+      if (!enrichedParameters[parameter.name]) {
+        enrichedParameters[parameter.name] = parameter.default
+      }
+    }
+  }
+
   // call the relevant CRUD method on the integration class
   try {
     const { rows, pagination, extra } = await Runner.run({
@@ -149,7 +159,7 @@ async function execute(ctx, opts = { rowsOnly: false }) {
       queryVerb: query.queryVerb,
       fields: query.fields,
       pagination: ctx.request.body.pagination,
-      parameters: ctx.request.body.parameters,
+      parameters: enrichedParameters,
       transformer: query.transformer,
       queryId: ctx.params.queryId,
     })
@@ -178,8 +188,9 @@ const removeDynamicVariables = async (db, queryId) => {
 
   if (dynamicVariables) {
     // delete dynamic variables from the datasource
-    const newVariables = dynamicVariables.filter(dv => dv.queryId !== queryId)
-    datasource.config.dynamicVariables = newVariables
+    datasource.config.dynamicVariables = dynamicVariables.filter(
+      dv => dv.queryId !== queryId
+    )
     await db.put(datasource)
 
     // invalidate the deleted variables

package/src/api/controllers/row/utils.js

@@ -52,10 +52,7 @@ exports.validate = async ({ appId, tableId, row, table }) => {
     const constraints = cloneDeep(table.schema[fieldName].constraints)
     const type = table.schema[fieldName].type
     // special case for options, need to always allow unselected (null)
-    if (
-      (type === FieldTypes.OPTIONS || type === FieldTypes.ARRAY) &&
-      constraints.inclusion
-    ) {
+    if (type === FieldTypes.OPTIONS && constraints.inclusion) {
       constraints.inclusion.push(null)
     }
     let res

package/src/api/controllers/static/index.js

@@ -5,7 +5,7 @@ const { resolve, join } = require("../../../utilities/centralPath")
 const uuid = require("uuid")
 const { ObjectStoreBuckets } = require("../../../constants")
 const { processString } = require("@budibase/string-templates")
-const { getDeployedApps } = require("../../../utilities/workerRequests")
+const { getAllApps } = require("@budibase/backend-core/db")
 const CouchDB = require("../../../db")
 const {
   loadHandlebarsFile,
@@ -39,12 +39,18 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) {
   }
 }
 
-async function checkForSelfHostedURL(ctx) {
-  // the "appId" component of the URL may actually be a specific self hosted URL
+async function getAppIdFromUrl(ctx) {
+  // the "appId" component of the URL can be the id or the custom url
   let possibleAppUrl = `/${encodeURI(ctx.params.appId).toLowerCase()}`
-  const apps = await getDeployedApps()
-  if (apps[possibleAppUrl] && apps[possibleAppUrl].appId) {
-    return apps[possibleAppUrl].appId
+
+  // search prod apps for a url that matches, exclude dev where id is always used
+  const apps = await getAllApps(CouchDB, { dev: false })
+  const app = apps.filter(
+    a => a.url && a.url.toLowerCase() === possibleAppUrl
+  )[0]
+
+  if (app && app.appId) {
+    return app.appId
   } else {
     return ctx.params.appId
   }
@@ -77,10 +83,7 @@ exports.uploadFile = async function (ctx) {
 }
 
 exports.serveApp = async function (ctx) {
-  let appId = ctx.params.appId
-  if (env.SELF_HOSTED) {
-    appId = await checkForSelfHostedURL(ctx)
-  }
+  let appId = await getAppIdFromUrl(ctx)
   const App = require("./templates/BudibaseApp.svelte").default
   const db = new CouchDB(appId, { skip_setup: true })
   const appInfo = await db.get(DocumentTypes.APP_METADATA)

package/src/api/controllers/user.js

@@ -167,6 +167,8 @@ exports.updateSelfMetadata = async function (ctx) {
   ctx.request.body._id = ctx.user._id
   // make sure no stale rev
   delete ctx.request.body._rev
+  // make sure no csrf token
+  delete ctx.request.body.csrfToken
   await exports.updateMetadata(ctx)
 }
 

package/src/api/routes/index.js

@@ -20,7 +20,6 @@ const integrationRoutes = require("./integration")
 const permissionRoutes = require("./permission")
 const datasourceRoutes = require("./datasource")
 const queryRoutes = require("./query")
-const hostingRoutes = require("./hosting")
 const backupRoutes = require("./backup")
 const metadataRoutes = require("./metadata")
 const devRoutes = require("./dev")
@@ -46,7 +45,6 @@ exports.mainRoutes = [
   permissionRoutes,
   datasourceRoutes,
   queryRoutes,
-  hostingRoutes,
   backupRoutes,
   metadataRoutes,
   devRoutes,

package/src/api/routes/tests/application.spec.js

@@ -53,8 +53,8 @@ describe("/applications", () => {
 
   describe("fetch", () => {
     it("lists all applications", async () => {
-      await config.createApp(request, "app1")
-      await config.createApp(request, "app2")
+      await config.createApp("app1")
+      await config.createApp("app2")
 
       const res = await request
         .get(`/api/applications?status=${AppStatus.DEV}`)

package/src/api/routes/tests/auth.spec.js

@@ -13,10 +13,9 @@ describe("/authenticate", () => {
 
   describe("fetch self", () => {
     it("should be able to fetch self", async () => {
-      const headers = await config.login()
       const res = await request
         .get(`/api/self`)
-        .set(headers)
+        .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
       expect(res.body._id).toEqual(generateUserMetadataID("us_uuid1"))

package/src/constants/index.js

@@ -75,6 +75,10 @@ exports.DataSourceOperation = {
   DELETE_TABLE: "DELETE_TABLE",
 }
 
+exports.DatasourceAuthTypes = {
+  GOOGLE: "google",
+}
+
 exports.SortDirection = {
   ASCENDING: "ASCENDING",
   DESCENDING: "DESCENDING",

package/src/definitions/datasource.ts

@@ -47,6 +47,7 @@ export enum SourceNames {
   ARANGODB = "ARANGODB",
   REST = "REST",
   ORACLE = "ORACLE",
+  GOOGLE_SHEETS = "GOOGLE_SHEETS",
 }
 
 export enum IncludeRelationships {
@@ -86,6 +87,8 @@ export interface ExtraQueryConfig {
 export interface Integration {
   docs: string
   plus?: boolean
+  auth?: { type: string }
+  relationships?: boolean
   description: string
   friendlyName: string
   datasource: {}