@budibase/server 1.0.46-alpha.4 → 1.0.46-alpha.8

package/src/integrations/googlesheets.ts

@@ -0,0 +1,353 @@
+import {
+  DatasourceFieldTypes,
+  Integration,
+  QueryJson,
+  QueryTypes,
+} from "../definitions/datasource"
+import { OAuth2Client } from "google-auth-library"
+import { DatasourcePlus } from "./base/datasourcePlus"
+import { Row, Table, TableSchema } from "../definitions/common"
+import { buildExternalTableId } from "./utils"
+import { DataSourceOperation, FieldTypes } from "../constants"
+import { GoogleSpreadsheet } from "google-spreadsheet"
+import { table } from "console"
+
+module GoogleSheetsModule {
+  const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+  const { getScopedConfig } = require("@budibase/backend-core/db")
+  const { Configs } = require("@budibase/backend-core/constants")
+
+  interface GoogleSheetsConfig {
+    spreadsheetId: string
+    auth: OAuthClientConfig
+  }
+
+  interface OAuthClientConfig {
+    appId: string
+    accessToken: string
+    refreshToken: string
+  }
+
+  const SCHEMA: Integration = {
+    plus: true,
+    auth: {
+      type: "google",
+    },
+    relationships: false,
+    docs: "https://developers.google.com/sheets/api/quickstart/nodejs",
+    description:
+      "Create and collaborate on online spreadsheets in real-time and from any device. ",
+    friendlyName: "Google Sheets",
+    datasource: {
+      spreadsheetId: {
+        type: DatasourceFieldTypes.STRING,
+        required: true,
+      },
+    },
+    query: {
+      create: {
+        type: QueryTypes.FIELDS,
+        fields: {
+          sheet: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+          row: {
+            type: QueryTypes.JSON,
+            required: true,
+          },
+        },
+      },
+      read: {
+        type: QueryTypes.FIELDS,
+        fields: {
+          sheet: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+        },
+      },
+      update: {
+        type: QueryTypes.FIELDS,
+        fields: {
+          sheet: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+          rowIndex: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+          row: {
+            type: QueryTypes.JSON,
+            required: true,
+          },
+        },
+      },
+      delete: {
+        type: QueryTypes.FIELDS,
+        fields: {
+          sheet: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+          rowIndex: {
+            type: DatasourceFieldTypes.NUMBER,
+            required: true,
+          },
+        },
+      },
+    },
+  }
+
+  class GoogleSheetsIntegration implements DatasourcePlus {
+    private readonly config: GoogleSheetsConfig
+    private client: any
+    public tables: Record<string, Table> = {}
+    public schemaErrors: Record<string, string> = {}
+
+    constructor(config: GoogleSheetsConfig) {
+      this.config = config
+      const spreadsheetId = this.cleanSpreadsheetUrl(this.config.spreadsheetId)
+      this.client = new GoogleSpreadsheet(spreadsheetId)
+    }
+
+    /**
+     * Pull the spreadsheet ID out from a valid google sheets URL
+     * @param spreadsheetId - the URL or standard spreadsheetId of the google sheet
+     * @returns spreadsheet Id of the google sheet
+     */
+    cleanSpreadsheetUrl(spreadsheetId: string) {
+      if (!spreadsheetId) {
+        throw new Error(
+          "You must set a spreadsheet ID in your configuration to fetch tables."
+        )
+      }
+      const parts = spreadsheetId.split("/")
+      return parts.length > 5 ? parts[5] : spreadsheetId
+    }
+
+    async connect() {
+      try {
+        // Initialise oAuth client
+        const db = getGlobalDB()
+        const googleConfig = await getScopedConfig(db, {
+          type: Configs.GOOGLE,
+        })
+        const oauthClient = new OAuth2Client({
+          clientId: googleConfig.clientID,
+          clientSecret: googleConfig.clientSecret,
+        })
+        oauthClient.credentials.access_token = this.config.auth.accessToken
+        oauthClient.credentials.refresh_token = this.config.auth.refreshToken
+        this.client.useOAuth2Client(oauthClient)
+        await this.client.loadInfo()
+      } catch (err) {
+        console.error("Error connecting to google sheets", err)
+        throw err
+      }
+    }
+
+    async buildSchema(datasourceId: string) {
+      await this.connect()
+      const sheets = await this.client.sheetsByIndex
+      const tables: Record<string, Table> = {}
+      for (let sheet of sheets) {
+        // must fetch rows to determine schema
+        await sheet.getRows()
+        // build schema
+        const schema: TableSchema = {}
+
+        // build schema from headers
+        for (let header of sheet.headerValues) {
+          schema[header] = {
+            name: header,
+            type: FieldTypes.STRING,
+          }
+        }
+
+        // create tables
+        tables[sheet.title] = {
+          _id: buildExternalTableId(datasourceId, sheet.title),
+          name: sheet.title,
+          primary: ["rowNumber"],
+          schema,
+        }
+      }
+
+      this.tables = tables
+    }
+
+    async query(json: QueryJson) {
+      const sheet = json.endpoint.entityId
+
+      const handlers = {
+        [DataSourceOperation.CREATE]: () =>
+          this.create({ sheet, row: json.body }),
+        [DataSourceOperation.READ]: () => this.read({ sheet }),
+        [DataSourceOperation.UPDATE]: () =>
+          this.update({
+            // exclude the header row and zero index
+            rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
+            sheet,
+            row: json.body,
+          }),
+        [DataSourceOperation.DELETE]: () =>
+          this.delete({
+            // exclude the header row and zero index
+            rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
+            sheet,
+          }),
+        [DataSourceOperation.CREATE_TABLE]: () =>
+          this.createTable(json?.table?.name),
+        [DataSourceOperation.UPDATE_TABLE]: () => this.updateTable(json.table),
+        [DataSourceOperation.DELETE_TABLE]: () =>
+          this.deleteTable(json?.table?.name),
+      }
+
+      const internalQueryMethod = handlers[json.endpoint.operation]
+
+      return await internalQueryMethod()
+    }
+
+    buildRowObject(headers: string[], values: string[], rowNumber: number) {
+      const rowObject: { rowNumber: number; [key: string]: any } = { rowNumber }
+      for (let i = 0; i < headers.length; i++) {
+        rowObject._id = rowNumber
+        rowObject[headers[i]] = values[i]
+      }
+      return rowObject
+    }
+
+    async createTable(name?: string) {
+      try {
+        await this.connect()
+        const sheet = await this.client.addSheet({ title: name })
+        return sheet
+      } catch (err) {
+        console.error("Error creating new table in google sheets", err)
+        throw err
+      }
+    }
+
+    async updateTable(table?: any) {
+      try {
+        await this.connect()
+        const sheet = await this.client.sheetsByTitle[table.name]
+        await sheet.loadHeaderRow()
+
+        if (table._rename) {
+          const headers = []
+          for (let header of sheet.headerValues) {
+            if (header === table._rename.old) {
+              headers.push(table._rename.updated)
+            } else {
+              headers.push(header)
+            }
+          }
+          await sheet.setHeaderRow(headers)
+        } else {
+          let newField = Object.keys(table.schema).find(
+            key => !sheet.headerValues.includes(key)
+          )
+          await sheet.setHeaderRow([...sheet.headerValues, newField])
+        }
+      } catch (err) {
+        console.error("Error updating table in google sheets", err)
+        throw err
+      }
+    }
+
+    async deleteTable(sheet: any) {
+      try {
+        await this.connect()
+        const sheetToDelete = await this.client.sheetsByTitle[sheet]
+        return await sheetToDelete.delete()
+      } catch (err) {
+        console.error("Error deleting table in google sheets", err)
+        throw err
+      }
+    }
+
+    async create(query: { sheet: string; row: any }) {
+      try {
+        await this.connect()
+        const sheet = await this.client.sheetsByTitle[query.sheet]
+        const rowToInsert =
+          typeof query.row === "string" ? JSON.parse(query.row) : query.row
+        const row = await sheet.addRow(rowToInsert)
+        return [
+          this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber),
+        ]
+      } catch (err) {
+        console.error("Error writing to google sheets", err)
+        throw err
+      }
+    }
+
+    async read(query: { sheet: string }) {
+      try {
+        await this.connect()
+        const sheet = await this.client.sheetsByTitle[query.sheet]
+        const rows = await sheet.getRows()
+        const headerValues = sheet.headerValues
+        const response = []
+        for (let row of rows) {
+          response.push(
+            this.buildRowObject(headerValues, row._rawData, row._rowNumber)
+          )
+        }
+        return response
+      } catch (err) {
+        console.error("Error reading from google sheets", err)
+        throw err
+      }
+    }
+
+    async update(query: { sheet: string; rowIndex: number; row: any }) {
+      try {
+        await this.connect()
+        const sheet = await this.client.sheetsByTitle[query.sheet]
+        const rows = await sheet.getRows()
+        const row = rows[query.rowIndex]
+        if (row) {
+          const updateValues = query.row
+          for (let key in updateValues) {
+            row[key] = updateValues[key]
+          }
+          await row.save()
+          return [
+            this.buildRowObject(
+              sheet.headerValues,
+              row._rawData,
+              row._rowNumber
+            ),
+          ]
+        } else {
+          throw new Error("Row does not exist.")
+        }
+      } catch (err) {
+        console.error("Error reading from google sheets", err)
+        throw err
+      }
+    }
+
+    async delete(query: { sheet: string; rowIndex: number }) {
+      await this.connect()
+      const sheet = await this.client.sheetsByTitle[query.sheet]
+      const rows = await sheet.getRows()
+      const row = rows[query.rowIndex]
+      if (row) {
+        await row.delete()
+        return [{ deleted: query.rowIndex }]
+      } else {
+        throw new Error("Row does not exist.")
+      }
+    }
+  }
+
+  module.exports = {
+    schema: SCHEMA,
+    integration: GoogleSheetsIntegration,
+  }
+}

package/src/integrations/index.ts

@@ -9,6 +9,7 @@ const airtable = require("./airtable")
 const mysql = require("./mysql")
 const arangodb = require("./arangodb")
 const rest = require("./rest")
+const googlesheets = require("./googlesheets")
 const { SourceNames } = require("../definitions/datasource")
 
 const DEFINITIONS = {
@@ -23,6 +24,7 @@ const DEFINITIONS = {
   [SourceNames.MYSQL]: mysql.schema,
   [SourceNames.ARANGODB]: arangodb.schema,
   [SourceNames.REST]: rest.schema,
+  [SourceNames.GOOGLE_SHEETS]: googlesheets.schema,
 }
 
 const INTEGRATIONS = {
@@ -37,6 +39,7 @@ const INTEGRATIONS = {
   [SourceNames.MYSQL]: mysql.integration,
   [SourceNames.ARANGODB]: arangodb.integration,
   [SourceNames.REST]: rest.integration,
+  [SourceNames.GOOGLE_SHEETS]: googlesheets.integration,
 }
 
 // optionally add oracle integration if the oracle binary can be installed

package/src/integrations/utils.ts

@@ -153,8 +153,15 @@ export function isIsoDateString(str: string) {
  * @param column The column to check, to see if it is a valid relationship.
  * @param tableIds The IDs of the tables which currently exist.
  */
-function shouldCopyRelationship(column: { type: string, tableId?: string }, tableIds: [string]) {
-  return column.type === FieldTypes.LINK && column.tableId && tableIds.includes(column.tableId)
+function shouldCopyRelationship(
+  column: { type: string; tableId?: string },
+  tableIds: [string]
+) {
+  return (
+    column.type === FieldTypes.LINK &&
+    column.tableId &&
+    tableIds.includes(column.tableId)
+  )
 }
 
 /**
@@ -165,9 +172,15 @@ function shouldCopyRelationship(column: { type: string, tableId?: string }, tabl
  * @param column The column to check for options or boolean type.
  * @param fetchedColumn The fetched column to check for the type in the external database.
  */
-function shouldCopySpecialColumn(column: { type: string }, fetchedColumn: { type: string } | undefined) {
-  return column.type === FieldTypes.OPTIONS ||
-    ((!fetchedColumn || fetchedColumn.type === FieldTypes.NUMBER) && column.type === FieldTypes.BOOLEAN)
+function shouldCopySpecialColumn(
+  column: { type: string },
+  fetchedColumn: { type: string } | undefined
+) {
+  return (
+    column.type === FieldTypes.OPTIONS ||
+    ((!fetchedColumn || fetchedColumn.type === FieldTypes.NUMBER) &&
+      column.type === FieldTypes.BOOLEAN)
+  )
 }
 
 /**

package/src/middleware/authorized.js

@@ -9,11 +9,59 @@ const {
 } = require("@budibase/backend-core/permissions")
 const builderMiddleware = require("./builder")
 const { isWebhookEndpoint } = require("./utils")
+const { buildCsrfMiddleware } = require("@budibase/backend-core/auth")
 
 function hasResource(ctx) {
   return ctx.resourceId != null
 }
 
+const csrf = buildCsrfMiddleware()
+
+/**
+ * Apply authorization to the requested resource:
+ * - If this is a builder resource the user must be a builder.
+ * - Builders can access all resources.
+ * - Otherwise the user must have the required role.
+ */
+const checkAuthorized = async (ctx, resourceRoles, permType, permLevel) => {
+  // check if this is a builder api and the user is not a builder
+  const isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global
+  const isBuilderApi = permType === PermissionTypes.BUILDER
+  if (isBuilderApi && !isBuilder) {
+    return ctx.throw(403, "Not Authorized")
+  }
+
+  // check for resource authorization
+  if (!isBuilder) {
+    await checkAuthorizedResource(ctx, resourceRoles, permType, permLevel)
+  }
+}
+
+const checkAuthorizedResource = async (
+  ctx,
+  resourceRoles,
+  permType,
+  permLevel
+) => {
+  // get the user's roles
+  const roleId = ctx.roleId || BUILTIN_ROLE_IDS.PUBLIC
+  const userRoles = await getUserRoleHierarchy(ctx.appId, roleId, {
+    idOnly: false,
+  })
+  const permError = "User does not have permission"
+  // check if the user has the required role
+  if (resourceRoles.length > 0) {
+    // deny access if the user doesn't have the required resource role
+    const found = userRoles.find(role => resourceRoles.indexOf(role._id) !== -1)
+    if (!found) {
+      ctx.throw(403, permError)
+    }
+    // fallback to the base permissions when no resource roles are found
+  } else if (!doesHaveBasePermission(permType, permLevel, userRoles)) {
+    ctx.throw(403, permError)
+  }
+}
+
 module.exports =
   (permType, permLevel = null) =>
   async (ctx, next) => {
@@ -31,40 +79,26 @@ module.exports =
     // to find API endpoints which are builder focused
     await builderMiddleware(ctx, permType)
 
-    const isAuthed = ctx.isAuthenticated
-    // builders for now have permission to do anything
-    let isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global
-    const isBuilderApi = permType === PermissionTypes.BUILDER
-    if (isBuilder) {
-      return next()
-    } else if (isBuilderApi && !isBuilder) {
-      return ctx.throw(403, "Not Authorized")
+    // get the resource roles
+    let resourceRoles = []
+    if (ctx.appId && hasResource(ctx)) {
+      resourceRoles = await getRequiredResourceRole(ctx.appId, permLevel, ctx)
     }
 
-    // need to check this first, in-case public access, don't check authed until last
-    const roleId = ctx.roleId || BUILTIN_ROLE_IDS.PUBLIC
-    const hierarchy = await getUserRoleHierarchy(ctx.appId, roleId, {
-      idOnly: false,
-    })
-    const permError = "User does not have permission"
-    let possibleRoleIds = []
-    if (hasResource(ctx)) {
-      possibleRoleIds = await getRequiredResourceRole(ctx.appId, permLevel, ctx)
-    }
-    // check if we found a role, if not fallback to base permissions
-    if (possibleRoleIds.length > 0) {
-      const found = hierarchy.find(
-        role => possibleRoleIds.indexOf(role._id) !== -1
-      )
-      return found ? next() : ctx.throw(403, permError)
-    } else if (!doesHaveBasePermission(permType, permLevel, hierarchy)) {
-      ctx.throw(403, permError)
+    // if the resource is public, proceed
+    const isPublicResource = resourceRoles.includes(BUILTIN_ROLE_IDS.PUBLIC)
+    if (isPublicResource) {
+      return next()
     }
 
-    // if they are not authed, then anything using the authorized middleware will fail
-    if (!isAuthed) {
-      ctx.throw(403, "Session not authenticated")
+    // check authenticated
+    if (!ctx.isAuthenticated) {
+      return ctx.throw(403, "Session not authenticated")
     }
 
-    return next()
+    // check authorized
+    await checkAuthorized(ctx, resourceRoles, permType, permLevel)
+
+    // csrf protection
+    return csrf(ctx, next)
   }

package/src/middleware/currentapp.js

@@ -47,15 +47,6 @@ module.exports = async (ctx, next) => {
     (!ctx.user || !ctx.user.builder || !ctx.user.builder.global)
   ) {
     clearCookie(ctx, Cookies.CurrentApp)
-    // have to set the return url on the server side as client side is not available
-    setCookie(ctx, ctx.url, Cookies.RETURN_URL, {
-      // don't sign so the browser can easily read
-      sign: false,
-      // use the request domain to match how ui handles the return url cookie.
-      // it's important we don't use the shared domain here as the builder
-      // can't delete from it without awareness of the domain.
-      requestDomain: true,
-    })
     return ctx.redirect("/")
   }
 

package/src/middleware/tests/authorized.spec.js

@@ -17,6 +17,7 @@ class TestConfiguration {
     this.middleware = authorizedMiddleware(role)
     this.next = jest.fn()
     this.throw = jest.fn()
+    this.headers = {}
     this.ctx = {
       headers: {},
       request: {
@@ -25,7 +26,8 @@ class TestConfiguration {
       appId: "",
       auth: {},
       next: this.next,
-      throw: this.throw
+      throw: this.throw,
+      get: (name) => this.headers[name],
     }
   }
 
@@ -46,7 +48,7 @@ class TestConfiguration {
   }
 
   setAuthenticated(isAuthed) {
-    this.ctx.auth = { authenticated: isAuthed }
+    this.ctx.isAuthenticated = isAuthed
   }
 
   setRequestUrl(url) {
@@ -107,7 +109,7 @@ describe("Authorization middleware", () => {
       expect(config.next).toHaveBeenCalled()
     })
     
-    it("throws if the user has only builder permissions", async () => {
+    it("throws if the user does not have builder permissions", async () => {
       config.setEnvironment(false)
       config.setMiddlewareRequiredPermission(PermissionTypes.BUILDER)
       config.setUser({
@@ -133,7 +135,7 @@ describe("Authorization middleware", () => {
       expect(config.next).toHaveBeenCalled()
     })
 
-    it("throws if the user session is not authenticated after permission checks", async () => {
+    it("throws if the user session is not authenticated", async () => {
       config.setUser({
         role: {
           _id: ""

package/src/migrations/tests/usageQuotas/syncRows.spec.js

@@ -27,7 +27,7 @@ describe("syncRows", () => {
     await config.createTable()
     await config.createRow()
     // app 2
-    await config.createApp()
+    await config.createApp("second-app")
     await config.createTable()
     await config.createRow()
     await config.createRow()

package/src/tests/utilities/TestConfiguration.js

@@ -22,10 +22,12 @@ const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 const { createASession } = require("@budibase/backend-core/sessions")
 const { user: userCache } = require("@budibase/backend-core/cache")
 const CouchDB = require("../../db")
+const newid = require("../../db/newid")
 core.init(CouchDB)
 
 const GLOBAL_USER_ID = "us_uuid1"
 const EMAIL = "babs@babs.com"
+const CSRF_TOKEN = "e3727778-7af0-4226-b5eb-f43cbe60a306"
 
 class TestConfiguration {
   constructor(openServer = true) {
@@ -85,7 +87,11 @@ class TestConfiguration {
       roles: roles || {},
       tenantId: TENANT_ID,
     }
-    await createASession(id, { sessionId: "sessionid", tenantId: TENANT_ID })
+    await createASession(id, {
+      sessionId: "sessionid",
+      tenantId: TENANT_ID,
+      csrfToken: CSRF_TOKEN,
+    })
     if (builder) {
       user.builder = { global: true }
     } else {
@@ -98,7 +104,8 @@ class TestConfiguration {
     }
   }
 
-  async init(appName = "test_application") {
+  // use a new id as the name to avoid name collisions
+  async init(appName = newid()) {
     await this.globalUser()
     return this.createApp(appName)
   }
@@ -131,6 +138,7 @@ class TestConfiguration {
         `${Cookies.Auth}=${authToken}`,
         `${Cookies.CurrentApp}=${appToken}`,
       ],
+      [Headers.CSRF_TOKEN]: CSRF_TOKEN,
     }
     if (this.appId) {
       headers[Headers.APP_ID] = this.appId
@@ -424,10 +432,6 @@ class TestConfiguration {
         roles: { [this.prodAppId]: roleId },
       })
     }
-    await createASession(userId, {
-      sessionId: "sessionid",
-      tenantId: TENANT_ID,
-    })
     // have to fake this
     const auth = {
       userId,

package/src/threads/query.js

@@ -78,6 +78,11 @@ class QueryRunner {
       return this.execute()
     }
 
+    // check for undefined response
+    if (!rows) {
+      rows = []
+    }
+
     // needs to an array for next step
     if (!Array.isArray(rows)) {
       rows = [rows]

package/src/utilities/workerRequests.js

@@ -58,29 +58,6 @@ exports.sendSmtpEmail = async (to, from, subject, contents, automation) => {
   return response.json()
 }
 
-exports.getDeployedApps = async () => {
-  try {
-    const response = await fetch(
-      checkSlashesInUrl(env.WORKER_URL + `/api/apps`),
-      request(null, {
-        method: "GET",
-      })
-    )
-    const json = await response.json()
-    const apps = {}
-    for (let [key, value] of Object.entries(json)) {
-      if (value.url) {
-        value.url = value.url.toLowerCase()
-        apps[key.toLowerCase()] = value
-      }
-    }
-    return apps
-  } catch (err) {
-    // error, cannot determine deployed apps, don't stop app creation - sort this later
-    return {}
-  }
-}
-
 exports.getGlobalSelf = async (ctx, appId = null) => {
   const endpoint = `/api/global/users/self`
   const response = await fetch(