@budibase/server 2.4.40 → 2.4.42-alpha.0

package/src/integrations/googlesheets.ts

@@ -1,22 +1,25 @@
 import {
   DatasourceFieldType,
   DatasourcePlus,
+  FieldType,
   Integration,
+  Operation,
   PaginationJson,
   QueryJson,
   QueryType,
+  Row,
   SearchFilters,
   SortJson,
   Table,
-  TableSchema,
+  TableRequest,
 } from "@budibase/types"
 import { OAuth2Client } from "google-auth-library"
-import { buildExternalTableId } from "./utils"
-import { DataSourceOperation, FieldTypes } from "../constants"
+import { buildExternalTableId, finaliseExternalTables } from "./utils"
 import { GoogleSpreadsheet } from "google-spreadsheet"
 import fetch from "node-fetch"
 import { configs, HTTPError } from "@budibase/backend-core"
 import { dataFilters } from "@budibase/shared-core"
+import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
 
 interface GoogleSheetsConfig {
   spreadsheetId: string
@@ -39,6 +42,17 @@ interface AuthTokenResponse {
   access_token: string
 }
 
+const ALLOWED_TYPES = [
+  FieldType.STRING,
+  FieldType.FORMULA,
+  FieldType.NUMBER,
+  FieldType.LONGFORM,
+  FieldType.DATETIME,
+  FieldType.OPTIONS,
+  FieldType.BOOLEAN,
+  FieldType.BARCODEQR,
+]
+
 const SCHEMA: Integration = {
   plus: true,
   auth: {
@@ -199,73 +213,90 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
       this.client.useOAuth2Client(oauthClient)
       await this.client.loadInfo()
-    } catch (err) {
+    } catch (err: any) {
+      // this happens for xlsx imports
+      if (err.message?.includes("operation is not supported")) {
+        err.message =
+          "This operation is not supported - XLSX sheets must be converted."
+      }
       console.error("Error connecting to google sheets", err)
       throw err
     }
   }
 
-  async buildSchema(datasourceId: string) {
+  getTableSchema(title: string, headerValues: string[], id?: string) {
+    // base table
+    const table: Table = {
+      name: title,
+      primary: [GOOGLE_SHEETS_PRIMARY_KEY],
+      schema: {},
+    }
+    if (id) {
+      table._id = id
+    }
+    // build schema from headers
+    for (let header of headerValues) {
+      table.schema[header] = {
+        name: header,
+        type: FieldType.STRING,
+      }
+    }
+    return table
+  }
+
+  async buildSchema(datasourceId: string, entities: Record<string, Table>) {
     await this.connect()
     const sheets = this.client.sheetsByIndex
     const tables: Record<string, Table> = {}
     for (let sheet of sheets) {
       // must fetch rows to determine schema
       await sheet.getRows()
-      // build schema
-      const schema: TableSchema = {}
-
-      // build schema from headers
-      for (let header of sheet.headerValues) {
-        schema[header] = {
-          name: header,
-          type: FieldTypes.STRING,
-        }
-      }
 
-      // create tables
-      tables[sheet.title] = {
-        _id: buildExternalTableId(datasourceId, sheet.title),
-        name: sheet.title,
-        primary: ["rowNumber"],
-        schema,
-      }
+      const id = buildExternalTableId(datasourceId, sheet.title)
+      tables[sheet.title] = this.getTableSchema(
+        sheet.title,
+        sheet.headerValues,
+        id
+      )
     }
-
-    this.tables = tables
+    const final = finaliseExternalTables(tables, entities)
+    this.tables = final.tables
+    this.schemaErrors = final.errors
   }
 
   async query(json: QueryJson) {
     const sheet = json.endpoint.entityId
-
-    const handlers = {
-      [DataSourceOperation.CREATE]: () =>
-        this.create({ sheet, row: json.body }),
-      [DataSourceOperation.READ]: () => this.read({ ...json, sheet }),
-      [DataSourceOperation.UPDATE]: () =>
-        this.update({
+    switch (json.endpoint.operation) {
+      case Operation.CREATE:
+        return this.create({ sheet, row: json.body as Row })
+      case Operation.BULK_CREATE:
+        return this.createBulk({ sheet, rows: json.body as Row[] })
+      case Operation.READ:
+        return this.read({ ...json, sheet })
+      case Operation.UPDATE:
+        return this.update({
           // exclude the header row and zero index
           rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
           sheet,
           row: json.body,
-        }),
-      [DataSourceOperation.DELETE]: () =>
-        this.delete({
+        })
+      case Operation.DELETE:
+        return this.delete({
           // exclude the header row and zero index
           rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
           sheet,
-        }),
-      [DataSourceOperation.CREATE_TABLE]: () =>
-        this.createTable(json?.table?.name),
-      [DataSourceOperation.UPDATE_TABLE]: () => this.updateTable(json.table),
-      [DataSourceOperation.DELETE_TABLE]: () =>
-        this.deleteTable(json?.table?.name),
+        })
+      case Operation.CREATE_TABLE:
+        return this.createTable(json?.table?.name)
+      case Operation.UPDATE_TABLE:
+        return this.updateTable(json.table!)
+      case Operation.DELETE_TABLE:
+        return this.deleteTable(json?.table?.name)
+      default:
+        throw new Error(
+          `GSheets integration does not support "${json.endpoint.operation}".`
+        )
     }
-
-    // @ts-ignore
-    const internalQueryMethod = handlers[json.endpoint.operation]
-
-    return await internalQueryMethod()
   }
 
   buildRowObject(headers: string[], values: string[], rowNumber: number) {
@@ -278,47 +309,70 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   }
 
   async createTable(name?: string) {
+    if (!name) {
+      throw new Error("Must provide name for new sheet.")
+    }
     try {
       await this.connect()
-      return await this.client.addSheet({ title: name, headerValues: ["test"] })
+      return await this.client.addSheet({ title: name, headerValues: [name] })
     } catch (err) {
       console.error("Error creating new table in google sheets", err)
       throw err
     }
   }
 
-  async updateTable(table?: any) {
-    try {
-      await this.connect()
-      const sheet = this.client.sheetsByTitle[table.name]
-      await sheet.loadHeaderRow()
-
-      if (table._rename) {
-        const headers = []
-        for (let header of sheet.headerValues) {
-          if (header === table._rename.old) {
-            headers.push(table._rename.updated)
-          } else {
-            headers.push(header)
-          }
+  async updateTable(table: TableRequest) {
+    await this.connect()
+    const sheet = this.client.sheetsByTitle[table.name]
+    await sheet.loadHeaderRow()
+
+    if (table._rename) {
+      const headers = []
+      for (let header of sheet.headerValues) {
+        if (header === table._rename.old) {
+          headers.push(table._rename.updated)
+        } else {
+          headers.push(header)
         }
+      }
+      try {
         await sheet.setHeaderRow(headers)
-      } else {
-        const updatedHeaderValues = [...sheet.headerValues]
-
-        const newField = Object.keys(table.schema).find(
-          key => !sheet.headerValues.includes(key)
-        )
+      } catch (err) {
+        console.error("Error updating column name in google sheets", err)
+        throw err
+      }
+    } else {
+      const updatedHeaderValues = [...sheet.headerValues]
+
+      // add new column - doesn't currently exist
+      for (let [key, column] of Object.entries(table.schema)) {
+        if (!ALLOWED_TYPES.includes(column.type)) {
+          throw new Error(
+            `Column type: ${column.type} not allowed for GSheets integration.`
+          )
+        }
+        if (
+          !sheet.headerValues.includes(key) &&
+          column.type !== FieldType.FORMULA
+        ) {
+          updatedHeaderValues.push(key)
+        }
+      }
 
-        if (newField) {
-          updatedHeaderValues.push(newField)
+      // clear out deleted columns
+      for (let key of sheet.headerValues) {
+        if (!Object.keys(table.schema).includes(key)) {
+          const idx = updatedHeaderValues.indexOf(key)
+          updatedHeaderValues.splice(idx, 1)
         }
+      }
 
+      try {
         await sheet.setHeaderRow(updatedHeaderValues)
+      } catch (err) {
+        console.error("Error updating table in google sheets", err)
+        throw err
       }
-    } catch (err) {
-      console.error("Error updating table in google sheets", err)
-      throw err
     }
   }
 
@@ -349,6 +403,24 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     }
   }
 
+  async createBulk(query: { sheet: string; rows: any[] }) {
+    try {
+      await this.connect()
+      const sheet = this.client.sheetsByTitle[query.sheet]
+      let rowsToInsert = []
+      for (let row of query.rows) {
+        rowsToInsert.push(typeof row === "string" ? JSON.parse(row) : row)
+      }
+      const rows = await sheet.addRows(rowsToInsert)
+      return rows.map(row =>
+        this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber)
+      )
+    } catch (err) {
+      console.error("Error bulk writing to google sheets", err)
+      throw err
+    }
+  }
+
   async read(query: {
     sheet: string
     filters?: SearchFilters

package/src/integrations/redis.ts

@@ -92,7 +92,7 @@ class RedisIntegration {
   }
 
   async disconnect() {
-    return this.client.disconnect()
+    return this.client.quit()
   }
 
   async redisContext(query: Function) {

package/src/integrations/tests/googlesheets.spec.ts

@@ -39,6 +39,10 @@ describe("Google Sheets Integration", () => {
     config.setGoogleAuth("test")
   })
 
+  afterAll(async () => {
+    await config.end()
+  })
+
   beforeEach(async () => {
     integration = new GoogleSheetsIntegration.integration({
       spreadsheetId: "randomId",
@@ -99,8 +103,8 @@ describe("Google Sheets Integration", () => {
       })
     })
 
-    test("removing an existing field will not remove the data from the spreadsheet", async () => {
-      await config.doInContext(structures.uuid(), async () => {
+    test("removing an existing field will remove the header from the google sheet", async () => {
+      const sheet = await config.doInContext(structures.uuid(), async () => {
         const tableColumns = ["name"]
         const table = createBasicTable(structures.uuid(), tableColumns)
 
@@ -109,18 +113,14 @@ describe("Google Sheets Integration", () => {
         })
         sheetsByTitle[table.name] = sheet
         await integration.updateTable(table)
-
-        expect(sheet.loadHeaderRow).toBeCalledTimes(1)
-        expect(sheet.setHeaderRow).toBeCalledTimes(1)
-        expect(sheet.setHeaderRow).toBeCalledWith([
-          "name",
-          "description",
-          "location",
-        ])
-
-        // No undefineds are sent
-        expect((sheet.setHeaderRow as any).mock.calls[0][0]).toHaveLength(3)
+        return sheet
       })
+      expect(sheet.loadHeaderRow).toBeCalledTimes(1)
+      expect(sheet.setHeaderRow).toBeCalledTimes(1)
+      expect(sheet.setHeaderRow).toBeCalledWith(["name"])
+
+      // No undefined are sent
+      expect((sheet.setHeaderRow as any).mock.calls[0][0]).toHaveLength(1)
     })
   })
 })

package/src/integrations/tests/redis.spec.ts

@@ -3,17 +3,17 @@ import { default as RedisIntegration } from "../redis"
 
 class TestConfiguration {
   integration: any
-  redis: any
 
   constructor(config: any = {}) {
     this.integration = new RedisIntegration.integration(config)
-    this.redis = new Redis({
+    // have to kill the basic integration before replacing it
+    this.integration.client.quit()
+    this.integration.client = new Redis({
       data: {
         test: "test",
         result: "1",
       },
     })
-    this.integration.client = this.redis
   }
 }
 
@@ -24,13 +24,17 @@ describe("Redis Integration", () => {
     config = new TestConfiguration()
   })
 
+  afterAll(() => {
+    config.integration.disconnect()
+  })
+
   it("calls the create method with the correct params", async () => {
     const body = {
       key: "key",
       value: "value",
     }
     await config.integration.create(body)
-    expect(await config.redis.get("key")).toEqual("value")
+    expect(await config.integration.client.get("key")).toEqual("value")
   })
 
   it("calls the read method with the correct params", async () => {
@@ -46,7 +50,7 @@ describe("Redis Integration", () => {
       key: "test",
     }
     await config.integration.delete(body)
-    expect(await config.redis.get(body.key)).toEqual(null)
+    expect(await config.integration.client.get(body.key)).toEqual(null)
   })
 
   it("calls the pipeline method with the correct params", async () => {

package/src/integrations/utils.ts

@@ -4,6 +4,7 @@ import { FieldTypes, BuildSchemaErrors, InvalidColumns } from "../constants"
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
+const ENCODED_SPACE = encodeURIComponent(" ")
 
 const SQL_NUMBER_TYPE_MAP = {
   integer: FieldTypes.NUMBER,
@@ -79,6 +80,10 @@ export function isExternalTable(tableId: string) {
 }
 
 export function buildExternalTableId(datasourceId: string, tableName: string) {
+  // encode spaces
+  if (tableName.includes(" ")) {
+    tableName = encodeURIComponent(tableName)
+  }
   return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
 }
 
@@ -90,6 +95,10 @@ export function breakExternalTableId(tableId: string | undefined) {
   let datasourceId = parts.shift()
   // if they need joined
   let tableName = parts.join(DOUBLE_SEPARATOR)
+  // if contains encoded spaces, decode it
+  if (tableName.includes(ENCODED_SPACE)) {
+    tableName = decodeURIComponent(tableName)
+  }
   return { datasourceId, tableName }
 }
 
@@ -200,9 +209,9 @@ export function isIsoDateString(str: string) {
  * @param column The column to check, to see if it is a valid relationship.
  * @param tableIds The IDs of the tables which currently exist.
  */
-function shouldCopyRelationship(
+export function shouldCopyRelationship(
   column: { type: string; tableId?: string },
-  tableIds: [string]
+  tableIds: string[]
 ) {
   return (
     column.type === FieldTypes.LINK &&
@@ -219,7 +228,7 @@ function shouldCopyRelationship(
  * @param column The column to check for options or boolean type.
  * @param fetchedColumn The fetched column to check for the type in the external database.
  */
-function shouldCopySpecialColumn(
+export function shouldCopySpecialColumn(
   column: { type: string },
   fetchedColumn: { type: string } | undefined
 ) {
@@ -257,9 +266,12 @@ function copyExistingPropsOver(
   tableIds: [string]
 ) {
   if (entities && entities[tableName]) {
-    if (entities[tableName].primaryDisplay) {
+    if (entities[tableName]?.primaryDisplay) {
       table.primaryDisplay = entities[tableName].primaryDisplay
     }
+    if (entities[tableName]?.created) {
+      table.created = entities[tableName]?.created
+    }
     const existingTableSchema = entities[tableName].schema
     for (let key in existingTableSchema) {
       if (!existingTableSchema.hasOwnProperty(key)) {

package/src/middleware/currentapp.ts

@@ -10,9 +10,9 @@ import { generateUserMetadataID, isDevAppID } from "../db/utils"
 import { getCachedSelf } from "../utilities/global"
 import env from "../environment"
 import { isWebhookEndpoint } from "./utils"
-import { BBContext } from "@budibase/types"
+import { UserCtx } from "@budibase/types"
 
-export default async (ctx: BBContext, next: any) => {
+export default async (ctx: UserCtx, next: any) => {
   // try to get the appID from the request
   let requestAppId = await utils.getAppIdFromCtx(ctx)
   // get app cookie if it exists

package/src/sdk/users/utils.ts

@@ -9,7 +9,10 @@ import { isEqual } from "lodash"
 
 export function combineMetadataAndUser(user: any, metadata: any) {
   // skip users with no access
-  if (user.roleId === rolesCore.BUILTIN_ROLE_IDS.PUBLIC) {
+  if (
+    user.roleId == null ||
+    user.roleId === rolesCore.BUILTIN_ROLE_IDS.PUBLIC
+  ) {
     return null
   }
   delete user._rev

package/src/tests/jestEnv.ts

@@ -9,3 +9,4 @@ process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
 process.env.ENABLE_4XX_HTTP_LOGGING = "0"
 process.env.MOCK_REDIS = "1"
 process.env.PLATFORM_URL = "http://localhost:10000"
+process.env.REDIS_PASSWORD = "budibase"

package/src/tests/jestSetup.ts

@@ -1,6 +1,6 @@
 import "./logging"
 import env from "../environment"
-import { env as coreEnv } from "@budibase/backend-core"
+import { env as coreEnv, timers } from "@budibase/backend-core"
 import { testContainerUtils } from "@budibase/backend-core/tests"
 
 if (!process.env.DEBUG) {
@@ -17,3 +17,7 @@ if (!process.env.CI) {
 }
 
 testContainerUtils.setupEnv(env, coreEnv)
+
+afterAll(() => {
+  timers.cleanup()
+})

package/src/tests/utilities/TestConfiguration.ts

@@ -46,6 +46,7 @@ import {
   Row,
   SourceName,
   Table,
+  SearchFilters,
 } from "@budibase/types"
 
 type DefaultUserValues = {
@@ -164,6 +165,8 @@ class TestConfiguration {
     }
     if (this.server) {
       this.server.close()
+    } else {
+      require("../../app").default.close()
     }
     if (this.allApps) {
       cleanup(this.allApps.map(app => app.appId))
@@ -568,6 +571,16 @@ class TestConfiguration {
     return this._req(null, { tableId }, controllers.row.fetch)
   }
 
+  async searchRows(tableId: string, searchParams: SearchFilters = {}) {
+    if (!tableId && this.table) {
+      tableId = this.table._id
+    }
+    const body = {
+      query: searchParams,
+    }
+    return this._req(body, { tableId }, controllers.row.search)
+  }
+
   // ROLE
 
   async createRole(config?: any) {

package/src/tests/utilities/structures.ts

@@ -106,7 +106,7 @@ export function newAutomation({ steps, trigger }: any = {}) {
   return automation
 }
 
-export function basicAutomation() {
+export function basicAutomation(appId?: string) {
   return {
     name: "My Automation",
     screenId: "kasdkfldsafkl",
@@ -114,11 +114,23 @@ export function basicAutomation() {
     uiTree: {},
     definition: {
       trigger: {
+        stepId: AutomationTriggerStepId.APP,
+        name: "test",
+        tagline: "test",
+        icon: "test",
+        description: "test",
+        type: "trigger",
+        id: "test",
         inputs: {},
+        schema: {
+          inputs: {},
+          outputs: {},
+        },
       },
       steps: [],
     },
     type: "automation",
+    appId,
   }
 }
 

package/src/utilities/global.ts

@@ -8,7 +8,7 @@ import {
 } from "@budibase/backend-core"
 import env from "../environment"
 import { groups } from "@budibase/pro"
-import { BBContext, ContextUser, User } from "@budibase/types"
+import { UserCtx, ContextUser, User, UserGroup } from "@budibase/types"
 
 export function updateAppRole(
   user: ContextUser,
@@ -43,33 +43,40 @@ export function updateAppRole(
 
 async function checkGroupRoles(
   user: ContextUser,
-  { appId }: { appId?: string } = {}
+  opts: { appId?: string; groups?: UserGroup[] } = {}
 ) {
   if (user.roleId && user.roleId !== roles.BUILTIN_ROLE_IDS.PUBLIC) {
     return user
   }
-  if (appId) {
-    user.roleId = await groups.getGroupRoleId(user as User, appId)
+  if (opts.appId) {
+    user.roleId = await groups.getGroupRoleId(user as User, opts.appId, {
+      groups: opts.groups,
+    })
+  }
+  // final fallback, simply couldn't find a role - user must be public
+  if (!user.roleId) {
+    user.roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
   }
   return user
 }
 
 async function processUser(
   user: ContextUser,
-  { appId }: { appId?: string } = {}
+  opts: { appId?: string; groups?: UserGroup[] } = {}
 ) {
   if (user) {
     delete user.password
   }
-  user = await updateAppRole(user, { appId })
+  const appId = opts.appId || context.getAppId()
+  user = updateAppRole(user, { appId })
   if (!user.roleId && user?.userGroups?.length) {
-    user = await checkGroupRoles(user, { appId })
+    user = await checkGroupRoles(user, { appId, groups: opts?.groups })
   }
 
   return user
 }
 
-export async function getCachedSelf(ctx: BBContext, appId: string) {
+export async function getCachedSelf(ctx: UserCtx, appId: string) {
   // this has to be tenant aware, can't depend on the context to find it out
   // running some middlewares before the tenancy causes context to break
   const user = await cache.user.getUser(ctx.user?._id!)
@@ -90,6 +97,7 @@ export async function getGlobalUser(userId: string) {
 export async function getGlobalUsers(users?: ContextUser[]) {
   const appId = context.getAppId()
   const db = tenancy.getGlobalDB()
+  const allGroups = await groups.fetch()
   let globalUsers
   if (users) {
     const globalIds = users.map(user =>
@@ -118,7 +126,11 @@ export async function getGlobalUsers(users?: ContextUser[]) {
     return globalUsers
   }
 
-  return globalUsers.map(user => updateAppRole(user))
+  // pass in the groups, meaning we don't actually need to retrieve them for
+  // each user individually
+  return Promise.all(
+    globalUsers.map(user => processUser(user, { groups: allGroups }))
+  )
 }
 
 export async function getGlobalUsersFromMetadata(users: ContextUser[]) {