@budibase/server 2.3.18-alpha.3 → 2.3.18-alpha.30

package/src/integrations/googlesheets.ts

@@ -11,9 +11,8 @@ import { OAuth2Client } from "google-auth-library"
 import { buildExternalTableId } from "./utils"
 import { DataSourceOperation, FieldTypes } from "../constants"
 import { GoogleSpreadsheet } from "google-spreadsheet"
-import env from "../environment"
-import { tenancy, db as dbCore, constants } from "@budibase/backend-core"
-const fetch = require("node-fetch")
+import fetch from "node-fetch"
+import { configs, HTTPError } from "@budibase/backend-core"
 
 interface GoogleSheetsConfig {
   spreadsheetId: string
@@ -112,7 +111,7 @@ const SCHEMA: Integration = {
 
 class GoogleSheetsIntegration implements DatasourcePlus {
   private readonly config: GoogleSheetsConfig
-  private client: any
+  private client: GoogleSpreadsheet
   public tables: Record<string, Table> = {}
   public schemaErrors: Record<string, string> = {}
 
@@ -173,16 +172,9 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   async connect() {
     try {
       // Initialise oAuth client
-      const db = tenancy.getGlobalDB()
-      let googleConfig = await dbCore.getScopedConfig(db, {
-        type: constants.Config.GOOGLE,
-      })
-
+      let googleConfig = await configs.getGoogleDatasourceConfig()
       if (!googleConfig) {
-        googleConfig = {
-          clientID: env.GOOGLE_CLIENT_ID,
-          clientSecret: env.GOOGLE_CLIENT_SECRET,
-        }
+        throw new HTTPError("Google config not found", 400)
       }
 
       const oauthClient = new OAuth2Client({
@@ -211,7 +203,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
   async buildSchema(datasourceId: string) {
     await this.connect()
-    const sheets = await this.client.sheetsByIndex
+    const sheets = this.client.sheetsByIndex
     const tables: Record<string, Table> = {}
     for (let sheet of sheets) {
       // must fetch rows to determine schema
@@ -294,7 +286,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   async updateTable(table?: any) {
     try {
       await this.connect()
-      const sheet = await this.client.sheetsByTitle[table.name]
+      const sheet = this.client.sheetsByTitle[table.name]
       await sheet.loadHeaderRow()
 
       if (table._rename) {
@@ -308,10 +300,17 @@ class GoogleSheetsIntegration implements DatasourcePlus {
         }
         await sheet.setHeaderRow(headers)
       } else {
-        let newField = Object.keys(table.schema).find(
+        const updatedHeaderValues = [...sheet.headerValues]
+
+        const newField = Object.keys(table.schema).find(
           key => !sheet.headerValues.includes(key)
         )
-        await sheet.setHeaderRow([...sheet.headerValues, newField])
+
+        if (newField) {
+          updatedHeaderValues.push(newField)
+        }
+
+        await sheet.setHeaderRow(updatedHeaderValues)
       }
     } catch (err) {
       console.error("Error updating table in google sheets", err)
@@ -322,7 +321,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   async deleteTable(sheet: any) {
     try {
       await this.connect()
-      const sheetToDelete = await this.client.sheetsByTitle[sheet]
+      const sheetToDelete = this.client.sheetsByTitle[sheet]
       return await sheetToDelete.delete()
     } catch (err) {
       console.error("Error deleting table in google sheets", err)
@@ -333,7 +332,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   async create(query: { sheet: string; row: any }) {
     try {
       await this.connect()
-      const sheet = await this.client.sheetsByTitle[query.sheet]
+      const sheet = this.client.sheetsByTitle[query.sheet]
       const rowToInsert =
         typeof query.row === "string" ? JSON.parse(query.row) : query.row
       const row = await sheet.addRow(rowToInsert)
@@ -349,7 +348,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   async read(query: { sheet: string }) {
     try {
       await this.connect()
-      const sheet = await this.client.sheetsByTitle[query.sheet]
+      const sheet = this.client.sheetsByTitle[query.sheet]
       const rows = await sheet.getRows()
       const headerValues = sheet.headerValues
       const response = []
@@ -368,7 +367,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   async update(query: { sheet: string; rowIndex: number; row: any }) {
     try {
       await this.connect()
-      const sheet = await this.client.sheetsByTitle[query.sheet]
+      const sheet = this.client.sheetsByTitle[query.sheet]
       const rows = await sheet.getRows()
       const row = rows[query.rowIndex]
       if (row) {
@@ -392,7 +391,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
   async delete(query: { sheet: string; rowIndex: number }) {
     await this.connect()
-    const sheet = await this.client.sheetsByTitle[query.sheet]
+    const sheet = this.client.sheetsByTitle[query.sheet]
     const rows = await sheet.getRows()
     const row = rows[query.rowIndex]
     if (row) {

package/src/integrations/tests/googlesheets.spec.ts

@@ -0,0 +1,122 @@
+import type { GoogleSpreadsheetWorksheet } from "google-spreadsheet"
+
+jest.mock("google-auth-library")
+const { OAuth2Client } = require("google-auth-library")
+
+const setCredentialsMock = jest.fn()
+const getAccessTokenMock = jest.fn()
+
+OAuth2Client.mockImplementation(() => {
+  return {
+    setCredentials: setCredentialsMock,
+    getAccessToken: getAccessTokenMock,
+  }
+})
+
+jest.mock("google-spreadsheet")
+const { GoogleSpreadsheet } = require("google-spreadsheet")
+
+const sheetsByTitle: { [title: string]: GoogleSpreadsheetWorksheet } = {}
+
+GoogleSpreadsheet.mockImplementation(() => {
+  return {
+    useOAuth2Client: jest.fn(),
+    loadInfo: jest.fn(),
+    sheetsByTitle,
+  }
+})
+
+import { structures } from "@budibase/backend-core/tests"
+import TestConfiguration from "../../tests/utilities/TestConfiguration"
+import GoogleSheetsIntegration from "../googlesheets"
+import { FieldType, Table, TableSchema } from "../../../../types/src/documents"
+
+describe("Google Sheets Integration", () => {
+  let integration: any,
+    config = new TestConfiguration()
+
+  beforeEach(async () => {
+    integration = new GoogleSheetsIntegration.integration({
+      spreadsheetId: "randomId",
+      auth: {
+        appId: "appId",
+        accessToken: "accessToken",
+        refreshToken: "refreshToken",
+      },
+    })
+    await config.init()
+  })
+
+  function createBasicTable(name: string, columns: string[]): Table {
+    return {
+      name,
+      schema: {
+        ...columns.reduce((p, c) => {
+          p[c] = {
+            name: c,
+            type: FieldType.STRING,
+            constraints: {
+              type: "string",
+            },
+          }
+          return p
+        }, {} as TableSchema),
+      },
+    }
+  }
+
+  function createSheet({
+    headerValues,
+  }: {
+    headerValues: string[]
+  }): GoogleSpreadsheetWorksheet {
+    return {
+      // to ignore the unmapped fields
+      ...({} as any),
+      loadHeaderRow: jest.fn(),
+      headerValues,
+      setHeaderRow: jest.fn(),
+    }
+  }
+
+  describe("update table", () => {
+    test("adding a new field will be adding a new header row", async () => {
+      await config.doInContext(structures.uuid(), async () => {
+        const tableColumns = ["name", "description", "new field"]
+        const table = createBasicTable(structures.uuid(), tableColumns)
+
+        const sheet = createSheet({ headerValues: ["name", "description"] })
+        sheetsByTitle[table.name] = sheet
+        await integration.updateTable(table)
+
+        expect(sheet.loadHeaderRow).toBeCalledTimes(1)
+        expect(sheet.setHeaderRow).toBeCalledTimes(1)
+        expect(sheet.setHeaderRow).toBeCalledWith(tableColumns)
+      })
+    })
+
+    test("removing an existing field will not remove the data from the spreadsheet", async () => {
+      await config.doInContext(structures.uuid(), async () => {
+        const tableColumns = ["name"]
+        const table = createBasicTable(structures.uuid(), tableColumns)
+
+        const sheet = createSheet({
+          headerValues: ["name", "description", "location"],
+        })
+        sheetsByTitle[table.name] = sheet
+        await integration.updateTable(table)
+
+        expect(sheet.loadHeaderRow).toBeCalledTimes(1)
+        expect(sheet.setHeaderRow).toBeCalledTimes(1)
+        expect(sheet.setHeaderRow).toBeCalledWith([
+          "name",
+          "description",
+          "location",
+        ])
+
+        // No undefineds are sent
+        expect((sheet.setHeaderRow as any).mock.calls[0][0]).toHaveLength(3)
+      })
+    })
+  })
+})

package/src/middleware/authorized.ts

@@ -79,10 +79,6 @@ export default (
       return ctx.throw(403, "No user info found")
     }
 
-    // check general builder stuff, this middleware is a good way
-    // to find API endpoints which are builder focused
-    await builderMiddleware(ctx, permType)
-
     // get the resource roles
     let resourceRoles: any = []
     let otherLevelRoles: any = []
@@ -112,6 +108,12 @@ export default (
       return ctx.throw(403, "Session not authenticated")
     }
 
+    // check general builder stuff, this middleware is a good way
+    // to find API endpoints which are builder focused
+    if (permType === permissions.PermissionType.BUILDER) {
+      await builderMiddleware(ctx)
+    }
+
     try {
       // check authorized
       await checkAuthorized(ctx, resourceRoles, permType, permLevel)

package/src/middleware/builder.ts

@@ -64,13 +64,18 @@ async function updateAppUpdatedAt(ctx: BBContext) {
   })
 }
 
-export default async function builder(ctx: BBContext, permType: string) {
+export default async function builder(ctx: BBContext) {
   const appId = ctx.appId
   // this only functions within an app context
   if (!appId) {
     return
   }
-  const isBuilderApi = permType === permissions.PermissionType.BUILDER
+
+  // check authenticated
+  if (!ctx.isAuthenticated) {
+    return ctx.throw(403, "Session not authenticated")
+  }
+
   const referer = ctx.headers["referer"]
 
   const overviewPath = "/builder/portal/overview/"
@@ -82,7 +87,7 @@ export default async function builder(ctx: BBContext, permType: string) {
   const hasAppId = !referer ? false : referer.includes(appId)
   const editingApp = referer ? hasAppId : false
   // check this is a builder call and editing
-  if (!isBuilderApi || !editingApp) {
+  if (!editingApp) {
     return
   }
   // check locks

package/src/migrations/functions/backfill/global/configs.ts

@@ -1,4 +1,9 @@
-import { events, db as dbUtils } from "@budibase/backend-core"
+import {
+  events,
+  DocumentType,
+  SEPARATOR,
+  UNICODE_MAX,
+} from "@budibase/backend-core"
 import {
   Config,
   isSMTPConfig,
@@ -9,15 +14,16 @@ import {
 } from "@budibase/types"
 import env from "./../../../../environment"
 
+export const getConfigParams = () => {
+  return {
+    include_docs: true,
+    startkey: `${DocumentType.CONFIG}${SEPARATOR}`,
+    endkey: `${DocumentType.CONFIG}${SEPARATOR}${UNICODE_MAX}`,
+  }
+}
+
 const getConfigs = async (globalDb: any): Promise<Config[]> => {
-  const response = await globalDb.allDocs(
-    dbUtils.getConfigParams(
-      {},
-      {
-        include_docs: true,
-      }
-    )
-  )
+  const response = await globalDb.allDocs(getConfigParams())
   return response.rows.map((row: any) => row.doc)
 }
 

package/src/migrations/functions/tests/userEmailViewCasing.spec.js

@@ -8,9 +8,8 @@ jest.mock("@budibase/backend-core", () => {
     }
   }
 })
-const { tenancy, db: dbCore } = require("@budibase/backend-core")
+const { context, db: dbCore } = require("@budibase/backend-core")
 const TestConfig = require("../../../tests/utilities/TestConfiguration")
-const { TENANT_ID } = require("../../../tests/utilities/structures")
 
 // mock email view creation
 
@@ -26,8 +25,8 @@ describe("run", () => {
     afterAll(config.end)
 
     it("runs successfully", async () => {
-      await tenancy.doInTenant(TENANT_ID, async () => {
-        const globalDb = tenancy.getGlobalDB()
+      await config.doInTenant(async () => {
+        const globalDb = context.getGlobalDB()
         await migration.run(globalDb)
         expect(dbCore.createNewUserEmailView).toHaveBeenCalledTimes(1)
       })

package/src/migrations/tests/helpers.ts

@@ -1,7 +1,7 @@
 // Mimic configs test configuration from worker, creation configs directly in database
 
 import * as structures from "./structures"
-import { db } from "@budibase/backend-core"
+import { configs } from "@budibase/backend-core"
 import { Config } from "@budibase/types"
 
 export const saveSettingsConfig = async (globalDb: any) => {
@@ -25,7 +25,7 @@ export const saveSmtpConfig = async (globalDb: any) => {
 }
 
 const saveConfig = async (config: Config, globalDb: any) => {
-  config._id = db.generateConfigID({ type: config.type })
+  config._id = configs.generateConfigID(config.type)
 
   let response
   try {

package/src/migrations/tests/structures.ts

@@ -20,6 +20,7 @@ export const oidc = (conf?: OIDCConfig): OIDCConfig => {
           name: "Active Directory",
           uuid: utils.newid(),
           activated: true,
+          scopes: [],
           ...conf,
         },
       ],

package/src/sdk/app/backups/constants.ts

@@ -1,2 +1,3 @@
 export const DB_EXPORT_FILE = "db.txt"
 export const GLOBAL_DB_EXPORT_FILE = "global.txt"
+export const STATIC_APP_FILES = ["manifest.json", "budibase-client.js"]

package/src/sdk/app/backups/exports.ts

@@ -7,10 +7,15 @@ import {
   TABLE_ROW_PREFIX,
   USER_METDATA_PREFIX,
 } from "../../../db/utils"
-import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
+import {
+  DB_EXPORT_FILE,
+  GLOBAL_DB_EXPORT_FILE,
+  STATIC_APP_FILES,
+} from "./constants"
 import fs from "fs"
 import { join } from "path"
 import env from "../../../environment"
+
 const uuid = require("uuid/v4")
 const tar = require("tar")
 const MemoryStream = require("memorystream")
@@ -91,14 +96,25 @@ export async function exportApp(appId: string, config?: ExportOpts) {
   const prodAppId = dbCore.getProdAppID(appId)
   const appPath = `${prodAppId}/`
   // export bucket contents
-  let tmpPath
+  let tmpPath = createTempFolder(uuid())
   if (!env.isTest()) {
-    tmpPath = await objectStore.retrieveDirectory(
-      ObjectStoreBuckets.APPS,
-      appPath
-    )
-  } else {
-    tmpPath = createTempFolder(uuid())
+    // write just the static files
+    if (config?.excludeRows) {
+      for (let path of STATIC_APP_FILES) {
+        const contents = await objectStore.retrieve(
+          ObjectStoreBuckets.APPS,
+          join(appPath, path)
+        )
+        fs.writeFileSync(join(tmpPath, path), contents)
+      }
+    }
+    // get all of the files
+    else {
+      tmpPath = await objectStore.retrieveDirectory(
+        ObjectStoreBuckets.APPS,
+        appPath
+      )
+    }
   }
   const downloadedPath = join(tmpPath, appPath)
   if (fs.existsSync(downloadedPath)) {

package/src/sdk/app/rows/attachments.ts

@@ -13,13 +13,13 @@ function generateAttachmentFindParams(
 ) {
   const params: CouchFindOptions = {
     selector: {
+      $or: attachmentCols.map(col => ({ [col]: { $exists: true } })),
       _id: {
         $regex: `^${DocumentType.ROW}${SEPARATOR}${tableId}`,
       },
     },
     limit: FIND_LIMIT,
   }
-  attachmentCols.forEach(col => (params.selector[col] = { $exists: true }))
   if (bookmark) {
     params.bookmark = bookmark
   }

package/src/startup.ts

@@ -5,7 +5,7 @@ import {
   generateApiKey,
   getChecklist,
 } from "./utilities/workerRequests"
-import { installation, tenancy, logging } from "@budibase/backend-core"
+import { installation, tenancy, logging, events } from "@budibase/backend-core"
 import fs from "fs"
 import { watch } from "./watch"
 import * as automations from "./automations"
@@ -124,6 +124,9 @@ export async function startup(app?: any, server?: any) {
   // get the references to the queue promises, don't await as
   // they will never end, unless the processing stops
   let queuePromises = []
+  // configure events to use the pro audit log write
+  // can't integrate directly into backend-core due to cyclic issues
+  queuePromises.push(events.processors.init(pro.sdk.auditLogs.write))
   queuePromises.push(automations.init())
   queuePromises.push(initPro())
   if (app) {

package/src/tests/jestEnv.ts

@@ -8,3 +8,4 @@ process.env.BUDIBASE_DIR = tmpdir("budibase-unittests")
 process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
 process.env.ENABLE_4XX_HTTP_LOGGING = "0"
 process.env.MOCK_REDIS = "1"
+process.env.PLATFORM_URL = "http://localhost:10000"

package/src/tests/utilities/TestConfiguration.ts

@@ -21,7 +21,6 @@ import {
   basicScreen,
   basicLayout,
   basicWebhook,
-  TENANT_ID,
 } from "./structures"
 import {
   constants,
@@ -41,8 +40,8 @@ import { generateUserMetadataID } from "../../db/utils"
 import { startup } from "../../startup"
 import supertest from "supertest"
 import {
+  App,
   AuthToken,
-  Database,
   Datasource,
   Row,
   SourceName,
@@ -63,7 +62,7 @@ class TestConfiguration {
   started: boolean
   appId: string | null
   allApps: any[]
-  app: any
+  app?: App
   prodApp: any
   prodAppId: any
   user: any
@@ -73,7 +72,7 @@ class TestConfiguration {
   linkedTable: any
   automation: any
   datasource: any
-  tenantId: string | null
+  tenantId?: string
   defaultUserValues: DefaultUserValues
 
   constructor(openServer = true) {
@@ -89,7 +88,6 @@ class TestConfiguration {
     }
     this.appId = null
     this.allApps = []
-    this.tenantId = null
     this.defaultUserValues = this.populateDefaultUserValues()
   }
 
@@ -154,19 +152,10 @@ class TestConfiguration {
 
   // use a new id as the name to avoid name collisions
   async init(appName = newid()) {
-    this.defaultUserValues = this.populateDefaultUserValues()
-    if (context.isMultiTenant()) {
-      this.tenantId = structures.tenant.id()
-    }
-
     if (!this.started) {
       await startup()
     }
-    this.user = await this.globalUser()
-    this.globalUserId = this.user._id
-    this.userMetadataId = generateUserMetadataID(this.globalUserId)
-
-    return this.createApp(appName)
+    return this.newTenant(appName)
   }
 
   end() {
@@ -182,24 +171,22 @@ class TestConfiguration {
   }
 
   // MODES
-  #setMultiTenancy = (value: boolean) => {
+  setMultiTenancy = (value: boolean) => {
     env._set("MULTI_TENANCY", value)
     coreEnv._set("MULTI_TENANCY", value)
   }
 
-  #setSelfHosted = (value: boolean) => {
+  setSelfHosted = (value: boolean) => {
     env._set("SELF_HOSTED", value)
     coreEnv._set("SELF_HOSTED", value)
   }
 
   modeCloud = () => {
-    this.#setSelfHosted(false)
-    this.#setMultiTenancy(true)
+    this.setSelfHosted(false)
   }
 
   modeSelf = () => {
-    this.#setSelfHosted(true)
-    this.#setMultiTenancy(false)
+    this.setSelfHosted(true)
   }
 
   // UTILS
@@ -354,6 +341,8 @@ class TestConfiguration {
     })
   }
 
+  // HEADERS
+
   defaultHeaders(extras = {}) {
     const tenantId = this.getTenantId()
     const authObj: AuthToken = {
@@ -374,6 +363,7 @@ class TestConfiguration {
         `${constants.Cookie.CurrentApp}=${appToken}`,
       ],
       [constants.Header.CSRF_TOKEN]: this.defaultUserValues.csrfToken,
+      Host: this.tenantHost(),
       ...extras,
     }
 
@@ -383,10 +373,6 @@ class TestConfiguration {
     return headers
   }
 
-  getTenantId() {
-    return this.tenantId || TENANT_ID
-  }
-
   publicHeaders({ prodApp = true } = {}) {
     const appId = prodApp ? this.prodAppId : this.appId
 
@@ -397,9 +383,7 @@ class TestConfiguration {
       headers[constants.Header.APP_ID] = appId
     }
 
-    if (this.tenantId) {
-      headers[constants.Header.TENANT_ID] = this.tenantId
-    }
+    headers[constants.Header.TENANT_ID] = this.getTenantId()
 
     return headers
   }
@@ -413,6 +397,34 @@ class TestConfiguration {
     return this.login({ email, roleId, builder, prodApp })
   }
 
+  // TENANCY
+
+  tenantHost() {
+    const tenantId = this.getTenantId()
+    const platformHost = new URL(coreEnv.PLATFORM_URL).host.split(":")[0]
+    return `${tenantId}.${platformHost}`
+  }
+
+  getTenantId() {
+    if (!this.tenantId) {
+      throw new Error("no test tenant id - init has not been called")
+    }
+    return this.tenantId
+  }
+
+  async newTenant(appName = newid()): Promise<App> {
+    this.defaultUserValues = this.populateDefaultUserValues()
+    this.tenantId = structures.tenant.id()
+    this.user = await this.globalUser()
+    this.globalUserId = this.user._id
+    this.userMetadataId = generateUserMetadataID(this.globalUserId)
+    return this.createApp(appName)
+  }
+
+  doInTenant(task: any) {
+    return context.doInTenant(this.getTenantId(), task)
+  }
+
   // API
 
   async generateApiKey(userId = this.defaultUserValues.globalUserId) {
@@ -432,7 +444,7 @@ class TestConfiguration {
   }
 
   // APP
-  async createApp(appName: string) {
+  async createApp(appName: string): Promise<App> {
     // create dev app
     // clear any old app
     this.appId = null
@@ -442,7 +454,7 @@ class TestConfiguration {
         null,
         controllers.app.create
       )
-      this.appId = this.app.appId
+      this.appId = this.app?.appId!
     })
     return await context.doInAppContext(this.appId, async () => {
       // create production app

package/src/tests/utilities/structures.ts

@@ -13,8 +13,6 @@ import {
 
 const { v4: uuidv4 } = require("uuid")
 
-export const TENANT_ID = "default"
-
 export function basicTable() {
   return {
     name: "TestTable",