@budibase/server 2.3.19 → 2.3.21-alpha.0

package/src/integrations/base/sql.ts

@@ -6,11 +6,11 @@ import {
   SearchFilters,
   SortDirection,
 } from "@budibase/types"
+import { db as dbCore } from "@budibase/backend-core"
 import { QueryOptions } from "../../definitions/datasource"
 import { isIsoDateString, SqlClient } from "../utils"
 import SqlTableQueryBuilder from "./sqlTable"
 import environment from "../../environment"
-import { removeKeyNumbering } from "./utils"
 
 const envLimit = environment.SQL_MAX_ROWS
   ? parseInt(environment.SQL_MAX_ROWS)
@@ -90,10 +90,15 @@ function parseFilters(filters: SearchFilters | undefined): SearchFilters {
 function generateSelectStatement(
   json: QueryJson,
   knex: Knex
-): (string | Knex.Raw)[] {
+): (string | Knex.Raw)[] | "*" {
   const { resource, meta } = json
+
+  if (!resource) {
+    return "*"
+  }
+
   const schema = meta?.table?.schema
-  return resource!.fields.map(field => {
+  return resource.fields.map(field => {
     const fieldNames = field.split(/\./g)
     const tableName = fieldNames[0]
     const columnName = fieldNames[1]
@@ -131,7 +136,7 @@ class InternalBuilder {
       fn: (key: string, value: any) => void
     ) {
       for (let [key, value] of Object.entries(structure)) {
-        const updatedKey = removeKeyNumbering(key)
+        const updatedKey = dbCore.removeKeyNumbering(key)
         const isRelationshipField = updatedKey.includes(".")
         if (!opts.relationship && !isRelationshipField) {
           fn(`${opts.tableName}.${updatedKey}`, value)
@@ -405,6 +410,7 @@ class InternalBuilder {
         delete parsedBody[key]
       }
     }
+
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.insert(parsedBody)
@@ -509,7 +515,7 @@ class InternalBuilder {
     if (opts.disableReturning) {
       return query.delete()
     } else {
-      return query.delete().returning("*")
+      return query.delete().returning(generateSelectStatement(json, knex))
     }
   }
 }

package/src/integrations/googlesheets.ts

@@ -11,9 +11,8 @@ import { OAuth2Client } from "google-auth-library"
 import { buildExternalTableId } from "./utils"
 import { DataSourceOperation, FieldTypes } from "../constants"
 import { GoogleSpreadsheet } from "google-spreadsheet"
-import env from "../environment"
-import { tenancy, db as dbCore, constants } from "@budibase/backend-core"
-const fetch = require("node-fetch")
+import fetch from "node-fetch"
+import { configs, HTTPError } from "@budibase/backend-core"
 
 interface GoogleSheetsConfig {
   spreadsheetId: string
@@ -112,7 +111,7 @@ const SCHEMA: Integration = {
 
 class GoogleSheetsIntegration implements DatasourcePlus {
   private readonly config: GoogleSheetsConfig
-  private client: any
+  private client: GoogleSpreadsheet
   public tables: Record<string, Table> = {}
   public schemaErrors: Record<string, string> = {}
 
@@ -173,16 +172,9 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   async connect() {
     try {
       // Initialise oAuth client
-      const db = tenancy.getGlobalDB()
-      let googleConfig = await dbCore.getScopedConfig(db, {
-        type: constants.Config.GOOGLE,
-      })
-
+      let googleConfig = await configs.getGoogleDatasourceConfig()
       if (!googleConfig) {
-        googleConfig = {
-          clientID: env.GOOGLE_CLIENT_ID,
-          clientSecret: env.GOOGLE_CLIENT_SECRET,
-        }
+        throw new HTTPError("Google config not found", 400)
       }
 
       const oauthClient = new OAuth2Client({
@@ -211,7 +203,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
   async buildSchema(datasourceId: string) {
     await this.connect()
-    const sheets = await this.client.sheetsByIndex
+    const sheets = this.client.sheetsByIndex
     const tables: Record<string, Table> = {}
     for (let sheet of sheets) {
       // must fetch rows to determine schema
@@ -294,7 +286,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   async updateTable(table?: any) {
     try {
       await this.connect()
-      const sheet = await this.client.sheetsByTitle[table.name]
+      const sheet = this.client.sheetsByTitle[table.name]
       await sheet.loadHeaderRow()
 
       if (table._rename) {
@@ -308,10 +300,17 @@ class GoogleSheetsIntegration implements DatasourcePlus {
         }
         await sheet.setHeaderRow(headers)
       } else {
-        let newField = Object.keys(table.schema).find(
+        const updatedHeaderValues = [...sheet.headerValues]
+
+        const newField = Object.keys(table.schema).find(
           key => !sheet.headerValues.includes(key)
         )
-        await sheet.setHeaderRow([...sheet.headerValues, newField])
+
+        if (newField) {
+          updatedHeaderValues.push(newField)
+        }
+
+        await sheet.setHeaderRow(updatedHeaderValues)
       }
     } catch (err) {
       console.error("Error updating table in google sheets", err)
@@ -322,7 +321,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   async deleteTable(sheet: any) {
     try {
       await this.connect()
-      const sheetToDelete = await this.client.sheetsByTitle[sheet]
+      const sheetToDelete = this.client.sheetsByTitle[sheet]
       return await sheetToDelete.delete()
     } catch (err) {
       console.error("Error deleting table in google sheets", err)
@@ -333,7 +332,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   async create(query: { sheet: string; row: any }) {
     try {
       await this.connect()
-      const sheet = await this.client.sheetsByTitle[query.sheet]
+      const sheet = this.client.sheetsByTitle[query.sheet]
       const rowToInsert =
         typeof query.row === "string" ? JSON.parse(query.row) : query.row
       const row = await sheet.addRow(rowToInsert)
@@ -349,7 +348,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   async read(query: { sheet: string }) {
     try {
       await this.connect()
-      const sheet = await this.client.sheetsByTitle[query.sheet]
+      const sheet = this.client.sheetsByTitle[query.sheet]
       const rows = await sheet.getRows()
       const headerValues = sheet.headerValues
       const response = []
@@ -368,7 +367,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   async update(query: { sheet: string; rowIndex: number; row: any }) {
     try {
       await this.connect()
-      const sheet = await this.client.sheetsByTitle[query.sheet]
+      const sheet = this.client.sheetsByTitle[query.sheet]
       const rows = await sheet.getRows()
       const row = rows[query.rowIndex]
       if (row) {
@@ -392,7 +391,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
   async delete(query: { sheet: string; rowIndex: number }) {
     await this.connect()
-    const sheet = await this.client.sheetsByTitle[query.sheet]
+    const sheet = this.client.sheetsByTitle[query.sheet]
     const rows = await sheet.getRows()
     const row = rows[query.rowIndex]
     if (row) {

package/src/integrations/oracle.ts

@@ -247,7 +247,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
     )
   }
 
-  private internalConvertType(column: OracleColumn): { type: string } {
+  private internalConvertType(column: OracleColumn): { type: FieldTypes } {
     if (this.isBooleanType(column)) {
       return { type: FieldTypes.BOOLEAN }
     }

package/src/integrations/tests/couchdb.spec.ts

@@ -1,5 +1,3 @@
-import { DatabaseWithConnection } from "@budibase/backend-core/src/db"
-
 jest.mock("@budibase/backend-core", () => {
   const core = jest.requireActual("@budibase/backend-core")
   return {

package/src/integrations/tests/googlesheets.spec.ts

@@ -0,0 +1,122 @@
+import type { GoogleSpreadsheetWorksheet } from "google-spreadsheet"
+
+jest.mock("google-auth-library")
+const { OAuth2Client } = require("google-auth-library")
+
+const setCredentialsMock = jest.fn()
+const getAccessTokenMock = jest.fn()
+
+OAuth2Client.mockImplementation(() => {
+  return {
+    setCredentials: setCredentialsMock,
+    getAccessToken: getAccessTokenMock,
+  }
+})
+
+jest.mock("google-spreadsheet")
+const { GoogleSpreadsheet } = require("google-spreadsheet")
+
+const sheetsByTitle: { [title: string]: GoogleSpreadsheetWorksheet } = {}
+
+GoogleSpreadsheet.mockImplementation(() => {
+  return {
+    useOAuth2Client: jest.fn(),
+    loadInfo: jest.fn(),
+    sheetsByTitle,
+  }
+})
+
+import { structures } from "@budibase/backend-core/tests"
+import TestConfiguration from "../../tests/utilities/TestConfiguration"
+import GoogleSheetsIntegration from "../googlesheets"
+import { FieldType, Table, TableSchema } from "../../../../types/src/documents"
+
+describe("Google Sheets Integration", () => {
+  let integration: any,
+    config = new TestConfiguration()
+
+  beforeEach(async () => {
+    integration = new GoogleSheetsIntegration.integration({
+      spreadsheetId: "randomId",
+      auth: {
+        appId: "appId",
+        accessToken: "accessToken",
+        refreshToken: "refreshToken",
+      },
+    })
+    await config.init()
+  })
+
+  function createBasicTable(name: string, columns: string[]): Table {
+    return {
+      name,
+      schema: {
+        ...columns.reduce((p, c) => {
+          p[c] = {
+            name: c,
+            type: FieldType.STRING,
+            constraints: {
+              type: "string",
+            },
+          }
+          return p
+        }, {} as TableSchema),
+      },
+    }
+  }
+
+  function createSheet({
+    headerValues,
+  }: {
+    headerValues: string[]
+  }): GoogleSpreadsheetWorksheet {
+    return {
+      // to ignore the unmapped fields
+      ...({} as any),
+      loadHeaderRow: jest.fn(),
+      headerValues,
+      setHeaderRow: jest.fn(),
+    }
+  }
+
+  describe("update table", () => {
+    test("adding a new field will be adding a new header row", async () => {
+      await config.doInContext(structures.uuid(), async () => {
+        const tableColumns = ["name", "description", "new field"]
+        const table = createBasicTable(structures.uuid(), tableColumns)
+
+        const sheet = createSheet({ headerValues: ["name", "description"] })
+        sheetsByTitle[table.name] = sheet
+        await integration.updateTable(table)
+
+        expect(sheet.loadHeaderRow).toBeCalledTimes(1)
+        expect(sheet.setHeaderRow).toBeCalledTimes(1)
+        expect(sheet.setHeaderRow).toBeCalledWith(tableColumns)
+      })
+    })
+
+    test("removing an existing field will not remove the data from the spreadsheet", async () => {
+      await config.doInContext(structures.uuid(), async () => {
+        const tableColumns = ["name"]
+        const table = createBasicTable(structures.uuid(), tableColumns)
+
+        const sheet = createSheet({
+          headerValues: ["name", "description", "location"],
+        })
+        sheetsByTitle[table.name] = sheet
+        await integration.updateTable(table)
+
+        expect(sheet.loadHeaderRow).toBeCalledTimes(1)
+        expect(sheet.setHeaderRow).toBeCalledTimes(1)
+        expect(sheet.setHeaderRow).toBeCalledWith([
+          "name",
+          "description",
+          "location",
+        ])
+
+        // No undefineds are sent
+        expect((sheet.setHeaderRow as any).mock.calls[0][0]).toHaveLength(3)
+      })
+    })
+  })
+})

package/src/middleware/authorized.ts

@@ -79,10 +79,6 @@ export default (
       return ctx.throw(403, "No user info found")
     }
 
-    // check general builder stuff, this middleware is a good way
-    // to find API endpoints which are builder focused
-    await builderMiddleware(ctx, permType)
-
     // get the resource roles
     let resourceRoles: any = []
     let otherLevelRoles: any = []
@@ -112,6 +108,12 @@ export default (
       return ctx.throw(403, "Session not authenticated")
     }
 
+    // check general builder stuff, this middleware is a good way
+    // to find API endpoints which are builder focused
+    if (permType === permissions.PermissionType.BUILDER) {
+      await builderMiddleware(ctx)
+    }
+
     try {
       // check authorized
       await checkAuthorized(ctx, resourceRoles, permType, permLevel)

package/src/middleware/builder.ts

@@ -64,13 +64,18 @@ async function updateAppUpdatedAt(ctx: BBContext) {
   })
 }
 
-export default async function builder(ctx: BBContext, permType: string) {
+export default async function builder(ctx: BBContext) {
   const appId = ctx.appId
   // this only functions within an app context
   if (!appId) {
     return
   }
-  const isBuilderApi = permType === permissions.PermissionType.BUILDER
+
+  // check authenticated
+  if (!ctx.isAuthenticated) {
+    return ctx.throw(403, "Session not authenticated")
+  }
+
   const referer = ctx.headers["referer"]
 
   const overviewPath = "/builder/portal/overview/"
@@ -82,7 +87,7 @@ export default async function builder(ctx: BBContext, permType: string) {
   const hasAppId = !referer ? false : referer.includes(appId)
   const editingApp = referer ? hasAppId : false
   // check this is a builder call and editing
-  if (!isBuilderApi || !editingApp) {
+  if (!editingApp) {
     return
   }
   // check locks

package/src/migrations/functions/backfill/global/configs.ts

@@ -1,4 +1,9 @@
-import { events, db as dbUtils } from "@budibase/backend-core"
+import {
+  events,
+  DocumentType,
+  SEPARATOR,
+  UNICODE_MAX,
+} from "@budibase/backend-core"
 import {
   Config,
   isSMTPConfig,
@@ -9,15 +14,16 @@ import {
 } from "@budibase/types"
 import env from "./../../../../environment"
 
+export const getConfigParams = () => {
+  return {
+    include_docs: true,
+    startkey: `${DocumentType.CONFIG}${SEPARATOR}`,
+    endkey: `${DocumentType.CONFIG}${SEPARATOR}${UNICODE_MAX}`,
+  }
+}
+
 const getConfigs = async (globalDb: any): Promise<Config[]> => {
-  const response = await globalDb.allDocs(
-    dbUtils.getConfigParams(
-      {},
-      {
-        include_docs: true,
-      }
-    )
-  )
+  const response = await globalDb.allDocs(getConfigParams())
   return response.rows.map((row: any) => row.doc)
 }
 

package/src/migrations/functions/tests/appUrls.spec.js

@@ -6,7 +6,7 @@ const migration = require("../appUrls")
 describe("run", () => {
   let config = new TestConfig(false)
 
-  beforeEach(async () => {
+  beforeAll(async () => {
     await config.init()
   })
 

package/src/migrations/functions/tests/syncQuotas.spec.js

@@ -11,7 +11,7 @@ const migration = require("../syncQuotas")
 describe("run", () => {
   let config = new TestConfig(false)
 
-  beforeEach(async () => {
+  beforeAll(async () => {
     await config.init()
   })
 

package/src/migrations/functions/tests/userEmailViewCasing.spec.js

@@ -8,9 +8,8 @@ jest.mock("@budibase/backend-core", () => {
     }
   }
 })
-const { tenancy, db: dbCore } = require("@budibase/backend-core")
+const { context, db: dbCore } = require("@budibase/backend-core")
 const TestConfig = require("../../../tests/utilities/TestConfiguration")
-const { TENANT_ID } = require("../../../tests/utilities/structures")
 
 // mock email view creation
 
@@ -19,15 +18,15 @@ const migration = require("../userEmailViewCasing")
 describe("run", () => {
     let config = new TestConfig(false)
 
-    beforeEach(async () => {
+    beforeAll(async () => {
       await config.init()
     })
 
     afterAll(config.end)
 
     it("runs successfully", async () => {
-      await tenancy.doInTenant(TENANT_ID, async () => {
-        const globalDb = tenancy.getGlobalDB()
+      await config.doInTenant(async () => {
+        const globalDb = context.getGlobalDB()
         await migration.run(globalDb)
         expect(dbCore.createNewUserEmailView).toHaveBeenCalledTimes(1)
       })

package/src/migrations/tests/helpers.ts

@@ -1,7 +1,7 @@
 // Mimic configs test configuration from worker, creation configs directly in database
 
 import * as structures from "./structures"
-import { db } from "@budibase/backend-core"
+import { configs } from "@budibase/backend-core"
 import { Config } from "@budibase/types"
 
 export const saveSettingsConfig = async (globalDb: any) => {
@@ -25,7 +25,7 @@ export const saveSmtpConfig = async (globalDb: any) => {
 }
 
 const saveConfig = async (config: Config, globalDb: any) => {
-  config._id = db.generateConfigID({ type: config.type })
+  config._id = configs.generateConfigID(config.type)
 
   let response
   try {

package/src/migrations/tests/index.spec.ts

@@ -10,8 +10,9 @@ import * as structures from "../../tests/utilities/structures"
 import { MIGRATIONS } from "../"
 import * as helpers from "./helpers"
 
-const { mocks } = require("@budibase/backend-core/tests")
-const timestamp = mocks.date.MOCK_DATE.toISOString()
+import tk from "timekeeper"
+const timestamp = new Date().toISOString()
+tk.freeze(timestamp)
 
 const clearMigrations = async () => {
   const dbs = [context.getDevAppDB(), context.getProdAppDB()]
@@ -92,24 +93,16 @@ describe("migrations", () => {
       await clearMigrations()
       const appId = config.prodAppId
       const roles = { [appId]: "role_12345" }
-      await config.createUser(
-        undefined,
-        undefined,
-        undefined,
-        undefined,
-        false,
-        true,
-        roles
-      ) // admin only
-      await config.createUser(
-        undefined,
-        undefined,
-        undefined,
-        undefined,
-        false,
-        false,
-        roles
-      ) // non admin non builder
+      await config.createUser({
+        builder: false,
+        admin: true,
+        roles,
+      }) // admin only
+      await config.createUser({
+        builder: false,
+        admin: false,
+        roles,
+      }) // non admin non builder
       await config.createTable()
       await config.createRow()
       await config.createRow()

package/src/migrations/tests/structures.ts

@@ -20,6 +20,7 @@ export const oidc = (conf?: OIDCConfig): OIDCConfig => {
           name: "Active Directory",
           uuid: utils.newid(),
           activated: true,
+          scopes: [],
           ...conf,
         },
       ],

package/src/sdk/app/backups/constants.ts

@@ -1,2 +1,3 @@
 export const DB_EXPORT_FILE = "db.txt"
 export const GLOBAL_DB_EXPORT_FILE = "global.txt"
+export const STATIC_APP_FILES = ["manifest.json", "budibase-client.js"]

package/src/sdk/app/backups/exports.ts

@@ -7,10 +7,15 @@ import {
   TABLE_ROW_PREFIX,
   USER_METDATA_PREFIX,
 } from "../../../db/utils"
-import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
+import {
+  DB_EXPORT_FILE,
+  GLOBAL_DB_EXPORT_FILE,
+  STATIC_APP_FILES,
+} from "./constants"
 import fs from "fs"
 import { join } from "path"
 import env from "../../../environment"
+
 const uuid = require("uuid/v4")
 const tar = require("tar")
 const MemoryStream = require("memorystream")
@@ -45,12 +50,18 @@ function tarFilesToTmp(tmpDir: string, files: string[]) {
  * @return {*} either a readable stream or a string
  */
 export async function exportDB(dbName: string, opts: ExportOpts = {}) {
+  const exportOpts = {
+    filter: opts?.filter,
+    batch_size: 1000,
+    batch_limit: 5,
+    style: "main_only",
+  }
   return dbCore.doWithDB(dbName, async (db: any) => {
     // Write the dump to file if required
     if (opts?.exportPath) {
       const path = opts?.exportPath
       const writeStream = fs.createWriteStream(path)
-      await db.dump(writeStream, { filter: opts?.filter })
+      await db.dump(writeStream, exportOpts)
       return path
     } else {
       // Stringify the dump in memory if required
@@ -59,7 +70,7 @@ export async function exportDB(dbName: string, opts: ExportOpts = {}) {
       memStream.on("data", (chunk: any) => {
         appString += chunk.toString()
       })
-      await db.dump(memStream, { filter: opts?.filter })
+      await db.dump(memStream, exportOpts)
       return appString
     }
   })
@@ -85,14 +96,25 @@ export async function exportApp(appId: string, config?: ExportOpts) {
   const prodAppId = dbCore.getProdAppID(appId)
   const appPath = `${prodAppId}/`
   // export bucket contents
-  let tmpPath
+  let tmpPath = createTempFolder(uuid())
   if (!env.isTest()) {
-    tmpPath = await objectStore.retrieveDirectory(
-      ObjectStoreBuckets.APPS,
-      appPath
-    )
-  } else {
-    tmpPath = createTempFolder(uuid())
+    // write just the static files
+    if (config?.excludeRows) {
+      for (let path of STATIC_APP_FILES) {
+        const contents = await objectStore.retrieve(
+          ObjectStoreBuckets.APPS,
+          join(appPath, path)
+        )
+        fs.writeFileSync(join(tmpPath, path), contents)
+      }
+    }
+    // get all of the files
+    else {
+      tmpPath = await objectStore.retrieveDirectory(
+        ObjectStoreBuckets.APPS,
+        appPath
+      )
+    }
   }
   const downloadedPath = join(tmpPath, appPath)
   if (fs.existsSync(downloadedPath)) {

package/src/sdk/app/datasources/datasources.ts

@@ -88,6 +88,9 @@ export async function removeSecrets(datasources: Datasource[]) {
   const definitions = await getDefinitions()
   for (let datasource of datasources) {
     const schema = definitions[datasource.source]
+    if (!schema) {
+      continue
+    }
     if (datasource.config) {
       // strip secrets from response, so they don't show in the network request
       if (datasource.config.auth) {

package/src/sdk/app/rows/attachments.ts

@@ -13,13 +13,13 @@ function generateAttachmentFindParams(
 ) {
   const params: CouchFindOptions = {
     selector: {
+      $or: attachmentCols.map(col => ({ [col]: { $exists: true } })),
       _id: {
         $regex: `^${DocumentType.ROW}${SEPARATOR}${tableId}`,
       },
     },
     limit: FIND_LIMIT,
   }
-  attachmentCols.forEach(col => (params.selector[col] = { $exists: true }))
   if (bookmark) {
     params.bookmark = bookmark
   }

package/src/startup.ts

@@ -5,7 +5,7 @@ import {
   generateApiKey,
   getChecklist,
 } from "./utilities/workerRequests"
-import { installation, tenancy, logging } from "@budibase/backend-core"
+import { installation, tenancy, logging, events } from "@budibase/backend-core"
 import fs from "fs"
 import { watch } from "./watch"
 import * as automations from "./automations"
@@ -124,6 +124,9 @@ export async function startup(app?: any, server?: any) {
   // get the references to the queue promises, don't await as
   // they will never end, unless the processing stops
   let queuePromises = []
+  // configure events to use the pro audit log write
+  // can't integrate directly into backend-core due to cyclic issues
+  queuePromises.push(events.processors.init(pro.sdk.auditLogs.write))
   queuePromises.push(automations.init())
   queuePromises.push(initPro())
   if (app) {