@budibase/server 2.7.21-alpha.3 → 2.7.23

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.7.21-alpha.3",
+  "version": "2.7.23",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -46,12 +46,12 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.7.21-alpha.3",
-    "@budibase/client": "2.7.21-alpha.3",
-    "@budibase/pro": "2.7.21-alpha.3",
-    "@budibase/shared-core": "2.7.21-alpha.3",
-    "@budibase/string-templates": "2.7.21-alpha.3",
-    "@budibase/types": "2.7.21-alpha.3",
+    "@budibase/backend-core": "2.7.23",
+    "@budibase/client": "2.7.23",
+    "@budibase/pro": "2.7.23",
+    "@budibase/shared-core": "2.7.23",
+    "@budibase/string-templates": "2.7.23",
+    "@budibase/types": "2.7.23",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -97,7 +97,7 @@
     "koa2-ratelimit": "1.1.1",
     "lodash": "4.17.21",
     "memorystream": "0.3.1",
-    "mongodb": "5.6",
+    "mongodb": "4.9",
     "mssql": "6.2.3",
     "mysql2": "2.3.3",
     "node-fetch": "2.6.7",
@@ -117,7 +117,7 @@
     "socket.io": "4.6.1",
     "svelte": "3.49.0",
     "swagger-parser": "10.0.3",
-    "tar": "6.1.15",
+    "tar": "6.1.11",
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
@@ -150,7 +150,7 @@
     "@types/redis": "4.0.11",
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.12",
-    "@types/tar": "6.1.5",
+    "@types/tar": "6.1.3",
     "@typescript-eslint/parser": "5.45.0",
     "apidoc": "0.50.4",
     "babel-jest": "29.5.0",
@@ -195,5 +195,5 @@
       }
     }
   },
-  "gitHead": "4c79ad23a8dac8fe3fd165602fae40db1d256fb5"
+  "gitHead": "807e9a80d749b8beac7fc57d38499b59036fdeb7"
 }

package/src/api/controllers/backup.ts

@@ -1,31 +1,17 @@
 import sdk from "../../sdk"
-import { events, context, db } from "@budibase/backend-core"
+import { events, context } from "@budibase/backend-core"
 import { DocumentType } from "../../db/utils"
-import { Ctx } from "@budibase/types"
-
-interface ExportAppDumpRequest {
-  excludeRows: boolean
-  encryptPassword?: string
-}
-
-export async function exportAppDump(ctx: Ctx<ExportAppDumpRequest>) {
-  const { appId } = ctx.query as any
-  const { excludeRows, encryptPassword } = ctx.request.body
-
-  const [app] = await db.getAppsByIDs([appId])
-  const appName = app.name
+import { isQsTrue } from "../../utilities"
 
+export async function exportAppDump(ctx: any) {
+  let { appId, excludeRows } = ctx.query
   // remove the 120 second limit for the request
   ctx.req.setTimeout(0)
-
-  const extension = encryptPassword ? "enc.tar.gz" : "tar.gz"
-  const backupIdentifier = `${appName}-export-${new Date().getTime()}.${extension}`
+  const appName = decodeURI(ctx.query.appname)
+  excludeRows = isQsTrue(excludeRows)
+  const backupIdentifier = `${appName}-export-${new Date().getTime()}.tar.gz`
   ctx.attachment(backupIdentifier)
-  ctx.body = await sdk.backups.streamExportApp({
-    appId,
-    excludeRows,
-    encryptPassword,
-  })
+  ctx.body = await sdk.backups.streamExportApp(appId, excludeRows)
 
   await context.doInAppContext(appId, async () => {
     const appDb = context.getAppDB()

package/src/api/controllers/datasource.ts

@@ -11,7 +11,7 @@ import { BuildSchemaErrors, InvalidColumns } from "../../constants"
 import { getIntegration } from "../../integrations"
 import { getDatasourceAndQuery } from "./row/utils"
 import { invalidateDynamicVariables } from "../../threads/utils"
-import { db as dbCore, context, events, cache } from "@budibase/backend-core"
+import { db as dbCore, context, events } from "@budibase/backend-core"
 import {
   UserCtx,
   Datasource,
@@ -25,11 +25,9 @@ import {
   FetchDatasourceInfoResponse,
   IntegrationBase,
   DatasourcePlus,
-  SourceName,
 } from "@budibase/types"
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
-import { setupCreationAuth as googleSetupCreationAuth } from "../../integrations/googlesheets"
 
 function getErrorTables(errors: any, errorType: string) {
   return Object.entries(errors)
@@ -103,22 +101,6 @@ async function buildSchemaHelper(datasource: Datasource) {
   return { tables: connector.tables, error }
 }
 
-async function buildFilteredSchema(datasource: Datasource, filter?: string[]) {
-  let { tables, error } = await buildSchemaHelper(datasource)
-  let finalTables = tables
-  if (filter) {
-    finalTables = {}
-    for (let key in tables) {
-      if (
-        filter.some((filter: any) => filter.toLowerCase() === key.toLowerCase())
-      ) {
-        finalTables[key] = tables[key]
-      }
-    }
-  }
-  return { tables: finalTables, error }
-}
-
 export async function fetch(ctx: UserCtx) {
   // Get internal tables
   const db = context.getAppDB()
@@ -190,28 +172,43 @@ export async function information(
   }
   const tableNames = await connector.getTableNames()
   ctx.body = {
-    tableNames: tableNames.sort(),
+    tableNames,
   }
 }
 
 export async function buildSchemaFromDb(ctx: UserCtx) {
   const db = context.getAppDB()
-  const tablesFilter = ctx.request.body.tablesFilter
   const datasource = await sdk.datasources.get(ctx.params.datasourceId)
+  const tablesFilter = ctx.request.body.tablesFilter
 
-  const { tables, error } = await buildFilteredSchema(datasource, tablesFilter)
-  datasource.entities = tables
+  let { tables, error } = await buildSchemaHelper(datasource)
+  if (tablesFilter) {
+    if (!datasource.entities) {
+      datasource.entities = {}
+    }
+    for (let key in tables) {
+      if (
+        tablesFilter.some(
+          (filter: any) => filter.toLowerCase() === key.toLowerCase()
+        )
+      ) {
+        datasource.entities[key] = tables[key]
+      }
+    }
+  } else {
+    datasource.entities = tables
+  }
 
   setDefaultDisplayColumns(datasource)
   const dbResp = await db.put(datasource)
   datasource._rev = dbResp.rev
   const cleanedDatasource = await sdk.datasources.removeSecretSingle(datasource)
 
-  const res: any = { datasource: cleanedDatasource }
+  const response: any = { datasource: cleanedDatasource }
   if (error) {
-    res.error = error
+    response.error = error
   }
-  ctx.body = res
+  ctx.body = response
 }
 
 /**
@@ -309,19 +306,12 @@ export async function update(ctx: UserCtx<any, UpdateDatasourceResponse>) {
   builderSocket?.emitDatasourceUpdate(ctx, datasource)
 }
 
-const preSaveAction: Partial<Record<SourceName, any>> = {
-  [SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => {
-    await googleSetupCreationAuth(datasource.config as any)
-  },
-}
-
 export async function save(
   ctx: UserCtx<CreateDatasourceRequest, CreateDatasourceResponse>
 ) {
   const db = context.getAppDB()
   const plus = ctx.request.body.datasource.plus
   const fetchSchema = ctx.request.body.fetchSchema
-  const tablesFilter = ctx.request.body.tablesFilter
 
   const datasource = {
     _id: generateDatasourceID({ plus }),
@@ -331,19 +321,12 @@ export async function save(
 
   let schemaError = null
   if (fetchSchema) {
-    const { tables, error } = await buildFilteredSchema(
-      datasource,
-      tablesFilter
-    )
+    const { tables, error } = await buildSchemaHelper(datasource)
     schemaError = error
     datasource.entities = tables
     setDefaultDisplayColumns(datasource)
   }
 
-  if (preSaveAction[datasource.source]) {
-    await preSaveAction[datasource.source](datasource)
-  }
-
   const dbResp = await db.put(datasource)
   await events.datasource.created(datasource)
   datasource._rev = dbResp.rev

package/src/api/controllers/role.ts

@@ -4,7 +4,7 @@ import {
   getUserMetadataParams,
   InternalTables,
 } from "../../db/utils"
-import { UserCtx, Database } from "@budibase/types"
+import { BBContext, Database } from "@budibase/types"
 
 const UpdateRolesOptions = {
   CREATED: "created",
@@ -38,15 +38,15 @@ async function updateRolesOnUserTable(
   }
 }
 
-export async function fetch(ctx: UserCtx) {
+export async function fetch(ctx: BBContext) {
   ctx.body = await roles.getAllRoles()
 }
 
-export async function find(ctx: UserCtx) {
+export async function find(ctx: BBContext) {
   ctx.body = await roles.getRole(ctx.params.roleId)
 }
 
-export async function save(ctx: UserCtx) {
+export async function save(ctx: BBContext) {
   const db = context.getAppDB()
   let { _id, name, inherits, permissionId } = ctx.request.body
   let isCreate = false
@@ -72,7 +72,7 @@ export async function save(ctx: UserCtx) {
   ctx.message = `Role '${role.name}' created successfully.`
 }
 
-export async function destroy(ctx: UserCtx) {
+export async function destroy(ctx: BBContext) {
   const db = context.getAppDB()
   const roleId = ctx.params.roleId
   const role = await db.get(roleId)

package/src/api/controllers/routing.ts

@@ -1,6 +1,6 @@
 import { getRoutingInfo } from "../../utilities/routing"
 import { roles } from "@budibase/backend-core"
-import { UserCtx } from "@budibase/types"
+import { BBContext } from "@budibase/types"
 
 const URL_SEPARATOR = "/"
 
@@ -56,11 +56,11 @@ async function getRoutingStructure() {
   return { routes: routing.json }
 }
 
-export async function fetch(ctx: UserCtx) {
+export async function fetch(ctx: BBContext) {
   ctx.body = await getRoutingStructure()
 }
 
-export async function clientFetch(ctx: UserCtx) {
+export async function clientFetch(ctx: BBContext) {
   const routing = await getRoutingStructure()
   let roleId = ctx.user?.role?._id
   const roleIds = (await roles.getUserRoleHierarchy(roleId, {

package/src/api/routes/backup.ts

@@ -5,7 +5,7 @@ import { permissions } from "@budibase/backend-core"
 
 const router: Router = new Router()
 
-router.post(
+router.get(
   "/api/backups/export",
   authorized(permissions.BUILDER),
   controller.exportAppDump

package/src/api/routes/tests/backup.spec.ts

@@ -1,9 +1,7 @@
-import tk from "timekeeper"
 import * as setup from "./utilities"
 import { events } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
-import { mocks } from "@budibase/backend-core/tests"
 
 describe("/backups", () => {
   let request = setup.getRequest()
@@ -18,7 +16,7 @@ describe("/backups", () => {
   describe("exportAppDump", () => {
     it("should be able to export app", async () => {
       const res = await request
-        .post(`/api/backups/export?appId=${config.getAppId()}`)
+        .get(`/api/backups/export?appId=${config.getAppId()}&appname=test`)
         .set(config.defaultHeaders())
         .expect(200)
       expect(res.headers["content-type"]).toEqual("application/gzip")
@@ -28,24 +26,10 @@ describe("/backups", () => {
     it("should apply authorization to endpoint", async () => {
       await checkBuilderEndpoint({
         config,
-        method: "POST",
+        method: "GET",
         url: `/api/backups/export?appId=${config.getAppId()}`,
       })
     })
-
-    it("should infer the app name from the app", async () => {
-      tk.freeze(mocks.date.MOCK_DATE)
-
-      const res = await request
-        .post(`/api/backups/export?appId=${config.getAppId()}`)
-        .set(config.defaultHeaders())
-
-      expect(res.headers["content-disposition"]).toEqual(
-        `attachment; filename="${
-          config.getApp()!.name
-        }-export-${mocks.date.MOCK_DATE.getTime()}.tar.gz"`
-      )
-    })
   })
 
   describe("calculateBackupStats", () => {

package/src/automations/steps/sendSmtpEmail.ts

@@ -48,35 +48,6 @@ export const definition: AutomationStepSchema = {
           type: AutomationIOType.STRING,
           title: "HTML Contents",
         },
-        addInvite: {
-          type: AutomationIOType.BOOLEAN,
-          title: "Add calendar invite",
-        },
-        startTime: {
-          type: AutomationIOType.DATE,
-          title: "Start Time",
-          dependsOn: "addInvite",
-        },
-        endTime: {
-          type: AutomationIOType.DATE,
-          title: "End Time",
-          dependsOn: "addInvite",
-        },
-        summary: {
-          type: AutomationIOType.STRING,
-          title: "Meeting Summary",
-          dependsOn: "addInvite",
-        },
-        location: {
-          type: AutomationIOType.STRING,
-          title: "Location",
-          dependsOn: "addInvite",
-        },
-        url: {
-          type: AutomationIOType.STRING,
-          title: "URL",
-          dependsOn: "addInvite",
-        },
       },
       required: ["to", "from", "subject", "contents"],
     },
@@ -97,43 +68,21 @@ export const definition: AutomationStepSchema = {
 }
 
 export async function run({ inputs }: AutomationStepInput) {
-  let {
-    to,
-    from,
-    subject,
-    contents,
-    cc,
-    bcc,
-    addInvite,
-    startTime,
-    endTime,
-    summary,
-    location,
-    url,
-  } = inputs
+  let { to, from, subject, contents, cc, bcc } = inputs
   if (!contents) {
     contents = "<h1>No content</h1>"
   }
   to = to || undefined
   try {
-    let response = await sendSmtpEmail({
+    let response = await sendSmtpEmail(
       to,
       from,
       subject,
       contents,
       cc,
       bcc,
-      automation: true,
-      invite: addInvite
-        ? {
-            startTime,
-            endTime,
-            summary,
-            location,
-            url,
-          }
-        : undefined,
-    })
+      true
+    )
     return {
       success: true,
       response,

package/src/automations/tests/sendSmtpEmail.spec.js

@@ -0,0 +1,71 @@
+
+function generateResponse(to, from) {
+    return {
+        "success": true,
+        "response": {
+            "accepted": [
+                to
+            ],
+            "envelope": {
+                "from": from,
+                "to": [
+                    to
+                ]
+            },
+            "message": `Email sent to ${to}.`
+        }
+
+    }
+}
+
+const mockFetch = jest.fn(() => ({
+    headers: {
+        raw: () => {
+            return { "content-type": ["application/json"] }
+        },
+        get: () => ["application/json"],
+    },
+    json: jest.fn(() => response),
+    status: 200,
+    text: jest.fn(),
+}))
+jest.mock("node-fetch", () => mockFetch)
+const setup = require("./utilities")
+
+
+describe("test the outgoing webhook action", () => {
+    let inputs
+    let config = setup.getConfig()
+    beforeAll(async () => {
+        await config.init()
+    })
+
+    afterAll(setup.afterAll)
+
+    it("should be able to run the action", async () => {
+        inputs = {
+            to: "user1@test.com",
+            from: "admin@test.com",
+            subject: "hello",
+            contents: "testing",
+        }
+        let resp = generateResponse(inputs.to, inputs.from)
+        mockFetch.mockImplementationOnce(() => ({
+            headers: {
+                raw: () => {
+                    return { "content-type": ["application/json"] }
+                },
+                get: () => ["application/json"],
+            },
+            json: jest.fn(() => resp),
+            status: 200,
+            text: jest.fn(),
+        }))
+        const res = await setup.runStep(setup.actions.SEND_EMAIL_SMTP.stepId, inputs)
+        expect(res.response).toEqual(resp)
+        expect(res.success).toEqual(true)
+
+    })
+
+
+})

package/src/events/docUpdates/syncUsers.ts

@@ -26,10 +26,6 @@ export default function process(updateCb?: UpdateCallback) {
       // if something not found - no changes to perform
       if (err?.status === 404) {
         return
-      }
-      // The user has already been sync in another process
-      else if (err?.status === 409) {
-        return
       } else {
         logging.logAlert("Failed to perform user/group app sync", err)
       }

package/src/integrations/googlesheets.ts

@@ -1,6 +1,5 @@
 import {
   ConnectionInfo,
-  Datasource,
   DatasourceFeature,
   DatasourceFieldType,
   DatasourcePlus,
@@ -20,15 +19,13 @@ import { OAuth2Client } from "google-auth-library"
 import { buildExternalTableId, finaliseExternalTables } from "./utils"
 import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
 import fetch from "node-fetch"
-import { cache, configs, context, HTTPError } from "@budibase/backend-core"
-import { dataFilters, utils } from "@budibase/shared-core"
+import { configs, HTTPError } from "@budibase/backend-core"
+import { dataFilters } from "@budibase/shared-core"
 import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
-import sdk from "../sdk"
 
 interface GoogleSheetsConfig {
   spreadsheetId: string
   auth: OAuthClientConfig
-  continueSetupId?: string
 }
 
 interface OAuthClientConfig {
@@ -75,7 +72,7 @@ const SCHEMA: Integration = {
   },
   datasource: {
     spreadsheetId: {
-      display: "Spreadsheet URL",
+      display: "Google Sheet URL",
       type: DatasourceFieldType.STRING,
       required: true,
     },
@@ -210,8 +207,6 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
   async connect() {
     try {
-      await setupCreationAuth(this.config)
-
       // Initialise oAuth client
       let googleConfig = await configs.getGoogleDatasourceConfig()
       if (!googleConfig) {
@@ -274,24 +269,24 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   }
 
   async buildSchema(datasourceId: string, entities: Record<string, Table>) {
+    // not fully configured yet
+    if (!this.config.auth) {
+      return
+    }
     await this.connect()
     const sheets = this.client.sheetsByIndex
     const tables: Record<string, Table> = {}
-    await utils.parallelForeach(
-      sheets,
-      async sheet => {
-        // must fetch rows to determine schema
-        await sheet.getRows({ limit: 0, offset: 0 })
-
-        const id = buildExternalTableId(datasourceId, sheet.title)
-        tables[sheet.title] = this.getTableSchema(
-          sheet.title,
-          sheet.headerValues,
-          id
-        )
-      },
-      10
-    )
+    for (let sheet of sheets) {
+      // must fetch rows to determine schema
+      await sheet.getRows()
+
+      const id = buildExternalTableId(datasourceId, sheet.title)
+      tables[sheet.title] = this.getTableSchema(
+        sheet.title,
+        sheet.headerValues,
+        id
+      )
+    }
     const final = finaliseExternalTables(tables, entities)
     this.tables = final.tables
     this.schemaErrors = final.errors
@@ -571,18 +566,6 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   }
 }
 
-export async function setupCreationAuth(datasouce: GoogleSheetsConfig) {
-  if (datasouce.continueSetupId) {
-    const appId = context.getAppId()
-    const tokens = await cache.get(
-      `datasource:creation:${appId}:google:${datasouce.continueSetupId}`
-    )
-
-    datasouce.auth = tokens.tokens
-    delete datasouce.continueSetupId
-  }
-}
-
 export default {
   schema: SCHEMA,
   integration: GoogleSheetsIntegration,

package/src/integrations/mongodb.ts

@@ -351,7 +351,7 @@ const SCHEMA: Integration = getSchema()
 
 class MongoIntegration implements IntegrationBase {
   private config: MongoDBConfig
-  private client: MongoClient
+  private client: any
 
   constructor(config: MongoDBConfig) {
     this.config = config
@@ -372,8 +372,6 @@ class MongoIntegration implements IntegrationBase {
       response.connected = true
     } catch (e: any) {
       response.error = e.message as string
-    } finally {
-      await this.client.close()
     }
     return response
   }
@@ -382,7 +380,7 @@ class MongoIntegration implements IntegrationBase {
     return this.client.connect()
   }
 
-  createObjectIds(json: any) {
+  createObjectIds(json: any): object {
     const self = this
     function interpolateObjectIds(json: any) {
       for (let field of Object.keys(json)) {