@budibase/server 2.7.23 → 2.7.25-alpha.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.7.23",
+  "version": "2.7.25-alpha.0",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -46,12 +46,12 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.7.23",
-    "@budibase/client": "2.7.23",
-    "@budibase/pro": "2.7.23",
-    "@budibase/shared-core": "2.7.23",
-    "@budibase/string-templates": "2.7.23",
-    "@budibase/types": "2.7.23",
+    "@budibase/backend-core": "2.7.25-alpha.0",
+    "@budibase/client": "2.7.25-alpha.0",
+    "@budibase/pro": "2.7.25-alpha.0",
+    "@budibase/shared-core": "2.7.25-alpha.0",
+    "@budibase/string-templates": "2.7.25-alpha.0",
+    "@budibase/types": "2.7.25-alpha.0",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -97,7 +97,7 @@
     "koa2-ratelimit": "1.1.1",
     "lodash": "4.17.21",
     "memorystream": "0.3.1",
-    "mongodb": "4.9",
+    "mongodb": "5.6",
     "mssql": "6.2.3",
     "mysql2": "2.3.3",
     "node-fetch": "2.6.7",
@@ -117,7 +117,7 @@
     "socket.io": "4.6.1",
     "svelte": "3.49.0",
     "swagger-parser": "10.0.3",
-    "tar": "6.1.11",
+    "tar": "6.1.15",
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
@@ -150,7 +150,7 @@
     "@types/redis": "4.0.11",
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.12",
-    "@types/tar": "6.1.3",
+    "@types/tar": "6.1.5",
     "@typescript-eslint/parser": "5.45.0",
     "apidoc": "0.50.4",
     "babel-jest": "29.5.0",
@@ -195,5 +195,5 @@
       }
     }
   },
-  "gitHead": "807e9a80d749b8beac7fc57d38499b59036fdeb7"
+  "gitHead": "8ecb20caca227224ab8bfd29f92b99ab7077a9c0"
 }

package/src/api/controllers/backup.ts

@@ -1,17 +1,31 @@
 import sdk from "../../sdk"
-import { events, context } from "@budibase/backend-core"
+import { events, context, db } from "@budibase/backend-core"
 import { DocumentType } from "../../db/utils"
-import { isQsTrue } from "../../utilities"
+import { Ctx } from "@budibase/types"
+
+interface ExportAppDumpRequest {
+  excludeRows: boolean
+  encryptPassword?: string
+}
+
+export async function exportAppDump(ctx: Ctx<ExportAppDumpRequest>) {
+  const { appId } = ctx.query as any
+  const { excludeRows, encryptPassword } = ctx.request.body
+
+  const [app] = await db.getAppsByIDs([appId])
+  const appName = app.name
 
-export async function exportAppDump(ctx: any) {
-  let { appId, excludeRows } = ctx.query
   // remove the 120 second limit for the request
   ctx.req.setTimeout(0)
-  const appName = decodeURI(ctx.query.appname)
-  excludeRows = isQsTrue(excludeRows)
-  const backupIdentifier = `${appName}-export-${new Date().getTime()}.tar.gz`
+
+  const extension = encryptPassword ? "enc.tar.gz" : "tar.gz"
+  const backupIdentifier = `${appName}-export-${new Date().getTime()}.${extension}`
   ctx.attachment(backupIdentifier)
-  ctx.body = await sdk.backups.streamExportApp(appId, excludeRows)
+  ctx.body = await sdk.backups.streamExportApp({
+    appId,
+    excludeRows,
+    encryptPassword,
+  })
 
   await context.doInAppContext(appId, async () => {
     const appDb = context.getAppDB()

package/src/api/controllers/datasource.ts

@@ -11,7 +11,7 @@ import { BuildSchemaErrors, InvalidColumns } from "../../constants"
 import { getIntegration } from "../../integrations"
 import { getDatasourceAndQuery } from "./row/utils"
 import { invalidateDynamicVariables } from "../../threads/utils"
-import { db as dbCore, context, events } from "@budibase/backend-core"
+import { db as dbCore, context, events, cache } from "@budibase/backend-core"
 import {
   UserCtx,
   Datasource,
@@ -25,9 +25,11 @@ import {
   FetchDatasourceInfoResponse,
   IntegrationBase,
   DatasourcePlus,
+  SourceName,
 } from "@budibase/types"
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
+import { setupCreationAuth as googleSetupCreationAuth } from "../../integrations/googlesheets"
 
 function getErrorTables(errors: any, errorType: string) {
   return Object.entries(errors)
@@ -101,6 +103,22 @@ async function buildSchemaHelper(datasource: Datasource) {
   return { tables: connector.tables, error }
 }
 
+async function buildFilteredSchema(datasource: Datasource, filter?: string[]) {
+  let { tables, error } = await buildSchemaHelper(datasource)
+  let finalTables = tables
+  if (filter) {
+    finalTables = {}
+    for (let key in tables) {
+      if (
+        filter.some((filter: any) => filter.toLowerCase() === key.toLowerCase())
+      ) {
+        finalTables[key] = tables[key]
+      }
+    }
+  }
+  return { tables: finalTables, error }
+}
+
 export async function fetch(ctx: UserCtx) {
   // Get internal tables
   const db = context.getAppDB()
@@ -172,43 +190,28 @@ export async function information(
   }
   const tableNames = await connector.getTableNames()
   ctx.body = {
-    tableNames,
+    tableNames: tableNames.sort(),
   }
 }
 
 export async function buildSchemaFromDb(ctx: UserCtx) {
   const db = context.getAppDB()
-  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
   const tablesFilter = ctx.request.body.tablesFilter
+  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
 
-  let { tables, error } = await buildSchemaHelper(datasource)
-  if (tablesFilter) {
-    if (!datasource.entities) {
-      datasource.entities = {}
-    }
-    for (let key in tables) {
-      if (
-        tablesFilter.some(
-          (filter: any) => filter.toLowerCase() === key.toLowerCase()
-        )
-      ) {
-        datasource.entities[key] = tables[key]
-      }
-    }
-  } else {
-    datasource.entities = tables
-  }
+  const { tables, error } = await buildFilteredSchema(datasource, tablesFilter)
+  datasource.entities = tables
 
   setDefaultDisplayColumns(datasource)
   const dbResp = await db.put(datasource)
   datasource._rev = dbResp.rev
   const cleanedDatasource = await sdk.datasources.removeSecretSingle(datasource)
 
-  const response: any = { datasource: cleanedDatasource }
+  const res: any = { datasource: cleanedDatasource }
   if (error) {
-    response.error = error
+    res.error = error
   }
-  ctx.body = response
+  ctx.body = res
 }
 
 /**
@@ -306,12 +309,19 @@ export async function update(ctx: UserCtx<any, UpdateDatasourceResponse>) {
   builderSocket?.emitDatasourceUpdate(ctx, datasource)
 }
 
+const preSaveAction: Partial<Record<SourceName, any>> = {
+  [SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => {
+    await googleSetupCreationAuth(datasource.config as any)
+  },
+}
+
 export async function save(
   ctx: UserCtx<CreateDatasourceRequest, CreateDatasourceResponse>
 ) {
   const db = context.getAppDB()
   const plus = ctx.request.body.datasource.plus
   const fetchSchema = ctx.request.body.fetchSchema
+  const tablesFilter = ctx.request.body.tablesFilter
 
   const datasource = {
     _id: generateDatasourceID({ plus }),
@@ -321,12 +331,19 @@ export async function save(
 
   let schemaError = null
   if (fetchSchema) {
-    const { tables, error } = await buildSchemaHelper(datasource)
+    const { tables, error } = await buildFilteredSchema(
+      datasource,
+      tablesFilter
+    )
     schemaError = error
     datasource.entities = tables
     setDefaultDisplayColumns(datasource)
   }
 
+  if (preSaveAction[datasource.source]) {
+    await preSaveAction[datasource.source](datasource)
+  }
+
   const dbResp = await db.put(datasource)
   await events.datasource.created(datasource)
   datasource._rev = dbResp.rev

package/src/api/controllers/role.ts

@@ -4,7 +4,7 @@ import {
   getUserMetadataParams,
   InternalTables,
 } from "../../db/utils"
-import { BBContext, Database } from "@budibase/types"
+import { UserCtx, Database } from "@budibase/types"
 
 const UpdateRolesOptions = {
   CREATED: "created",
@@ -38,15 +38,15 @@ async function updateRolesOnUserTable(
   }
 }
 
-export async function fetch(ctx: BBContext) {
+export async function fetch(ctx: UserCtx) {
   ctx.body = await roles.getAllRoles()
 }
 
-export async function find(ctx: BBContext) {
+export async function find(ctx: UserCtx) {
   ctx.body = await roles.getRole(ctx.params.roleId)
 }
 
-export async function save(ctx: BBContext) {
+export async function save(ctx: UserCtx) {
   const db = context.getAppDB()
   let { _id, name, inherits, permissionId } = ctx.request.body
   let isCreate = false
@@ -72,7 +72,7 @@ export async function save(ctx: BBContext) {
   ctx.message = `Role '${role.name}' created successfully.`
 }
 
-export async function destroy(ctx: BBContext) {
+export async function destroy(ctx: UserCtx) {
   const db = context.getAppDB()
   const roleId = ctx.params.roleId
   const role = await db.get(roleId)

package/src/api/controllers/routing.ts

@@ -1,6 +1,6 @@
 import { getRoutingInfo } from "../../utilities/routing"
 import { roles } from "@budibase/backend-core"
-import { BBContext } from "@budibase/types"
+import { UserCtx } from "@budibase/types"
 
 const URL_SEPARATOR = "/"
 
@@ -56,11 +56,11 @@ async function getRoutingStructure() {
   return { routes: routing.json }
 }
 
-export async function fetch(ctx: BBContext) {
+export async function fetch(ctx: UserCtx) {
   ctx.body = await getRoutingStructure()
 }
 
-export async function clientFetch(ctx: BBContext) {
+export async function clientFetch(ctx: UserCtx) {
   const routing = await getRoutingStructure()
   let roleId = ctx.user?.role?._id
   const roleIds = (await roles.getUserRoleHierarchy(roleId, {

package/src/api/routes/backup.ts

@@ -5,7 +5,7 @@ import { permissions } from "@budibase/backend-core"
 
 const router: Router = new Router()
 
-router.get(
+router.post(
   "/api/backups/export",
   authorized(permissions.BUILDER),
   controller.exportAppDump

package/src/api/routes/tests/backup.spec.ts

@@ -1,7 +1,9 @@
+import tk from "timekeeper"
 import * as setup from "./utilities"
 import { events } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
+import { mocks } from "@budibase/backend-core/tests"
 
 describe("/backups", () => {
   let request = setup.getRequest()
@@ -16,7 +18,7 @@ describe("/backups", () => {
   describe("exportAppDump", () => {
     it("should be able to export app", async () => {
       const res = await request
-        .get(`/api/backups/export?appId=${config.getAppId()}&appname=test`)
+        .post(`/api/backups/export?appId=${config.getAppId()}`)
         .set(config.defaultHeaders())
         .expect(200)
       expect(res.headers["content-type"]).toEqual("application/gzip")
@@ -26,10 +28,24 @@ describe("/backups", () => {
     it("should apply authorization to endpoint", async () => {
       await checkBuilderEndpoint({
         config,
-        method: "GET",
+        method: "POST",
         url: `/api/backups/export?appId=${config.getAppId()}`,
       })
     })
+
+    it("should infer the app name from the app", async () => {
+      tk.freeze(mocks.date.MOCK_DATE)
+
+      const res = await request
+        .post(`/api/backups/export?appId=${config.getAppId()}`)
+        .set(config.defaultHeaders())
+
+      expect(res.headers["content-disposition"]).toEqual(
+        `attachment; filename="${
+          config.getApp()!.name
+        }-export-${mocks.date.MOCK_DATE.getTime()}.tar.gz"`
+      )
+    })
   })
 
   describe("calculateBackupStats", () => {

package/src/automations/steps/sendSmtpEmail.ts

@@ -48,6 +48,35 @@ export const definition: AutomationStepSchema = {
           type: AutomationIOType.STRING,
           title: "HTML Contents",
         },
+        addInvite: {
+          type: AutomationIOType.BOOLEAN,
+          title: "Add calendar invite",
+        },
+        startTime: {
+          type: AutomationIOType.DATE,
+          title: "Start Time",
+          dependsOn: "addInvite",
+        },
+        endTime: {
+          type: AutomationIOType.DATE,
+          title: "End Time",
+          dependsOn: "addInvite",
+        },
+        summary: {
+          type: AutomationIOType.STRING,
+          title: "Meeting Summary",
+          dependsOn: "addInvite",
+        },
+        location: {
+          type: AutomationIOType.STRING,
+          title: "Location",
+          dependsOn: "addInvite",
+        },
+        url: {
+          type: AutomationIOType.STRING,
+          title: "URL",
+          dependsOn: "addInvite",
+        },
       },
       required: ["to", "from", "subject", "contents"],
     },
@@ -68,21 +97,43 @@ export const definition: AutomationStepSchema = {
 }
 
 export async function run({ inputs }: AutomationStepInput) {
-  let { to, from, subject, contents, cc, bcc } = inputs
+  let {
+    to,
+    from,
+    subject,
+    contents,
+    cc,
+    bcc,
+    addInvite,
+    startTime,
+    endTime,
+    summary,
+    location,
+    url,
+  } = inputs
   if (!contents) {
     contents = "<h1>No content</h1>"
   }
   to = to || undefined
   try {
-    let response = await sendSmtpEmail(
+    let response = await sendSmtpEmail({
       to,
       from,
       subject,
       contents,
       cc,
       bcc,
-      true
-    )
+      automation: true,
+      invite: addInvite
+        ? {
+            startTime,
+            endTime,
+            summary,
+            location,
+            url,
+          }
+        : undefined,
+    })
     return {
       success: true,
       response,

package/src/automations/tests/sendSmtpEmail.spec.ts

@@ -0,0 +1,74 @@
+import * as workerRequests from "../../utilities/workerRequests"
+
+jest.mock("../../utilities/workerRequests", () => ({
+  sendSmtpEmail: jest.fn(),
+}))
+
+function generateResponse(to: string, from: string) {
+  return {
+    success: true,
+    response: {
+      accepted: [to],
+      envelope: {
+        from: from,
+        to: [to],
+      },
+      message: `Email sent to ${to}.`,
+    },
+  }
+}
+
+const setup = require("./utilities")
+
+describe("test the outgoing webhook action", () => {
+  let inputs
+  let config = setup.getConfig()
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  afterAll(setup.afterAll)
+
+  it("should be able to run the action", async () => {
+    jest
+      .spyOn(workerRequests, "sendSmtpEmail")
+      .mockImplementationOnce(async () =>
+        generateResponse("user1@test.com", "admin@test.com")
+      )
+    const invite = {
+      startTime: new Date(),
+      endTime: new Date(),
+      summary: "summary",
+      location: "location",
+      url: "url",
+    }
+    inputs = {
+      to: "user1@test.com",
+      from: "admin@test.com",
+      subject: "hello",
+      contents: "testing",
+      cc: "cc",
+      bcc: "bcc",
+      addInvite: true,
+      ...invite,
+    }
+    let resp = generateResponse(inputs.to, inputs.from)
+    const res = await setup.runStep(
+      setup.actions.SEND_EMAIL_SMTP.stepId,
+      inputs
+    )
+    expect(res.response).toEqual(resp)
+    expect(res.success).toEqual(true)
+    expect(workerRequests.sendSmtpEmail).toHaveBeenCalledTimes(1)
+    expect(workerRequests.sendSmtpEmail).toHaveBeenCalledWith({
+      to: "user1@test.com",
+      from: "admin@test.com",
+      subject: "hello",
+      contents: "testing",
+      cc: "cc",
+      bcc: "bcc",
+      invite,
+      automation: true,
+    })
+  })
+})

package/src/events/docUpdates/syncUsers.ts

@@ -26,6 +26,10 @@ export default function process(updateCb?: UpdateCallback) {
       // if something not found - no changes to perform
       if (err?.status === 404) {
         return
+      }
+      // The user has already been sync in another process
+      else if (err?.status === 409) {
+        return
       } else {
         logging.logAlert("Failed to perform user/group app sync", err)
       }

package/src/integrations/googlesheets.ts

@@ -1,5 +1,6 @@
 import {
   ConnectionInfo,
+  Datasource,
   DatasourceFeature,
   DatasourceFieldType,
   DatasourcePlus,
@@ -19,13 +20,15 @@ import { OAuth2Client } from "google-auth-library"
 import { buildExternalTableId, finaliseExternalTables } from "./utils"
 import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
 import fetch from "node-fetch"
-import { configs, HTTPError } from "@budibase/backend-core"
-import { dataFilters } from "@budibase/shared-core"
+import { cache, configs, context, HTTPError } from "@budibase/backend-core"
+import { dataFilters, utils } from "@budibase/shared-core"
 import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
+import sdk from "../sdk"
 
 interface GoogleSheetsConfig {
   spreadsheetId: string
   auth: OAuthClientConfig
+  continueSetupId?: string
 }
 
 interface OAuthClientConfig {
@@ -72,7 +75,7 @@ const SCHEMA: Integration = {
   },
   datasource: {
     spreadsheetId: {
-      display: "Google Sheet URL",
+      display: "Spreadsheet URL",
       type: DatasourceFieldType.STRING,
       required: true,
     },
@@ -207,6 +210,8 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
   async connect() {
     try {
+      await setupCreationAuth(this.config)
+
       // Initialise oAuth client
       let googleConfig = await configs.getGoogleDatasourceConfig()
       if (!googleConfig) {
@@ -269,24 +274,24 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   }
 
   async buildSchema(datasourceId: string, entities: Record<string, Table>) {
-    // not fully configured yet
-    if (!this.config.auth) {
-      return
-    }
     await this.connect()
     const sheets = this.client.sheetsByIndex
     const tables: Record<string, Table> = {}
-    for (let sheet of sheets) {
-      // must fetch rows to determine schema
-      await sheet.getRows()
-
-      const id = buildExternalTableId(datasourceId, sheet.title)
-      tables[sheet.title] = this.getTableSchema(
-        sheet.title,
-        sheet.headerValues,
-        id
-      )
-    }
+    await utils.parallelForeach(
+      sheets,
+      async sheet => {
+        // must fetch rows to determine schema
+        await sheet.getRows({ limit: 0, offset: 0 })
+
+        const id = buildExternalTableId(datasourceId, sheet.title)
+        tables[sheet.title] = this.getTableSchema(
+          sheet.title,
+          sheet.headerValues,
+          id
+        )
+      },
+      10
+    )
     const final = finaliseExternalTables(tables, entities)
     this.tables = final.tables
     this.schemaErrors = final.errors
@@ -566,6 +571,18 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   }
 }
 
+export async function setupCreationAuth(datasouce: GoogleSheetsConfig) {
+  if (datasouce.continueSetupId) {
+    const appId = context.getAppId()
+    const tokens = await cache.get(
+      `datasource:creation:${appId}:google:${datasouce.continueSetupId}`
+    )
+
+    datasouce.auth = tokens.tokens
+    delete datasouce.continueSetupId
+  }
+}
+
 export default {
   schema: SCHEMA,
   integration: GoogleSheetsIntegration,

package/src/integrations/mongodb.ts

@@ -351,7 +351,7 @@ const SCHEMA: Integration = getSchema()
 
 class MongoIntegration implements IntegrationBase {
   private config: MongoDBConfig
-  private client: any
+  private client: MongoClient
 
   constructor(config: MongoDBConfig) {
     this.config = config
@@ -372,6 +372,8 @@ class MongoIntegration implements IntegrationBase {
       response.connected = true
     } catch (e: any) {
       response.error = e.message as string
+    } finally {
+      await this.client.close()
     }
     return response
   }
@@ -380,7 +382,7 @@ class MongoIntegration implements IntegrationBase {
     return this.client.connect()
   }
 
-  createObjectIds(json: any): object {
+  createObjectIds(json: any) {
     const self = this
     function interpolateObjectIds(json: any) {
       for (let field of Object.keys(json)) {