@budibase/server 2.7.16-alpha.0 → 2.7.18

package/src/integrations/googlesheets.ts

@@ -1,6 +1,5 @@
 import {
   ConnectionInfo,
-  Datasource,
   DatasourceFeature,
   DatasourceFieldType,
   DatasourcePlus,
@@ -20,15 +19,13 @@ import { OAuth2Client } from "google-auth-library"
 import { buildExternalTableId, finaliseExternalTables } from "./utils"
 import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
 import fetch from "node-fetch"
-import { cache, configs, context, HTTPError } from "@budibase/backend-core"
-import { dataFilters, utils } from "@budibase/shared-core"
+import { configs, HTTPError } from "@budibase/backend-core"
+import { dataFilters } from "@budibase/shared-core"
 import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
-import sdk from "../sdk"
 
 interface GoogleSheetsConfig {
   spreadsheetId: string
   auth: OAuthClientConfig
-  continueSetupId?: string
 }
 
 interface OAuthClientConfig {
@@ -75,7 +72,7 @@ const SCHEMA: Integration = {
   },
   datasource: {
     spreadsheetId: {
-      display: "Spreadsheet URL",
+      display: "Google Sheet URL",
       type: DatasourceFieldType.STRING,
       required: true,
     },
@@ -210,8 +207,6 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
   async connect() {
     try {
-      await setupCreationAuth(this.config)
-
       // Initialise oAuth client
       let googleConfig = await configs.getGoogleDatasourceConfig()
       if (!googleConfig) {
@@ -274,24 +269,24 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   }
 
   async buildSchema(datasourceId: string, entities: Record<string, Table>) {
+    // not fully configured yet
+    if (!this.config.auth) {
+      return
+    }
     await this.connect()
     const sheets = this.client.sheetsByIndex
     const tables: Record<string, Table> = {}
-    await utils.parallelForeach(
-      sheets,
-      async sheet => {
-        // must fetch rows to determine schema
-        await sheet.getRows({ limit: 0, offset: 0 })
-
-        const id = buildExternalTableId(datasourceId, sheet.title)
-        tables[sheet.title] = this.getTableSchema(
-          sheet.title,
-          sheet.headerValues,
-          id
-        )
-      },
-      10
-    )
+    for (let sheet of sheets) {
+      // must fetch rows to determine schema
+      await sheet.getRows()
+
+      const id = buildExternalTableId(datasourceId, sheet.title)
+      tables[sheet.title] = this.getTableSchema(
+        sheet.title,
+        sheet.headerValues,
+        id
+      )
+    }
     const final = finaliseExternalTables(tables, entities)
     this.tables = final.tables
     this.schemaErrors = final.errors
@@ -571,18 +566,6 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   }
 }
 
-export async function setupCreationAuth(datasouce: GoogleSheetsConfig) {
-  if (datasouce.continueSetupId) {
-    const appId = context.getAppId()
-    const tokens = await cache.get(
-      `datasource:creation:${appId}:google:${datasouce.continueSetupId}`
-    )
-
-    datasouce.auth = tokens.tokens
-    delete datasouce.continueSetupId
-  }
-}
-
 export default {
   schema: SCHEMA,
   integration: GoogleSheetsIntegration,

package/src/integrations/mongodb.ts

@@ -351,7 +351,7 @@ const SCHEMA: Integration = getSchema()
 
 class MongoIntegration implements IntegrationBase {
   private config: MongoDBConfig
-  private client: MongoClient
+  private client: any
 
   constructor(config: MongoDBConfig) {
     this.config = config
@@ -372,8 +372,6 @@ class MongoIntegration implements IntegrationBase {
       response.connected = true
     } catch (e: any) {
       response.error = e.message as string
-    } finally {
-      await this.client.close()
     }
     return response
   }
@@ -382,7 +380,7 @@ class MongoIntegration implements IntegrationBase {
     return this.client.connect()
   }
 
-  createObjectIds(json: any) {
+  createObjectIds(json: any): object {
     const self = this
     function interpolateObjectIds(json: any) {
       for (let field of Object.keys(json)) {

package/src/integrations/mysql.ts

@@ -91,7 +91,7 @@ const SCHEMA: Integration = {
   },
 }
 
-function bindingTypeCoerce(bindings: any[]) {
+export function bindingTypeCoerce(bindings: any[]) {
   for (let i = 0; i < bindings.length; i++) {
     const binding = bindings[i]
     if (typeof binding !== "string") {
@@ -109,7 +109,12 @@ function bindingTypeCoerce(bindings: any[]) {
       dayjs(binding).isValid() &&
       !binding.includes(",")
     ) {
-      bindings[i] = dayjs(binding).toDate()
+      let value: any
+      value = new Date(binding)
+      if (isNaN(value)) {
+        value = binding
+      }
+      bindings[i] = value
     }
   }
   return bindings

package/src/integrations/postgres.ts

@@ -322,8 +322,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
       await this.openConnection()
       const columnsResponse: { rows: PostgresColumn[] } =
         await this.client.query(this.COLUMNS_SQL)
-      const names = columnsResponse.rows.map(row => row.table_name)
-      return [...new Set(names)]
+      return columnsResponse.rows.map(row => row.table_name)
     } finally {
       await this.closeConnection()
     }

package/src/integrations/tests/mysql.spec.ts

@@ -1,4 +1,4 @@
-import { default as MySQLIntegration } from "../mysql"
+import { default as MySQLIntegration, bindingTypeCoerce } from "../mysql"
 jest.mock("mysql2")
 
 class TestConfiguration {
@@ -131,3 +131,21 @@ describe("MySQL Integration", () => {
     })
   })
 })
+
+describe("bindingTypeCoercion", () => {
+  it("shouldn't coerce something that looks like a date", () => {
+    const response = bindingTypeCoerce(["202205-1500"])
+    expect(response[0]).toBe("202205-1500")
+  })
+
+  it("should coerce an actual date", () => {
+    const date = new Date("2023-06-13T14:24:22.620Z")
+    const response = bindingTypeCoerce(["2023-06-13T14:24:22.620Z"])
+    expect(response[0]).toEqual(date)
+  })
+
+  it("should coerce numbers", () => {
+    const response = bindingTypeCoerce(["0"])
+    expect(response[0]).toEqual(0)
+  })
+})

package/src/integrations/utils.ts

@@ -328,3 +328,27 @@ export function finaliseExternalTables(
     .reduce((r, [k, v]) => ({ ...r, [k]: v }), {})
   return { tables: finalTables, errors }
 }
+
+/**
+ * Checks if the provided input is an object, but specifically not a date type object.
+ * Used during coercion of types and relationship handling, dates are considered valid
+ * and can be used as a display field, but objects and arrays cannot.
+ * @param testValue an unknown type which this function will attempt to extract
+ * a valid primary display string from.
+ */
+export function getPrimaryDisplay(testValue: unknown): string | undefined {
+  if (testValue instanceof Date) {
+    return testValue.toISOString()
+  }
+  if (
+    Array.isArray(testValue) &&
+    testValue[0] &&
+    typeof testValue[0] !== "object"
+  ) {
+    return testValue.join(", ")
+  }
+  if (typeof testValue === "object") {
+    return undefined
+  }
+  return testValue as string
+}

package/src/middleware/currentapp.ts

@@ -103,7 +103,7 @@ export default async (ctx: UserCtx, next: any) => {
         userId,
         globalId,
         roleId,
-        role: await roles.getRole(roleId, { defaultPublic: true }),
+        role: await roles.getRole(roleId),
       }
     }
 

package/src/sdk/app/backups/exports.ts

@@ -1,4 +1,4 @@
-import { db as dbCore, encryption, objectStore } from "@budibase/backend-core"
+import { db as dbCore, objectStore } from "@budibase/backend-core"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
 import { streamFile, createTempFolder } from "../../../utilities/fileSystem"
 import { ObjectStoreBuckets } from "../../../constants"
@@ -18,8 +18,7 @@ import { join } from "path"
 import env from "../../../environment"
 
 const uuid = require("uuid/v4")
-import tar from "tar"
-
+const tar = require("tar")
 const MemoryStream = require("memorystream")
 
 interface DBDumpOpts {
@@ -31,18 +30,16 @@ interface ExportOpts extends DBDumpOpts {
   tar?: boolean
   excludeRows?: boolean
   excludeLogs?: boolean
-  encryptPassword?: string
 }
 
 function tarFilesToTmp(tmpDir: string, files: string[]) {
-  const fileName = `${uuid()}.tar.gz`
-  const exportFile = join(budibaseTempDir(), fileName)
+  const exportFile = join(budibaseTempDir(), `${uuid()}.tar.gz`)
   tar.create(
     {
       sync: true,
       gzip: true,
       file: exportFile,
-      noDirRecurse: false,
+      recursive: true,
       cwd: tmpDir,
     },
     files
@@ -127,7 +124,6 @@ export async function exportApp(appId: string, config?: ExportOpts) {
       )
     }
   }
-
   const downloadedPath = join(tmpPath, appPath)
   if (fs.existsSync(downloadedPath)) {
     const allFiles = fs.readdirSync(downloadedPath)
@@ -145,27 +141,12 @@ export async function exportApp(appId: string, config?: ExportOpts) {
     filter: defineFilter(config?.excludeRows, config?.excludeLogs),
     exportPath: dbPath,
   })
-
-  if (config?.encryptPassword) {
-    for (let file of fs.readdirSync(tmpPath)) {
-      const path = join(tmpPath, file)
-
-      await encryption.encryptFile(
-        { dir: tmpPath, filename: file },
-        config.encryptPassword
-      )
-
-      fs.rmSync(path)
-    }
-  }
-
   // if tar requested, return where the tarball is
   if (config?.tar) {
     // now the tmpPath contains both the DB export and attachments, tar this
     const tarPath = tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath))
     // cleanup the tmp export files as tarball returned
     fs.rmSync(tmpPath, { recursive: true, force: true })
-
     return tarPath
   }
   // tar not requested, turn the directory where export is
@@ -180,20 +161,11 @@ export async function exportApp(appId: string, config?: ExportOpts) {
  * @param {boolean} excludeRows Flag to state whether the export should include data.
  * @returns {*} a readable stream of the backup which is written in real time
  */
-export async function streamExportApp({
-  appId,
-  excludeRows,
-  encryptPassword,
-}: {
-  appId: string
-  excludeRows: boolean
-  encryptPassword?: string
-}) {
+export async function streamExportApp(appId: string, excludeRows: boolean) {
   const tmpPath = await exportApp(appId, {
     excludeRows,
     excludeLogs: true,
     tar: true,
-    encryptPassword,
   })
   return streamFile(tmpPath)
 }

package/src/sdk/app/backups/imports.ts

@@ -1,4 +1,4 @@
-import { db as dbCore, encryption, objectStore } from "@budibase/backend-core"
+import { db as dbCore, objectStore } from "@budibase/backend-core"
 import { Database, Row } from "@budibase/types"
 import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
@@ -20,7 +20,6 @@ type TemplateType = {
   file?: {
     type: string
     path: string
-    password?: string
   }
   key?: string
 }
@@ -124,22 +123,6 @@ export function untarFile(file: { path: string }) {
   return tmpPath
 }
 
-async function decryptFiles(path: string, password: string) {
-  try {
-    for (let file of fs.readdirSync(path)) {
-      const inputPath = join(path, file)
-      const outputPath = inputPath.replace(/\.enc$/, "")
-      await encryption.decryptFile(inputPath, outputPath, password)
-      fs.rmSync(inputPath)
-    }
-  } catch (err: any) {
-    if (err.message === "incorrect header check") {
-      throw new Error("File cannot be imported")
-    }
-    throw err
-  }
-}
-
 export function getGlobalDBFile(tmpPath: string) {
   return fs.readFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), "utf8")
 }
@@ -160,9 +143,6 @@ export async function importApp(
     template.file && fs.lstatSync(template.file.path).isDirectory()
   if (template.file && (isTar || isDirectory)) {
     const tmpPath = isTar ? untarFile(template.file) : template.file.path
-    if (isTar && template.file.password) {
-      await decryptFiles(tmpPath, template.file.password)
-    }
     const contents = fs.readdirSync(tmpPath)
     // have to handle object import
     if (contents.length) {

package/src/sdk/app/datasources/datasources.ts

@@ -164,6 +164,5 @@ export function mergeConfigs(update: Datasource, old: Datasource) {
       delete update.config[key]
     }
   }
-
   return update
 }

package/src/utilities/workerRequests.ts

@@ -9,7 +9,7 @@ import {
   env as coreEnv,
 } from "@budibase/backend-core"
 import { updateAppRole } from "./global"
-import { BBContext, User, EmailInvite } from "@budibase/types"
+import { BBContext, User } from "@budibase/types"
 
 export function request(ctx?: BBContext, request?: any) {
   if (!request.headers) {
@@ -65,25 +65,15 @@ async function checkResponse(
 }
 
 // have to pass in the tenant ID as this could be coming from an automation
-export async function sendSmtpEmail({
-  to,
-  from,
-  subject,
-  contents,
-  cc,
-  bcc,
-  automation,
-  invite,
-}: {
-  to: string
-  from: string
-  subject: string
-  contents: string
-  cc: string
-  bcc: string
+export async function sendSmtpEmail(
+  to: string,
+  from: string,
+  subject: string,
+  contents: string,
+  cc: string,
+  bcc: string,
   automation: boolean
-  invite?: EmailInvite
-}) {
+) {
   // tenant ID will be set in header
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + `/api/global/email/send`),
@@ -98,7 +88,6 @@ export async function sendSmtpEmail({
         bcc,
         purpose: "custom",
         automation,
-        invite,
       },
     })
   )

package/src/automations/tests/sendSmtpEmail.spec.ts

@@ -1,74 +0,0 @@
-import * as workerRequests from "../../utilities/workerRequests"
-
-jest.mock("../../utilities/workerRequests", () => ({
-  sendSmtpEmail: jest.fn(),
-}))
-
-function generateResponse(to: string, from: string) {
-  return {
-    success: true,
-    response: {
-      accepted: [to],
-      envelope: {
-        from: from,
-        to: [to],
-      },
-      message: `Email sent to ${to}.`,
-    },
-  }
-}
-
-const setup = require("./utilities")
-
-describe("test the outgoing webhook action", () => {
-  let inputs
-  let config = setup.getConfig()
-  beforeAll(async () => {
-    await config.init()
-  })
-
-  afterAll(setup.afterAll)
-
-  it("should be able to run the action", async () => {
-    jest
-      .spyOn(workerRequests, "sendSmtpEmail")
-      .mockImplementationOnce(async () =>
-        generateResponse("user1@test.com", "admin@test.com")
-      )
-    const invite = {
-      startTime: new Date(),
-      endTime: new Date(),
-      summary: "summary",
-      location: "location",
-      url: "url",
-    }
-    inputs = {
-      to: "user1@test.com",
-      from: "admin@test.com",
-      subject: "hello",
-      contents: "testing",
-      cc: "cc",
-      bcc: "bcc",
-      addInvite: true,
-      ...invite,
-    }
-    let resp = generateResponse(inputs.to, inputs.from)
-    const res = await setup.runStep(
-      setup.actions.SEND_EMAIL_SMTP.stepId,
-      inputs
-    )
-    expect(res.response).toEqual(resp)
-    expect(res.success).toEqual(true)
-    expect(workerRequests.sendSmtpEmail).toHaveBeenCalledTimes(1)
-    expect(workerRequests.sendSmtpEmail).toHaveBeenCalledWith({
-      to: "user1@test.com",
-      from: "admin@test.com",
-      subject: "hello",
-      contents: "testing",
-      cc: "cc",
-      bcc: "bcc",
-      invite,
-      automation: true,
-    })
-  })
-})