@budibase/server 2.22.15 → 2.22.16

package/src/integration-test/mysql.spec.ts

@@ -3,7 +3,6 @@ import {
   generateMakeRequest,
   MakeRequestResponse,
 } from "../api/routes/public/tests/utils"
-import { v4 as uuidv4 } from "uuid"
 import * as setup from "../api/routes/tests/utilities"
 import {
   Datasource,
@@ -12,12 +11,23 @@ import {
   TableRequest,
   TableSourceType,
 } from "@budibase/types"
-import { databaseTestProviders } from "../integrations/tests/utils"
-import mysql from "mysql2/promise"
+import {
+  DatabaseName,
+  getDatasource,
+  rawQuery,
+} from "../integrations/tests/utils"
 import { builderSocket } from "../websockets"
+import { generator } from "@budibase/backend-core/tests"
 // @ts-ignore
 fetch.mockSearch()
 
+function uniqueTableName(length?: number): string {
+  return generator
+    .guid()
+    .replaceAll("-", "_")
+    .substring(0, length || 10)
+}
+
 const config = setup.getConfig()!
 
 jest.mock("../websockets", () => ({
@@ -37,7 +47,8 @@ jest.mock("../websockets", () => ({
 
 describe("mysql integrations", () => {
   let makeRequest: MakeRequestResponse,
-    mysqlDatasource: Datasource,
+    rawDatasource: Datasource,
+    datasource: Datasource,
     primaryMySqlTable: Table
 
   beforeAll(async () => {
@@ -46,18 +57,13 @@ describe("mysql integrations", () => {
 
     makeRequest = generateMakeRequest(apiKey, true)
 
-    mysqlDatasource = await config.api.datasource.create(
-      await databaseTestProviders.mysql.datasource()
-    )
-  })
-
-  afterAll(async () => {
-    await databaseTestProviders.mysql.stop()
+    rawDatasource = await getDatasource(DatabaseName.MYSQL)
+    datasource = await config.api.datasource.create(rawDatasource)
   })
 
   beforeEach(async () => {
     primaryMySqlTable = await config.createTable({
-      name: uuidv4(),
+      name: uniqueTableName(),
       type: "table",
       primary: ["id"],
       schema: {
@@ -79,7 +85,7 @@ describe("mysql integrations", () => {
           type: FieldType.NUMBER,
         },
       },
-      sourceId: mysqlDatasource._id,
+      sourceId: datasource._id,
       sourceType: TableSourceType.EXTERNAL,
     })
   })
@@ -87,18 +93,15 @@ describe("mysql integrations", () => {
   afterAll(config.end)
 
   it("validate table schema", async () => {
-    const res = await makeRequest(
-      "get",
-      `/api/datasources/${mysqlDatasource._id}`
-    )
+    const res = await makeRequest("get", `/api/datasources/${datasource._id}`)
 
     expect(res.status).toBe(200)
     expect(res.body).toEqual({
       config: {
-        database: "mysql",
-        host: mysqlDatasource.config!.host,
+        database: expect.any(String),
+        host: datasource.config!.host,
         password: "--secret-value--",
-        port: mysqlDatasource.config!.port,
+        port: datasource.config!.port,
         user: "root",
       },
       plus: true,
@@ -117,7 +120,7 @@ describe("mysql integrations", () => {
     it("should be able to verify the connection", async () => {
       await config.api.datasource.verify(
         {
-          datasource: await databaseTestProviders.mysql.datasource(),
+          datasource: rawDatasource,
         },
         {
           body: {
@@ -128,13 +131,12 @@ describe("mysql integrations", () => {
     })
 
     it("should state an invalid datasource cannot connect", async () => {
-      const dbConfig = await databaseTestProviders.mysql.datasource()
       await config.api.datasource.verify(
         {
           datasource: {
-            ...dbConfig,
+            ...rawDatasource,
             config: {
-              ...dbConfig.config,
+              ...rawDatasource.config,
               password: "wrongpassword",
             },
           },
@@ -154,7 +156,7 @@ describe("mysql integrations", () => {
     it("should fetch information about mysql datasource", async () => {
       const primaryName = primaryMySqlTable.name
       const response = await makeRequest("post", "/api/datasources/info", {
-        datasource: mysqlDatasource,
+        datasource: datasource,
       })
       expect(response.status).toBe(200)
       expect(response.body.tableNames).toBeDefined()
@@ -163,40 +165,38 @@ describe("mysql integrations", () => {
   })
 
   describe("Integration compatibility with mysql search_path", () => {
-    let client: mysql.Connection, pathDatasource: Datasource
-    const database = "test1"
-    const database2 = "test-2"
+    let datasource: Datasource, rawDatasource: Datasource
+    const database = generator.guid()
+    const database2 = generator.guid()
 
     beforeAll(async () => {
-      const dsConfig = await databaseTestProviders.mysql.datasource()
-      const dbConfig = dsConfig.config!
+      rawDatasource = await getDatasource(DatabaseName.MYSQL)
 
-      client = await mysql.createConnection(dbConfig)
-      await client.query(`CREATE DATABASE \`${database}\`;`)
-      await client.query(`CREATE DATABASE \`${database2}\`;`)
+      await rawQuery(rawDatasource, `CREATE DATABASE \`${database}\`;`)
+      await rawQuery(rawDatasource, `CREATE DATABASE \`${database2}\`;`)
 
       const pathConfig: any = {
-        ...dsConfig,
+        ...rawDatasource,
         config: {
-          ...dbConfig,
+          ...rawDatasource.config!,
           database,
         },
       }
-      pathDatasource = await config.api.datasource.create(pathConfig)
+      datasource = await config.api.datasource.create(pathConfig)
     })
 
     afterAll(async () => {
-      await client.query(`DROP DATABASE \`${database}\`;`)
-      await client.query(`DROP DATABASE \`${database2}\`;`)
-      await client.end()
+      await rawQuery(rawDatasource, `DROP DATABASE \`${database}\`;`)
+      await rawQuery(rawDatasource, `DROP DATABASE \`${database2}\`;`)
     })
 
     it("discovers tables from any schema in search path", async () => {
-      await client.query(
+      await rawQuery(
+        rawDatasource,
         `CREATE TABLE \`${database}\`.table1 (id1 SERIAL PRIMARY KEY);`
       )
       const response = await makeRequest("post", "/api/datasources/info", {
-        datasource: pathDatasource,
+        datasource: datasource,
       })
       expect(response.status).toBe(200)
       expect(response.body.tableNames).toBeDefined()
@@ -207,15 +207,17 @@ describe("mysql integrations", () => {
 
     it("does not mix columns from different tables", async () => {
       const repeated_table_name = "table_same_name"
-      await client.query(
+      await rawQuery(
+        rawDatasource,
         `CREATE TABLE \`${database}\`.${repeated_table_name} (id SERIAL PRIMARY KEY, val1 TEXT);`
       )
-      await client.query(
+      await rawQuery(
+        rawDatasource,
         `CREATE TABLE \`${database2}\`.${repeated_table_name} (id2 SERIAL PRIMARY KEY, val2 TEXT);`
       )
       const response = await makeRequest(
         "post",
-        `/api/datasources/${pathDatasource._id}/schema`,
+        `/api/datasources/${datasource._id}/schema`,
         {
           tablesFilter: [repeated_table_name],
         }
@@ -231,30 +233,14 @@ describe("mysql integrations", () => {
   })
 
   describe("POST /api/tables/", () => {
-    let client: mysql.Connection
     const emitDatasourceUpdateMock = jest.fn()
 
-    beforeEach(async () => {
-      client = await mysql.createConnection(
-        (
-          await databaseTestProviders.mysql.datasource()
-        ).config!
-      )
-      mysqlDatasource = await config.api.datasource.create(
-        await databaseTestProviders.mysql.datasource()
-      )
-    })
-
-    afterEach(async () => {
-      await client.end()
-    })
-
     it("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => {
       const addColumnToTable: TableRequest = {
         type: "table",
         sourceType: TableSourceType.EXTERNAL,
-        name: "table",
-        sourceId: mysqlDatasource._id!,
+        name: uniqueTableName(),
+        sourceId: datasource._id!,
         primary: ["id"],
         schema: {
           id: {
@@ -301,14 +287,16 @@ describe("mysql integrations", () => {
           },
         },
         created: true,
-        _id: `${mysqlDatasource._id}__table`,
+        _id: `${datasource._id}__${addColumnToTable.name}`,
       }
       delete expectedTable._add
 
       expect(emitDatasourceUpdateMock).toHaveBeenCalledTimes(1)
       const emittedDatasource: Datasource =
         emitDatasourceUpdateMock.mock.calls[0][1]
-      expect(emittedDatasource.entities!["table"]).toEqual(expectedTable)
+      expect(emittedDatasource.entities![expectedTable.name]).toEqual(
+        expectedTable
+      )
     })
 
     it("will rename a column", async () => {
@@ -346,17 +334,18 @@ describe("mysql integrations", () => {
         "/api/tables/",
         renameColumnOnTable
       )
-      mysqlDatasource = (
-        await makeRequest(
-          "post",
-          `/api/datasources/${mysqlDatasource._id}/schema`
-        )
+
+      const ds = (
+        await makeRequest("post", `/api/datasources/${datasource._id}/schema`)
       ).body.datasource
 
       expect(response.status).toEqual(200)
-      expect(
-        Object.keys(mysqlDatasource.entities![primaryMySqlTable.name].schema)
-      ).toEqual(["id", "name", "description", "age"])
+      expect(Object.keys(ds.entities![primaryMySqlTable.name].schema)).toEqual([
+        "id",
+        "name",
+        "description",
+        "age",
+      ])
     })
   })
 })

package/src/integration-test/postgres.spec.ts

@@ -16,8 +16,12 @@ import {
 import _ from "lodash"
 import { generator } from "@budibase/backend-core/tests"
 import { utils } from "@budibase/backend-core"
-import { databaseTestProviders } from "../integrations/tests/utils"
-import { Client } from "pg"
+import {
+  DatabaseName,
+  getDatasource,
+  rawQuery,
+} from "../integrations/tests/utils"
+
 // @ts-ignore
 fetch.mockSearch()
 
@@ -28,7 +32,8 @@ jest.mock("../websockets")
 
 describe("postgres integrations", () => {
   let makeRequest: MakeRequestResponse,
-    postgresDatasource: Datasource,
+    rawDatasource: Datasource,
+    datasource: Datasource,
     primaryPostgresTable: Table,
     oneToManyRelationshipInfo: ForeignTableInfo,
     manyToOneRelationshipInfo: ForeignTableInfo,
@@ -40,19 +45,17 @@ describe("postgres integrations", () => {
 
     makeRequest = generateMakeRequest(apiKey, true)
 
-    postgresDatasource = await config.api.datasource.create(
-      await databaseTestProviders.postgres.datasource()
-    )
-  })
-
-  afterAll(async () => {
-    await databaseTestProviders.postgres.stop()
+    rawDatasource = await getDatasource(DatabaseName.POSTGRES)
+    datasource = await config.api.datasource.create(rawDatasource)
   })
 
   beforeEach(async () => {
     async function createAuxTable(prefix: string) {
       return await config.createTable({
-        name: `${prefix}_${generator.word({ length: 6 })}`,
+        name: `${prefix}_${generator
+          .guid()
+          .replaceAll("-", "")
+          .substring(0, 6)}`,
         type: "table",
         primary: ["id"],
         primaryDisplay: "title",
@@ -67,7 +70,7 @@ describe("postgres integrations", () => {
             type: FieldType.STRING,
           },
         },
-        sourceId: postgresDatasource._id,
+        sourceId: datasource._id,
         sourceType: TableSourceType.EXTERNAL,
       })
     }
@@ -89,7 +92,7 @@ describe("postgres integrations", () => {
     }
 
     primaryPostgresTable = await config.createTable({
-      name: `p_${generator.word({ length: 6 })}`,
+      name: `p_${generator.guid().replaceAll("-", "").substring(0, 6)}`,
       type: "table",
       primary: ["id"],
       schema: {
@@ -144,7 +147,7 @@ describe("postgres integrations", () => {
           main: true,
         },
       },
-      sourceId: postgresDatasource._id,
+      sourceId: datasource._id,
       sourceType: TableSourceType.EXTERNAL,
     })
   })
@@ -251,7 +254,7 @@ describe("postgres integrations", () => {
 
   async function createDefaultPgTable() {
     return await config.createTable({
-      name: generator.word({ length: 10 }),
+      name: generator.guid().replaceAll("-", "").substring(0, 10),
       type: "table",
       primary: ["id"],
       schema: {
@@ -261,7 +264,7 @@ describe("postgres integrations", () => {
           autocolumn: true,
         },
       },
-      sourceId: postgresDatasource._id,
+      sourceId: datasource._id,
       sourceType: TableSourceType.EXTERNAL,
     })
   }
@@ -299,19 +302,16 @@ describe("postgres integrations", () => {
   }
 
   it("validate table schema", async () => {
-    const res = await makeRequest(
-      "get",
-      `/api/datasources/${postgresDatasource._id}`
-    )
+    const res = await makeRequest("get", `/api/datasources/${datasource._id}`)
 
     expect(res.status).toBe(200)
     expect(res.body).toEqual({
       config: {
         ca: false,
-        database: "postgres",
-        host: postgresDatasource.config!.host,
+        database: expect.any(String),
+        host: datasource.config!.host,
         password: "--secret-value--",
-        port: postgresDatasource.config!.port,
+        port: datasource.config!.port,
         rejectUnauthorized: false,
         schema: "public",
         ssl: false,
@@ -1043,7 +1043,7 @@ describe("postgres integrations", () => {
     it("should be able to verify the connection", async () => {
       await config.api.datasource.verify(
         {
-          datasource: await databaseTestProviders.postgres.datasource(),
+          datasource: await getDatasource(DatabaseName.POSTGRES),
         },
         {
           body: {
@@ -1054,7 +1054,7 @@ describe("postgres integrations", () => {
     })
 
     it("should state an invalid datasource cannot connect", async () => {
-      const dbConfig = await databaseTestProviders.postgres.datasource()
+      const dbConfig = await getDatasource(DatabaseName.POSTGRES)
       await config.api.datasource.verify(
         {
           datasource: {
@@ -1079,7 +1079,7 @@ describe("postgres integrations", () => {
     it("should fetch information about postgres datasource", async () => {
       const primaryName = primaryPostgresTable.name
       const response = await makeRequest("post", "/api/datasources/info", {
-        datasource: postgresDatasource,
+        datasource: datasource,
       })
       expect(response.status).toBe(200)
       expect(response.body.tableNames).toBeDefined()
@@ -1088,86 +1088,88 @@ describe("postgres integrations", () => {
   })
 
   describe("POST /api/datasources/:datasourceId/schema", () => {
-    let client: Client
+    let tableName: string
 
     beforeEach(async () => {
-      client = new Client(
-        (await databaseTestProviders.postgres.datasource()).config!
-      )
-      await client.connect()
+      tableName = generator.guid().replaceAll("-", "").substring(0, 10)
     })
 
     afterEach(async () => {
-      await client.query(`DROP TABLE IF EXISTS "table"`)
-      await client.end()
+      await rawQuery(rawDatasource, `DROP TABLE IF EXISTS "${tableName}"`)
     })
 
     it("recognises when a table has no primary key", async () => {
-      await client.query(`CREATE TABLE "table" (id SERIAL)`)
+      await rawQuery(rawDatasource, `CREATE TABLE "${tableName}" (id SERIAL)`)
 
       const response = await makeRequest(
         "post",
-        `/api/datasources/${postgresDatasource._id}/schema`
+        `/api/datasources/${datasource._id}/schema`
       )
 
       expect(response.body.errors).toEqual({
-        table: "Table must have a primary key.",
+        [tableName]: "Table must have a primary key.",
       })
     })
 
     it("recognises when a table is using a reserved column name", async () => {
-      await client.query(`CREATE TABLE "table" (_id SERIAL PRIMARY KEY) `)
+      await rawQuery(
+        rawDatasource,
+        `CREATE TABLE "${tableName}" (_id SERIAL PRIMARY KEY) `
+      )
 
       const response = await makeRequest(
         "post",
-        `/api/datasources/${postgresDatasource._id}/schema`
+        `/api/datasources/${datasource._id}/schema`
       )
 
       expect(response.body.errors).toEqual({
-        table: "Table contains invalid columns.",
+        [tableName]: "Table contains invalid columns.",
       })
     })
   })
 
   describe("Integration compatibility with postgres search_path", () => {
-    let client: Client, pathDatasource: Datasource
-    const schema1 = "test1",
-      schema2 = "test-2"
+    let rawDatasource: Datasource,
+      datasource: Datasource,
+      schema1: string,
+      schema2: string
 
-    beforeAll(async () => {
-      const dsConfig = await databaseTestProviders.postgres.datasource()
-      const dbConfig = dsConfig.config!
+    beforeEach(async () => {
+      schema1 = generator.guid().replaceAll("-", "")
+      schema2 = generator.guid().replaceAll("-", "")
 
-      client = new Client(dbConfig)
-      await client.connect()
-      await client.query(`CREATE SCHEMA "${schema1}";`)
-      await client.query(`CREATE SCHEMA "${schema2}";`)
+      rawDatasource = await getDatasource(DatabaseName.POSTGRES)
+      const dbConfig = rawDatasource.config!
+
+      await rawQuery(rawDatasource, `CREATE SCHEMA "${schema1}";`)
+      await rawQuery(rawDatasource, `CREATE SCHEMA "${schema2}";`)
 
       const pathConfig: any = {
-        ...dsConfig,
+        ...rawDatasource,
         config: {
           ...dbConfig,
           schema: `${schema1}, ${schema2}`,
         },
       }
-      pathDatasource = await config.api.datasource.create(pathConfig)
+      datasource = await config.api.datasource.create(pathConfig)
     })
 
-    afterAll(async () => {
-      await client.query(`DROP SCHEMA "${schema1}" CASCADE;`)
-      await client.query(`DROP SCHEMA "${schema2}" CASCADE;`)
-      await client.end()
+    afterEach(async () => {
+      await rawQuery(rawDatasource, `DROP SCHEMA "${schema1}" CASCADE;`)
+      await rawQuery(rawDatasource, `DROP SCHEMA "${schema2}" CASCADE;`)
     })
 
     it("discovers tables from any schema in search path", async () => {
-      await client.query(
+      await rawQuery(
+        rawDatasource,
         `CREATE TABLE "${schema1}".table1 (id1 SERIAL PRIMARY KEY);`
       )
-      await client.query(
+      await rawQuery(
+        rawDatasource,
         `CREATE TABLE "${schema2}".table2 (id2 SERIAL PRIMARY KEY);`
       )
       const response = await makeRequest("post", "/api/datasources/info", {
-        datasource: pathDatasource,
+        datasource: datasource,
       })
       expect(response.status).toBe(200)
       expect(response.body.tableNames).toBeDefined()
@@ -1178,15 +1180,17 @@ describe("postgres integrations", () => {
 
     it("does not mix columns from different tables", async () => {
       const repeated_table_name = "table_same_name"
-      await client.query(
+      await rawQuery(
+        rawDatasource,
         `CREATE TABLE "${schema1}".${repeated_table_name} (id SERIAL PRIMARY KEY, val1 TEXT);`
       )
-      await client.query(
+      await rawQuery(
+        rawDatasource,
         `CREATE TABLE "${schema2}".${repeated_table_name} (id2 SERIAL PRIMARY KEY, val2 TEXT);`
       )
       const response = await makeRequest(
         "post",
-        `/api/datasources/${pathDatasource._id}/schema`,
+        `/api/datasources/${datasource._id}/schema`,
         {
           tablesFilter: [repeated_table_name],
         }

package/src/integrations/tests/utils/index.ts

@@ -1,25 +1,90 @@
 jest.unmock("pg")
 
-import { Datasource } from "@budibase/types"
+import { Datasource, SourceName } from "@budibase/types"
 import * as postgres from "./postgres"
 import * as mongodb from "./mongodb"
 import * as mysql from "./mysql"
 import * as mssql from "./mssql"
 import * as mariadb from "./mariadb"
-import { StartedTestContainer } from "testcontainers"
+import { GenericContainer } from "testcontainers"
+import { testContainerUtils } from "@budibase/backend-core/tests"
 
-jest.setTimeout(30000)
+export type DatasourceProvider = () => Promise<Datasource>
 
-export interface DatabaseProvider {
-  start(): Promise<StartedTestContainer>
-  stop(): Promise<void>
-  datasource(): Promise<Datasource>
+export enum DatabaseName {
+  POSTGRES = "postgres",
+  MONGODB = "mongodb",
+  MYSQL = "mysql",
+  SQL_SERVER = "mssql",
+  MARIADB = "mariadb",
 }
 
-export const databaseTestProviders = {
-  postgres,
-  mongodb,
-  mysql,
-  mssql,
-  mariadb,
+const providers: Record<DatabaseName, DatasourceProvider> = {
+  [DatabaseName.POSTGRES]: postgres.getDatasource,
+  [DatabaseName.MONGODB]: mongodb.getDatasource,
+  [DatabaseName.MYSQL]: mysql.getDatasource,
+  [DatabaseName.SQL_SERVER]: mssql.getDatasource,
+  [DatabaseName.MARIADB]: mariadb.getDatasource,
+}
+
+export function getDatasourceProviders(
+  ...sourceNames: DatabaseName[]
+): Promise<Datasource>[] {
+  return sourceNames.map(sourceName => providers[sourceName]())
+}
+
+export function getDatasourceProvider(
+  sourceName: DatabaseName
+): DatasourceProvider {
+  return providers[sourceName]
+}
+
+export function getDatasource(sourceName: DatabaseName): Promise<Datasource> {
+  return providers[sourceName]()
+}
+
+export async function getDatasources(
+  ...sourceNames: DatabaseName[]
+): Promise<Datasource[]> {
+  return Promise.all(sourceNames.map(sourceName => providers[sourceName]()))
+}
+
+export async function rawQuery(ds: Datasource, sql: string): Promise<any> {
+  switch (ds.source) {
+    case SourceName.POSTGRES: {
+      return postgres.rawQuery(ds, sql)
+    }
+    case SourceName.MYSQL: {
+      return mysql.rawQuery(ds, sql)
+    }
+    case SourceName.SQL_SERVER: {
+      return mssql.rawQuery(ds, sql)
+    }
+    default: {
+      throw new Error(`Unsupported source: ${ds.source}`)
+    }
+  }
+}
+
+export async function startContainer(container: GenericContainer) {
+  if (process.env.REUSE_CONTAINERS) {
+    container = container.withReuse()
+  }
+
+  const startedContainer = await container.start()
+
+  const info = testContainerUtils.getContainerById(startedContainer.getId())
+  if (!info) {
+    throw new Error("Container not found")
+  }
+
+  // Some Docker runtimes, when you expose a port, will bind it to both
+  // 127.0.0.1 and ::1, so ipv4 and ipv6. The port spaces of ipv4 and ipv6
+  // addresses are not shared, and testcontainers will sometimes give you back
+  // the ipv6 port. There's no way to know that this has happened, and if you
+  // try to then connect to `localhost:port` you may attempt to bind to the v4
+  // address which could be unbound or even an entirely different container. For
+  // that reason, we don't use testcontainers' `getExposedPort` function,
+  // preferring instead our own method that guaranteed v4 ports.
+  return testContainerUtils.getExposedV4Ports(info)
 }