@budibase/server 2.22.15 → 2.22.16

package/src/integrations/tests/utils/mariadb.ts

@@ -1,8 +1,11 @@
 import { Datasource, SourceName } from "@budibase/types"
-import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+import { GenericContainer, Wait } from "testcontainers"
 import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy"
+import { rawQuery } from "./mysql"
+import { generator, testContainerUtils } from "@budibase/backend-core/tests"
+import { startContainer } from "."
 
-let container: StartedTestContainer | undefined
+let ports: Promise<testContainerUtils.Port[]>
 
 class MariaDBWaitStrategy extends AbstractWaitStrategy {
   async waitUntilReady(container: any, boundPorts: any, startTime?: Date) {
@@ -21,38 +24,38 @@ class MariaDBWaitStrategy extends AbstractWaitStrategy {
   }
 }
 
-export async function start(): Promise<StartedTestContainer> {
-  return await new GenericContainer("mariadb:lts")
-    .withExposedPorts(3306)
-    .withEnvironment({ MARIADB_ROOT_PASSWORD: "password" })
-    .withWaitStrategy(new MariaDBWaitStrategy())
-    .start()
-}
+export async function getDatasource(): Promise<Datasource> {
+  if (!ports) {
+    ports = startContainer(
+      new GenericContainer("mariadb:lts")
+        .withExposedPorts(3306)
+        .withEnvironment({ MARIADB_ROOT_PASSWORD: "password" })
+        .withWaitStrategy(new MariaDBWaitStrategy())
+    )
+  }
 
-export async function datasource(): Promise<Datasource> {
-  if (!container) {
-    container = await start()
+  const port = (await ports).find(x => x.container === 3306)?.host
+  if (!port) {
+    throw new Error("MariaDB port not found")
   }
-  const host = container.getHost()
-  const port = container.getMappedPort(3306)
 
-  return {
+  const config = {
+    host: "127.0.0.1",
+    port,
+    user: "root",
+    password: "password",
+    database: "mysql",
+  }
+
+  const datasource = {
     type: "datasource_plus",
     source: SourceName.MYSQL,
     plus: true,
-    config: {
-      host,
-      port,
-      user: "root",
-      password: "password",
-      database: "mysql",
-    },
+    config,
   }
-}
 
-export async function stop() {
-  if (container) {
-    await container.stop()
-    container = undefined
-  }
+  const database = generator.guid().replaceAll("-", "")
+  await rawQuery(datasource, `CREATE DATABASE \`${database}\``)
+  datasource.config.database = database
+  return datasource
 }

package/src/integrations/tests/utils/mongodb.ts

@@ -1,43 +1,39 @@
+import { generator, testContainerUtils } from "@budibase/backend-core/tests"
 import { Datasource, SourceName } from "@budibase/types"
-import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+import { GenericContainer, Wait } from "testcontainers"
+import { startContainer } from "."
 
-let container: StartedTestContainer | undefined
+let ports: Promise<testContainerUtils.Port[]>
 
-export async function start(): Promise<StartedTestContainer> {
-  return await new GenericContainer("mongo:7.0-jammy")
-    .withExposedPorts(27017)
-    .withEnvironment({
-      MONGO_INITDB_ROOT_USERNAME: "mongo",
-      MONGO_INITDB_ROOT_PASSWORD: "password",
-    })
-    .withWaitStrategy(
-      Wait.forSuccessfulCommand(
-        `mongosh --eval "db.version()"`
-      ).withStartupTimeout(10000)
+export async function getDatasource(): Promise<Datasource> {
+  if (!ports) {
+    ports = startContainer(
+      new GenericContainer("mongo:7.0-jammy")
+        .withExposedPorts(27017)
+        .withEnvironment({
+          MONGO_INITDB_ROOT_USERNAME: "mongo",
+          MONGO_INITDB_ROOT_PASSWORD: "password",
+        })
+        .withWaitStrategy(
+          Wait.forSuccessfulCommand(
+            `mongosh --eval "db.version()"`
+          ).withStartupTimeout(10000)
+        )
     )
-    .start()
-}
+  }
 
-export async function datasource(): Promise<Datasource> {
-  if (!container) {
-    container = await start()
+  const port = (await ports).find(x => x.container === 27017)
+  if (!port) {
+    throw new Error("MongoDB port not found")
   }
-  const host = container.getHost()
-  const port = container.getMappedPort(27017)
+
   return {
     type: "datasource",
     source: SourceName.MONGODB,
     plus: false,
     config: {
-      connectionString: `mongodb://mongo:password@${host}:${port}`,
-      db: "mongo",
+      connectionString: `mongodb://mongo:password@127.0.0.1:${port.host}`,
+      db: generator.guid(),
     },
   }
 }
-
-export async function stop() {
-  if (container) {
-    await container.stop()
-    container = undefined
-  }
-}

package/src/integrations/tests/utils/mssql.ts

@@ -1,43 +1,41 @@
 import { Datasource, SourceName } from "@budibase/types"
-import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
-
-let container: StartedTestContainer | undefined
-
-export async function start(): Promise<StartedTestContainer> {
-  return await new GenericContainer(
-    "mcr.microsoft.com/mssql/server:2022-latest"
-  )
-    .withExposedPorts(1433)
-    .withEnvironment({
-      ACCEPT_EULA: "Y",
-      MSSQL_SA_PASSWORD: "Password_123",
-      // This is important, as Microsoft allow us to use the "Developer" edition
-      // of SQL Server for development and testing purposes. We can't use other
-      // versions without a valid license, and we cannot use the Developer
-      // version in production.
-      MSSQL_PID: "Developer",
-    })
-    .withWaitStrategy(
-      Wait.forSuccessfulCommand(
-        "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Password_123 -q 'SELECT 1'"
-      )
-    )
-    .start()
-}
+import { GenericContainer, Wait } from "testcontainers"
+import mssql from "mssql"
+import { generator, testContainerUtils } from "@budibase/backend-core/tests"
+import { startContainer } from "."
+
+let ports: Promise<testContainerUtils.Port[]>
 
-export async function datasource(): Promise<Datasource> {
-  if (!container) {
-    container = await start()
+export async function getDatasource(): Promise<Datasource> {
+  if (!ports) {
+    ports = startContainer(
+      new GenericContainer("mcr.microsoft.com/mssql/server:2022-latest")
+        .withExposedPorts(1433)
+        .withEnvironment({
+          ACCEPT_EULA: "Y",
+          MSSQL_SA_PASSWORD: "Password_123",
+          // This is important, as Microsoft allow us to use the "Developer" edition
+          // of SQL Server for development and testing purposes. We can't use other
+          // versions without a valid license, and we cannot use the Developer
+          // version in production.
+          MSSQL_PID: "Developer",
+        })
+        .withWaitStrategy(
+          Wait.forSuccessfulCommand(
+            "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Password_123 -q 'SELECT 1'"
+          )
+        )
+    )
   }
-  const host = container.getHost()
-  const port = container.getMappedPort(1433)
 
-  return {
+  const port = (await ports).find(x => x.container === 1433)?.host
+
+  const datasource: Datasource = {
     type: "datasource_plus",
     source: SourceName.SQL_SERVER,
     plus: true,
     config: {
-      server: host,
+      server: "127.0.0.1",
       port,
       user: "sa",
       password: "Password_123",
@@ -46,11 +44,28 @@ export async function datasource(): Promise<Datasource> {
       },
     },
   }
+
+  const database = generator.guid().replaceAll("-", "")
+  await rawQuery(datasource, `CREATE DATABASE "${database}"`)
+  datasource.config!.database = database
+
+  return datasource
 }
 
-export async function stop() {
-  if (container) {
-    await container.stop()
-    container = undefined
+export async function rawQuery(ds: Datasource, sql: string) {
+  if (!ds.config) {
+    throw new Error("Datasource config is missing")
+  }
+  if (ds.source !== SourceName.SQL_SERVER) {
+    throw new Error("Datasource source is not SQL Server")
+  }
+
+  const pool = new mssql.ConnectionPool(ds.config! as mssql.config)
+  const client = await pool.connect()
+  try {
+    const { recordset } = await client.query(sql)
+    return recordset
+  } finally {
+    await pool.close()
   }
 }

package/src/integrations/tests/utils/mysql.ts

@@ -1,8 +1,11 @@
 import { Datasource, SourceName } from "@budibase/types"
-import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+import { GenericContainer, Wait } from "testcontainers"
 import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy"
+import mysql from "mysql2/promise"
+import { generator, testContainerUtils } from "@budibase/backend-core/tests"
+import { startContainer } from "."
 
-let container: StartedTestContainer | undefined
+let ports: Promise<testContainerUtils.Port[]>
 
 class MySQLWaitStrategy extends AbstractWaitStrategy {
   async waitUntilReady(container: any, boundPorts: any, startTime?: Date) {
@@ -24,38 +27,50 @@ class MySQLWaitStrategy extends AbstractWaitStrategy {
   }
 }
 
-export async function start(): Promise<StartedTestContainer> {
-  return await new GenericContainer("mysql:8.3")
-    .withExposedPorts(3306)
-    .withEnvironment({ MYSQL_ROOT_PASSWORD: "password" })
-    .withWaitStrategy(new MySQLWaitStrategy().withStartupTimeout(10000))
-    .start()
-}
-
-export async function datasource(): Promise<Datasource> {
-  if (!container) {
-    container = await start()
+export async function getDatasource(): Promise<Datasource> {
+  if (!ports) {
+    ports = startContainer(
+      new GenericContainer("mysql:8.3")
+        .withExposedPorts(3306)
+        .withEnvironment({ MYSQL_ROOT_PASSWORD: "password" })
+        .withWaitStrategy(new MySQLWaitStrategy().withStartupTimeout(10000))
+    )
   }
-  const host = container.getHost()
-  const port = container.getMappedPort(3306)
 
-  return {
+  const port = (await ports).find(x => x.container === 3306)?.host
+
+  const datasource: Datasource = {
     type: "datasource_plus",
     source: SourceName.MYSQL,
     plus: true,
     config: {
-      host,
+      host: "127.0.0.1",
       port,
       user: "root",
       password: "password",
       database: "mysql",
     },
   }
+
+  const database = generator.guid().replaceAll("-", "")
+  await rawQuery(datasource, `CREATE DATABASE \`${database}\``)
+  datasource.config!.database = database
+  return datasource
 }
 
-export async function stop() {
-  if (container) {
-    await container.stop()
-    container = undefined
+export async function rawQuery(ds: Datasource, sql: string) {
+  if (!ds.config) {
+    throw new Error("Datasource config is missing")
+  }
+  if (ds.source !== SourceName.MYSQL) {
+    throw new Error("Datasource source is not MySQL")
+  }
+
+  const connection = await mysql.createConnection(ds.config)
+  try {
+    const [rows] = await connection.query(sql)
+    return rows
+  } finally {
+    connection.end()
   }
 }

package/src/integrations/tests/utils/postgres.ts

@@ -1,33 +1,33 @@
 import { Datasource, SourceName } from "@budibase/types"
-import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
-
-let container: StartedTestContainer | undefined
-
-export async function start(): Promise<StartedTestContainer> {
-  return await new GenericContainer("postgres:16.1-bullseye")
-    .withExposedPorts(5432)
-    .withEnvironment({ POSTGRES_PASSWORD: "password" })
-    .withWaitStrategy(
-      Wait.forSuccessfulCommand(
-        "pg_isready -h localhost -p 5432"
-      ).withStartupTimeout(10000)
-    )
-    .start()
-}
+import { GenericContainer, Wait } from "testcontainers"
+import pg from "pg"
+import { generator, testContainerUtils } from "@budibase/backend-core/tests"
+import { startContainer } from "."
+
+let ports: Promise<testContainerUtils.Port[]>
 
-export async function datasource(): Promise<Datasource> {
-  if (!container) {
-    container = await start()
+export async function getDatasource(): Promise<Datasource> {
+  if (!ports) {
+    ports = startContainer(
+      new GenericContainer("postgres:16.1-bullseye")
+        .withExposedPorts(5432)
+        .withEnvironment({ POSTGRES_PASSWORD: "password" })
+        .withWaitStrategy(
+          Wait.forSuccessfulCommand(
+            "pg_isready -h localhost -p 5432"
+          ).withStartupTimeout(10000)
+        )
+    )
   }
-  const host = container.getHost()
-  const port = container.getMappedPort(5432)
 
-  return {
+  const port = (await ports).find(x => x.container === 5432)?.host
+
+  const datasource: Datasource = {
     type: "datasource_plus",
     source: SourceName.POSTGRES,
     plus: true,
     config: {
-      host,
+      host: "127.0.0.1",
       port,
       database: "postgres",
       user: "postgres",
@@ -38,11 +38,28 @@ export async function datasource(): Promise<Datasource> {
       ca: false,
     },
   }
+
+  const database = generator.guid().replaceAll("-", "")
+  await rawQuery(datasource, `CREATE DATABASE "${database}"`)
+  datasource.config!.database = database
+
+  return datasource
 }
 
-export async function stop() {
-  if (container) {
-    await container.stop()
-    container = undefined
+export async function rawQuery(ds: Datasource, sql: string) {
+  if (!ds.config) {
+    throw new Error("Datasource config is missing")
+  }
+  if (ds.source !== SourceName.POSTGRES) {
+    throw new Error("Datasource source is not Postgres")
+  }
+
+  const client = new pg.Client(ds.config)
+  await client.connect()
+  try {
+    const { rows } = await client.query(sql)
+    return rows
+  } finally {
+    await client.end()
   }
 }

package/src/migrations/tests/index.spec.ts

@@ -25,8 +25,6 @@ const clearMigrations = async () => {
   }
 }
 
-jest.setTimeout(10000)
-
 describe("migrations", () => {
   const config = new TestConfig()
 

package/src/sdk/app/rows/search/tests/external.spec.ts

@@ -17,8 +17,6 @@ import {
   generator,
 } from "@budibase/backend-core/tests"
 
-jest.setTimeout(30000)
-
 describe("external search", () => {
   const config = new TestConfiguration()
 

package/src/tests/jestSetup.ts

@@ -2,17 +2,11 @@ import env from "../environment"
 import { env as coreEnv, timers } from "@budibase/backend-core"
 import { testContainerUtils } from "@budibase/backend-core/tests"
 
-if (!process.env.DEBUG) {
-  global.console.log = jest.fn() // console.log are ignored in tests
-  global.console.warn = jest.fn() // console.warn are ignored in tests
-}
-
 if (!process.env.CI) {
-  // set a longer timeout in dev for debugging
-  // 100 seconds
+  // set a longer timeout in dev for debugging 100 seconds
   jest.setTimeout(100 * 1000)
 } else {
-  jest.setTimeout(10 * 1000)
+  jest.setTimeout(30 * 1000)
 }
 
 testContainerUtils.setupEnv(env, coreEnv)

package/src/tests/utilities/api/base.ts

@@ -1,6 +1,7 @@
 import TestConfiguration from "../TestConfiguration"
-import { SuperTest, Test, Response } from "supertest"
+import request, { SuperTest, Test, Response } from "supertest"
 import { ReadStream } from "fs"
+import { getServer } from "../../../app"
 
 type Headers = Record<string, string | string[] | undefined>
 type Method = "get" | "post" | "put" | "patch" | "delete"
@@ -76,7 +77,8 @@ export abstract class TestAPI {
   protected _requestRaw = async (
     method: "get" | "post" | "put" | "patch" | "delete",
     url: string,
-    opts?: RequestOpts
+    opts?: RequestOpts,
+    attempt = 0
   ): Promise<Response> => {
     const {
       headers = {},
@@ -107,26 +109,29 @@ export abstract class TestAPI {
     const headersFn = publicUser
       ? this.config.publicHeaders.bind(this.config)
       : this.config.defaultHeaders.bind(this.config)
-    let request = this.request[method](url).set(
+
+    const app = getServer()
+    let req = request(app)[method](url)
+    req = req.set(
       headersFn({
         "x-budibase-include-stacktrace": "true",
       })
     )
     if (headers) {
-      request = request.set(headers)
+      req = req.set(headers)
     }
     if (body) {
-      request = request.send(body)
+      req = req.send(body)
     }
     for (const [key, value] of Object.entries(fields)) {
-      request = request.field(key, value)
+      req = req.field(key, value)
     }
 
     for (const [key, value] of Object.entries(files)) {
       if (isAttachedFile(value)) {
-        request = request.attach(key, value.file, value.name)
+        req = req.attach(key, value.file, value.name)
       } else {
-        request = request.attach(key, value as any)
+        req = req.attach(key, value as any)
       }
     }
     if (expectations?.headers) {
@@ -136,11 +141,25 @@ export abstract class TestAPI {
             `Got an undefined expected value for header "${key}", if you want to check for the absence of a header, use headersNotPresent`
           )
         }
-        request = request.expect(key, value as any)
+        req = req.expect(key, value as any)
       }
     }
 
-    return await request
+    try {
+      return await req
+    } catch (e: any) {
+      // We've found that occasionally the connection between supertest and the
+      // server supertest starts gets reset. Not sure why, but retrying it
+      // appears to work. I don't particularly like this, but it's better than
+      // flakiness.
+      if (e.code === "ECONNRESET") {
+        if (attempt > 2) {
+          throw e
+        }
+        return await this._requestRaw(method, url, opts, attempt + 1)
+      }
+      throw e
+    }
   }
 
   protected _checkResponse = (
@@ -170,7 +189,18 @@ export abstract class TestAPI {
         }
       }
 
-      throw new Error(message)
+      if (response.error) {
+        // Sometimes the error can be between supertest and the app, and when
+        // that happens response.error is sometimes populated with `text` that
+        // gives more detail about the error. The `message` is almost always
+        // useless from what I've seen.
+        if (response.error.text) {
+          response.error.message = response.error.text
+        }
+        throw new Error(message, { cause: response.error })
+      } else {
+        throw new Error(message)
+      }
     }
 
     if (expectations?.headersNotPresent) {