@budibase/server 2.3.16 → 2.3.17-alpha.1

package/src/integrations/base/sql.ts

@@ -90,10 +90,15 @@ function parseFilters(filters: SearchFilters | undefined): SearchFilters {
 function generateSelectStatement(
   json: QueryJson,
   knex: Knex
-): (string | Knex.Raw)[] {
+): (string | Knex.Raw)[] | "*" {
   const { resource, meta } = json
+
+  if (!resource) {
+    return "*"
+  }
+
   const schema = meta?.table?.schema
-  return resource!.fields.map(field => {
+  return resource.fields.map(field => {
     const fieldNames = field.split(/\./g)
     const tableName = fieldNames[0]
     const columnName = fieldNames[1]
@@ -405,11 +410,14 @@ class InternalBuilder {
         delete parsedBody[key]
       }
     }
+
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.insert(parsedBody)
     } else {
-      return query.insert(parsedBody).returning("*")
+      return query
+        .insert(parsedBody)
+        .returning(generateSelectStatement(json, knex))
     }
   }
 
@@ -494,7 +502,9 @@ class InternalBuilder {
     if (opts.disableReturning) {
       return query.update(parsedBody)
     } else {
-      return query.update(parsedBody).returning("*")
+      return query
+        .update(parsedBody)
+        .returning(generateSelectStatement(json, knex))
     }
   }
 
@@ -509,7 +519,7 @@ class InternalBuilder {
     if (opts.disableReturning) {
       return query.delete()
     } else {
-      return query.delete().returning("*")
+      return query.delete().returning(generateSelectStatement(json, knex))
     }
   }
 }

package/src/integrations/oracle.ts

@@ -247,7 +247,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
     )
   }
 
-  private internalConvertType(column: OracleColumn): { type: string } {
+  private internalConvertType(column: OracleColumn): { type: FieldTypes } {
     if (this.isBooleanType(column)) {
       return { type: FieldTypes.BOOLEAN }
     }

package/src/migrations/functions/tests/appUrls.spec.js

@@ -6,7 +6,7 @@ const migration = require("../appUrls")
 describe("run", () => {
   let config = new TestConfig(false)
 
-  beforeEach(async () => {
+  beforeAll(async () => {
     await config.init()
   })
 

package/src/migrations/functions/tests/syncQuotas.spec.js

@@ -11,7 +11,7 @@ const migration = require("../syncQuotas")
 describe("run", () => {
   let config = new TestConfig(false)
 
-  beforeEach(async () => {
+  beforeAll(async () => {
     await config.init()
   })
 

package/src/migrations/functions/tests/userEmailViewCasing.spec.js

@@ -19,7 +19,7 @@ const migration = require("../userEmailViewCasing")
 describe("run", () => {
     let config = new TestConfig(false)
 
-    beforeEach(async () => {
+    beforeAll(async () => {
       await config.init()
     })
 

package/src/migrations/tests/index.spec.ts

@@ -10,8 +10,9 @@ import * as structures from "../../tests/utilities/structures"
 import { MIGRATIONS } from "../"
 import * as helpers from "./helpers"
 
-const { mocks } = require("@budibase/backend-core/tests")
-const timestamp = mocks.date.MOCK_DATE.toISOString()
+import tk from "timekeeper"
+const timestamp = new Date().toISOString()
+tk.freeze(timestamp)
 
 const clearMigrations = async () => {
   const dbs = [context.getDevAppDB(), context.getProdAppDB()]
@@ -92,24 +93,16 @@ describe("migrations", () => {
       await clearMigrations()
       const appId = config.prodAppId
       const roles = { [appId]: "role_12345" }
-      await config.createUser(
-        undefined,
-        undefined,
-        undefined,
-        undefined,
-        false,
-        true,
-        roles
-      ) // admin only
-      await config.createUser(
-        undefined,
-        undefined,
-        undefined,
-        undefined,
-        false,
-        false,
-        roles
-      ) // non admin non builder
+      await config.createUser({
+        builder: false,
+        admin: true,
+        roles,
+      }) // admin only
+      await config.createUser({
+        builder: false,
+        admin: false,
+        roles,
+      }) // non admin non builder
       await config.createTable()
       await config.createRow()
       await config.createRow()

package/src/sdk/app/backups/exports.ts

@@ -45,12 +45,18 @@ function tarFilesToTmp(tmpDir: string, files: string[]) {
  * @return {*} either a readable stream or a string
  */
 export async function exportDB(dbName: string, opts: ExportOpts = {}) {
+  const exportOpts = {
+    filter: opts?.filter,
+    batch_size: 1000,
+    batch_limit: 5,
+    style: "main_only",
+  }
   return dbCore.doWithDB(dbName, async (db: any) => {
     // Write the dump to file if required
     if (opts?.exportPath) {
       const path = opts?.exportPath
       const writeStream = fs.createWriteStream(path)
-      await db.dump(writeStream, { filter: opts?.filter })
+      await db.dump(writeStream, exportOpts)
       return path
     } else {
       // Stringify the dump in memory if required
@@ -59,7 +65,7 @@ export async function exportDB(dbName: string, opts: ExportOpts = {}) {
       memStream.on("data", (chunk: any) => {
         appString += chunk.toString()
       })
-      await db.dump(memStream, { filter: opts?.filter })
+      await db.dump(memStream, exportOpts)
       return appString
     }
   })

package/src/sdk/app/datasources/datasources.ts

@@ -88,6 +88,9 @@ export async function removeSecrets(datasources: Datasource[]) {
   const definitions = await getDefinitions()
   for (let datasource of datasources) {
     const schema = definitions[datasource.source]
+    if (!schema) {
+      continue
+    }
     if (datasource.config) {
       // strip secrets from response, so they don't show in the network request
       if (datasource.config.auth) {

package/src/tests/jestEnv.ts

@@ -3,11 +3,7 @@ import { tmpdir } from "os"
 
 env._set("SELF_HOSTED", "1")
 env._set("NODE_ENV", "jest")
-env._set("JWT_SECRET", "test-jwtsecret")
-env._set("CLIENT_ID", "test-client-id")
+env._set("MULTI_TENANCY", "1")
+// @ts-ignore
 env._set("BUDIBASE_DIR", tmpdir("budibase-unittests"))
 env._set("LOG_LEVEL", "silent")
-env._set("PORT", 0)
-env._set("MINIO_URL", "http://localhost")
-env._set("MINIO_ACCESS_KEY", "test")
-env._set("MINIO_SECRET_KEY", "test")

package/src/tests/jestSetup.ts

@@ -1,9 +1,6 @@
-import { mocks } from "@budibase/backend-core/tests"
-
-// mock all dates to 2020-01-01T00:00:00.000Z
-// use tk.reset() to use real dates in individual tests
-const tk = require("timekeeper")
-tk.freeze(mocks.date.MOCK_DATE)
+import env from "../environment"
+import { env as coreEnv } from "@budibase/backend-core"
+import { testContainerUtils } from "@budibase/backend-core/tests"
 
 if (!process.env.DEBUG) {
   global.console.log = jest.fn() // console.log are ignored in tests
@@ -13,5 +10,9 @@ if (!process.env.DEBUG) {
 if (!process.env.CI) {
   // set a longer timeout in dev for debugging
   // 100 seconds
-  jest.setTimeout(100000)
+  jest.setTimeout(100 * 1000)
+} else {
+  jest.setTimeout(10 * 1000)
 }
+
+testContainerUtils.setupEnv(env, coreEnv)

package/src/tests/utilities/TestConfiguration.ts

@@ -1,4 +1,4 @@
-import { mocks } from "@budibase/backend-core/tests"
+import { generator, mocks, structures } from "@budibase/backend-core/tests"
 
 // init the licensing mock
 import * as pro from "@budibase/pro"
@@ -10,6 +10,7 @@ mocks.licenses.useUnlimited()
 import { init as dbInit } from "../../db"
 dbInit()
 import env from "../../environment"
+import { env as coreEnv } from "@budibase/backend-core"
 import {
   basicTable,
   basicRow,
@@ -38,17 +39,27 @@ import { cleanup } from "../../utilities/fileSystem"
 import newid from "../../db/newid"
 import { generateUserMetadataID } from "../../db/utils"
 import { startup } from "../../startup"
-const supertest = require("supertest")
-
-const GLOBAL_USER_ID = "us_uuid1"
-const EMAIL = "babs@babs.com"
-const FIRSTNAME = "Barbara"
-const LASTNAME = "Barbington"
-const CSRF_TOKEN = "e3727778-7af0-4226-b5eb-f43cbe60a306"
+import supertest from "supertest"
+import {
+  AuthToken,
+  Database,
+  Datasource,
+  Row,
+  SourceName,
+  Table,
+} from "@budibase/types"
+
+type DefaultUserValues = {
+  globalUserId: string
+  email: string
+  firstName: string
+  lastName: string
+  csrfToken: string
+}
 
 class TestConfiguration {
   server: any
-  request: any
+  request: supertest.SuperTest<supertest.Test> | undefined
   started: boolean
   appId: string | null
   allApps: any[]
@@ -62,6 +73,8 @@ class TestConfiguration {
   linkedTable: any
   automation: any
   datasource: any
+  tenantId: string | null
+  defaultUserValues: DefaultUserValues
 
   constructor(openServer = true) {
     if (openServer) {
@@ -76,6 +89,18 @@ class TestConfiguration {
     }
     this.appId = null
     this.allApps = []
+    this.tenantId = null
+    this.defaultUserValues = this.populateDefaultUserValues()
+  }
+
+  populateDefaultUserValues(): DefaultUserValues {
+    return {
+      globalUserId: `us_${newid()}`,
+      email: generator.email(),
+      firstName: generator.first(),
+      lastName: generator.last(),
+      csrfToken: generator.hash(),
+    }
   }
 
   getRequest() {
@@ -100,10 +125,10 @@ class TestConfiguration {
 
   getUserDetails() {
     return {
-      globalId: GLOBAL_USER_ID,
-      email: EMAIL,
-      firstName: FIRSTNAME,
-      lastName: LASTNAME,
+      globalId: this.defaultUserValues.globalUserId,
+      email: this.defaultUserValues.email,
+      firstName: this.defaultUserValues.firstName,
+      lastName: this.defaultUserValues.lastName,
     }
   }
 
@@ -111,7 +136,9 @@ class TestConfiguration {
     if (!appId) {
       appId = this.appId
     }
-    return tenancy.doInTenant(TENANT_ID, () => {
+
+    const tenant = this.getTenantId()
+    return tenancy.doInTenant(tenant, () => {
       // check if already in a context
       if (context.getAppId() == null && appId !== null) {
         return context.doInAppContext(appId, async () => {
@@ -127,6 +154,11 @@ class TestConfiguration {
 
   // use a new id as the name to avoid name collisions
   async init(appName = newid()) {
+    this.defaultUserValues = this.populateDefaultUserValues()
+    if (context.isMultiTenant()) {
+      this.tenantId = structures.tenant.id()
+    }
+
     if (!this.started) {
       await startup()
     }
@@ -149,9 +181,30 @@ class TestConfiguration {
     }
   }
 
+  // MODES
+  #setMultiTenancy = (value: boolean) => {
+    env._set("MULTI_TENANCY", value)
+    coreEnv._set("MULTI_TENANCY", value)
+  }
+
+  #setSelfHosted = (value: boolean) => {
+    env._set("SELF_HOSTED", value)
+    coreEnv._set("SELF_HOSTED", value)
+  }
+
+  modeCloud = () => {
+    this.#setSelfHosted(false)
+    this.#setMultiTenancy(true)
+  }
+
+  modeSelf = () => {
+    this.#setSelfHosted(true)
+    this.#setMultiTenancy(false)
+  }
+
   // UTILS
 
-  async _req(body: any, params: any, controlFunc: any) {
+  _req(body: any, params: any, controlFunc: any) {
     // create a fake request ctx
     const request: any = {}
     const appId = this.appId
@@ -159,7 +212,7 @@ class TestConfiguration {
     // fake cookies, we don't need them
     request.cookies = { set: () => {}, get: () => {} }
     request.config = { jwtSecret: env.JWT_SECRET }
-    request.user = { appId, tenantId: TENANT_ID }
+    request.user = { appId, tenantId: this.getTenantId() }
     request.query = {}
     request.request = {
       body,
@@ -175,15 +228,15 @@ class TestConfiguration {
 
   // USER / AUTH
   async globalUser({
-    id = GLOBAL_USER_ID,
-    firstName = FIRSTNAME,
-    lastName = LASTNAME,
+    id = this.defaultUserValues.globalUserId,
+    firstName = this.defaultUserValues.firstName,
+    lastName = this.defaultUserValues.lastName,
     builder = true,
     admin = false,
-    email = EMAIL,
+    email = this.defaultUserValues.email,
     roles,
   }: any = {}) {
-    return tenancy.doWithGlobalDB(TENANT_ID, async (db: any) => {
+    return tenancy.doWithGlobalDB(this.getTenantId(), async (db: Database) => {
       let existing
       try {
         existing = await db.get(id)
@@ -194,14 +247,14 @@ class TestConfiguration {
         _id: id,
         ...existing,
         roles: roles || {},
-        tenantId: TENANT_ID,
+        tenantId: this.getTenantId(),
         firstName,
         lastName,
       }
       await sessions.createASession(id, {
         sessionId: "sessionid",
-        tenantId: TENANT_ID,
-        csrfToken: CSRF_TOKEN,
+        tenantId: this.getTenantId(),
+        csrfToken: this.defaultUserValues.csrfToken,
       })
       if (builder) {
         user.builder = { global: true }
@@ -215,21 +268,31 @@ class TestConfiguration {
       }
       const resp = await db.put(user)
       return {
-        _rev: resp._rev,
+        _rev: resp.rev,
         ...user,
       }
     })
   }
 
   async createUser(
-    id = null,
-    firstName = FIRSTNAME,
-    lastName = LASTNAME,
-    email = EMAIL,
-    builder = true,
-    admin = false,
-    roles = {}
+    user: {
+      id?: string
+      firstName?: string
+      lastName?: string
+      email?: string
+      builder?: boolean
+      admin?: boolean
+      roles?: any
+    } = {}
   ) {
+    let { id, firstName, lastName, email, builder, admin, roles } = user
+    firstName = firstName || this.defaultUserValues.firstName
+    lastName = lastName || this.defaultUserValues.lastName
+    email = email || this.defaultUserValues.email
+    roles = roles || {}
+    if (builder == null) {
+      builder = true
+    }
     const globalId = !id ? `us_${Math.random()}` : `us_${id}`
     const resp = await this.globalUser({
       id: globalId,
@@ -264,13 +327,13 @@ class TestConfiguration {
       }
       await sessions.createASession(userId, {
         sessionId: "sessionid",
-        tenantId: TENANT_ID,
+        tenantId: this.getTenantId(),
       })
       // have to fake this
       const authObj = {
         userId,
         sessionId: "sessionid",
-        tenantId: TENANT_ID,
+        tenantId: this.getTenantId(),
       }
       const app = {
         roleId: roleId,
@@ -293,10 +356,11 @@ class TestConfiguration {
   }
 
   defaultHeaders(extras = {}) {
-    const authObj = {
-      userId: GLOBAL_USER_ID,
+    const tenantId = this.getTenantId()
+    const authObj: AuthToken = {
+      userId: this.defaultUserValues.globalUserId,
       sessionId: "sessionid",
-      tenantId: TENANT_ID,
+      tenantId,
     }
     const app = {
       roleId: roles.BUILTIN_ROLE_IDS.ADMIN,
@@ -310,15 +374,20 @@ class TestConfiguration {
         `${constants.Cookie.Auth}=${authToken}`,
         `${constants.Cookie.CurrentApp}=${appToken}`,
       ],
-      [constants.Header.CSRF_TOKEN]: CSRF_TOKEN,
+      [constants.Header.CSRF_TOKEN]: this.defaultUserValues.csrfToken,
       ...extras,
     }
+
     if (this.appId) {
       headers[constants.Header.APP_ID] = this.appId
     }
     return headers
   }
 
+  getTenantId() {
+    return this.tenantId || TENANT_ID
+  }
+
   publicHeaders({ prodApp = true } = {}) {
     const appId = prodApp ? this.prodAppId : this.appId
 
@@ -328,11 +397,16 @@ class TestConfiguration {
     if (appId) {
       headers[constants.Header.APP_ID] = appId
     }
+
+    if (this.tenantId) {
+      headers[constants.Header.TENANT_ID] = this.tenantId
+    }
+
     return headers
   }
 
   async roleHeaders({
-    email = EMAIL,
+    email = this.defaultUserValues.email,
     roleId = roles.BUILTIN_ROLE_IDS.ADMIN,
     builder = false,
     prodApp = true,
@@ -342,8 +416,8 @@ class TestConfiguration {
 
   // API
 
-  async generateApiKey(userId = GLOBAL_USER_ID) {
-    return tenancy.doWithGlobalDB(TENANT_ID, async (db: any) => {
+  async generateApiKey(userId = this.defaultUserValues.globalUserId) {
+    return tenancy.doWithGlobalDB(this.getTenantId(), async (db: any) => {
       const id = dbCore.generateDevInfoID(userId)
       let devInfo
       try {
@@ -352,7 +426,7 @@ class TestConfiguration {
         devInfo = { _id: id, userId }
       }
       devInfo.apiKey = encryption.encrypt(
-        `${TENANT_ID}${dbCore.SEPARATOR}${newid()}`
+        `${this.getTenantId()}${dbCore.SEPARATOR}${newid()}`
       )
       await db.put(devInfo)
       return devInfo.apiKey
@@ -408,13 +482,13 @@ class TestConfiguration {
 
   // TABLE
 
-  async updateTable(config?: any) {
+  async updateTable(config?: any): Promise<Table> {
     config = config || basicTable()
     this.table = await this._req(config, null, controllers.table.save)
     return this.table
   }
 
-  async createTable(config?: any) {
+  async createTable(config?: Table) {
     if (config != null && config._id) {
       delete config._id
     }
@@ -458,7 +532,7 @@ class TestConfiguration {
 
   // ROW
 
-  async createRow(config: any = null) {
+  async createRow(config?: Row): Promise<Row> {
     if (!this.table) {
       throw "Test requires table to be configured."
     }
@@ -467,7 +541,7 @@ class TestConfiguration {
     return this._req(config, { tableId }, controllers.row.save)
   }
 
-  async getRow(tableId: string, rowId: string) {
+  async getRow(tableId: string, rowId: string): Promise<Row> {
     return this._req(null, { tableId, rowId }, controllers.row.find)
   }
 
@@ -549,7 +623,9 @@ class TestConfiguration {
 
   // DATASOURCE
 
-  async createDatasource(config?: any) {
+  async createDatasource(config?: {
+    datasource: Datasource
+  }): Promise<Datasource> {
     config = config || basicDatasource()
     const response = await this._req(config, null, controllers.datasource.save)
     this.datasource = response.datasource
@@ -570,7 +646,7 @@ class TestConfiguration {
     return this.createDatasource({
       datasource: {
         ...basicDatasource().datasource,
-        source: "REST",
+        source: SourceName.REST,
         config: cfg || {},
       },
     })
@@ -579,7 +655,7 @@ class TestConfiguration {
   async dynamicVariableDatasource() {
     let datasource = await this.restDatasource()
     const basedOnQuery = await this.createQuery({
-      ...basicQuery(datasource._id),
+      ...basicQuery(datasource._id!),
       fields: {
         path: "www.google.com",
       },
@@ -607,7 +683,7 @@ class TestConfiguration {
     datasource: any,
     fields: any,
     params: any,
-    verb: string
+    verb?: string
   ) {
     return request
       .post(`/api/queries/preview`)

package/src/tests/utilities/structures.ts

@@ -7,6 +7,8 @@ import {
   Automation,
   AutomationActionStepId,
   AutomationTriggerStepId,
+  Datasource,
+  SourceName,
 } from "@budibase/types"
 
 const { v4: uuidv4 } = require("uuid")
@@ -207,12 +209,12 @@ export function basicRole() {
   }
 }
 
-export function basicDatasource() {
+export function basicDatasource(): { datasource: Datasource } {
   return {
     datasource: {
       type: "datasource",
       name: "Test",
-      source: "POSTGRES",
+      source: SourceName.POSTGRES,
       config: {},
     },
   }
@@ -255,3 +257,15 @@ export function basicWebhook(automationId: string) {
     },
   }
 }
+
+export function basicEnvironmentVariable(
+  name: string,
+  prod: string,
+  dev?: string
+) {
+  return {
+    name,
+    production: prod,
+    development: dev || prod,
+  }
+}

package/src/utilities/retry.ts

@@ -0,0 +1,18 @@
+export async function retry<T extends (...arg0: any[]) => any>(
+  fn: T,
+  maxTry: number = 5,
+  retryCount = 1
+): Promise<Awaited<ReturnType<T>>> {
+  const currRetry = typeof retryCount === "number" ? retryCount : 1
+  try {
+    const result = await fn()
+    return result
+  } catch (e) {
+    console.log(`Retry ${currRetry} failed.`)
+    if (currRetry > maxTry) {
+      console.log(`All ${maxTry} retry attempts exhausted`)
+      throw e
+    }
+    return retry(fn, maxTry, currRetry + 1)
+  }
+}

package/tsconfig.build.json

@@ -20,6 +20,7 @@
     "dist",
     "src/tests",
     "src/api/routes/tests/utilities",
+    "src/api/routes/public/tests/utils.ts",
     "src/automations/tests/utilities",
     "**/*.spec.ts",
     "**/*.spec.js"