@budibase/server 2.2.26 → 2.2.27-alpha.1

package/src/sdk/app/datasources/index.ts

@@ -0,0 +1,5 @@
+import * as datasources from "./datasources"
+
+export default {
+  ...datasources,
+}

package/src/sdk/app/queries/index.ts

@@ -0,0 +1,5 @@
+import * as queries from "./queries"
+
+export default {
+  ...queries,
+}

package/src/sdk/app/queries/queries.ts

@@ -0,0 +1,50 @@
+import { getEnvironmentVariables } from "../../utils"
+import { processStringSync } from "@budibase/string-templates"
+
+export async function enrichContext(
+  fields: Record<string, any>,
+  inputs = {}
+): Promise<Record<string, any>> {
+  const enrichedQuery: Record<string, any> = Array.isArray(fields) ? [] : {}
+  if (!fields || !inputs) {
+    return enrichedQuery
+  }
+  const env = await getEnvironmentVariables()
+  const parameters = { ...inputs, env }
+  // enrich the fields with dynamic parameters
+  for (let key of Object.keys(fields)) {
+    if (fields[key] == null) {
+      continue
+    }
+    if (typeof fields[key] === "object") {
+      // enrich nested fields object
+      enrichedQuery[key] = await enrichContext(fields[key], parameters)
+    } else if (typeof fields[key] === "string") {
+      // enrich string value as normal
+      enrichedQuery[key] = processStringSync(fields[key], parameters, {
+        noEscaping: true,
+        noHelpers: true,
+        escapeNewlines: true,
+      })
+    } else {
+      enrichedQuery[key] = fields[key]
+    }
+  }
+  if (
+    enrichedQuery.json ||
+    enrichedQuery.customData ||
+    enrichedQuery.requestBody
+  ) {
+    try {
+      enrichedQuery.json = JSON.parse(
+        enrichedQuery.json ||
+          enrichedQuery.customData ||
+          enrichedQuery.requestBody
+      )
+    } catch (err) {
+      // no json found, ignore
+    }
+    delete enrichedQuery.customData
+  }
+  return enrichedQuery
+}

package/src/sdk/app/tables/index.ts

@@ -6,6 +6,7 @@ import {
   isSQL,
 } from "../../../integrations/utils"
 import { Table, Database } from "@budibase/types"
+import datasources from "../datasources"
 
 async function getAllInternalTables(db?: Database): Promise<Table[]> {
   if (!db) {
@@ -23,9 +24,11 @@ async function getAllInternalTables(db?: Database): Promise<Table[]> {
   }))
 }
 
-async function getAllExternalTables(datasourceId: any): Promise<Table[]> {
+async function getAllExternalTables(
+  datasourceId: any
+): Promise<Record<string, Table>> {
   const db = context.getAppDB()
-  const datasource = await db.get(datasourceId)
+  const datasource = await datasources.get(datasourceId, { enriched: true })
   if (!datasource || !datasource.entities) {
     throw "Datasource is not configured fully."
   }
@@ -44,7 +47,7 @@ async function getTable(tableId: any): Promise<Table> {
   const db = context.getAppDB()
   if (isExternalTable(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
-    const datasource = await db.get(datasourceId)
+    const datasource = await datasources.get(datasourceId!)
     const table = await getExternalTable(datasourceId, tableName)
     return { ...table, sql: isSQL(datasource) }
   } else {

package/src/sdk/index.ts

@@ -2,6 +2,8 @@ import { default as backups } from "./app/backups"
 import { default as tables } from "./app/tables"
 import { default as automations } from "./app/automations"
 import { default as applications } from "./app/applications"
+import { default as datasources } from "./app/datasources"
+import { default as queries } from "./app/queries"
 import { default as rows } from "./app/rows"
 import { default as users } from "./users"
 
@@ -12,6 +14,8 @@ const sdk = {
   applications,
   rows,
   users,
+  datasources,
+  queries,
 }
 
 // default export for TS

package/src/sdk/tests/attachments.spec.ts

@@ -69,10 +69,7 @@ describe("should be able to re-write attachment URLs", () => {
     const finalRows = await sdk.rows.getAllInternalRows(db.name)
     for (let rowToCheck of finalRows) {
       expect(rowToCheck.otherCol).toBe(row.otherCol)
-      expect(rowToCheck.photo[0].url).not.toBe(row.photo[0].url)
-      expect(rowToCheck.photo[0].url).toBe(
-        `/prod-budi-app-assets/${db.name}/attachments/a.png`
-      )
+      expect(rowToCheck.photo[0].url).toBe("")
       expect(rowToCheck.photo[0].key).toBe(`${db.name}/attachments/a.png`)
     }
   })

package/src/sdk/utils/index.ts

@@ -0,0 +1,16 @@
+import { environmentVariables } from "@budibase/pro"
+import { context, db as dbCore } from "@budibase/backend-core"
+import { AppEnvironment } from "@budibase/types"
+
+export async function getEnvironmentVariables() {
+  let envVars = context.getEnvironmentVariables()
+  if (!envVars) {
+    const appId = context.getAppId()
+    const appEnv = dbCore.isDevAppID(appId)
+      ? AppEnvironment.DEVELOPMENT
+      : AppEnvironment.PRODUCTION
+
+    envVars = await environmentVariables.fetchValues(appEnv)
+  }
+  return envVars
+}

package/src/startup.ts

@@ -1,16 +1,11 @@
-import * as env from "./environment"
+import env from "./environment"
 import * as redis from "./utilities/redis"
 import {
   createAdminUser,
   generateApiKey,
   getChecklist,
 } from "./utilities/workerRequests"
-import {
-  installation,
-  pinoSettings,
-  tenancy,
-  logging,
-} from "@budibase/backend-core"
+import { installation, tenancy, logging } from "@budibase/backend-core"
 import fs from "fs"
 import { watch } from "./watch"
 import * as automations from "./automations"
@@ -26,7 +21,7 @@ const pino = require("koa-pino-logger")
 let STARTUP_RAN = false
 
 async function initRoutes(app: any) {
-  app.use(pino(pinoSettings()))
+  app.use(pino(logging.pinoSettings()))
 
   if (!env.isTest()) {
     const plugin = await bullboard.init()

package/src/tests/jestEnv.ts

@@ -0,0 +1,9 @@
+import env from "../environment"
+import { tmpdir } from "os"
+
+env._set("SELF_HOSTED", "1")
+env._set("NODE_ENV", "jest")
+env._set("MULTI_TENANCY", "1")
+// @ts-ignore
+env._set("BUDIBASE_DIR", tmpdir("budibase-unittests"))
+env._set("LOG_LEVEL", "silent")

package/src/tests/jestSetup.ts

@@ -1,28 +1,16 @@
-const { tmpdir } = require("os")
 import env from "../environment"
+import { env as coreEnv } from "@budibase/backend-core"
+import { testContainerUtils } from "@budibase/backend-core/tests"
 
-env._set("SELF_HOSTED", "1")
-env._set("NODE_ENV", "jest")
-env._set("JWT_SECRET", "test-jwtsecret")
-env._set("CLIENT_ID", "test-client-id")
-env._set("BUDIBASE_DIR", tmpdir("budibase-unittests"))
-env._set("LOG_LEVEL", "silent")
-env._set("PORT", 0)
-env._set("MINIO_URL", "http://localhost")
-env._set("MINIO_ACCESS_KEY", "test")
-env._set("MINIO_SECRET_KEY", "test")
-
-import { mocks } from "@budibase/backend-core/tests"
-
-// mock all dates to 2020-01-01T00:00:00.000Z
-// use tk.reset() to use real dates in individual tests
-const tk = require("timekeeper")
-tk.freeze(mocks.date.MOCK_DATE)
-
-global.console.log = jest.fn() // console.log are ignored in tests
+if (!process.env.DEBUG) {
+  global.console.log = jest.fn() // console.log are ignored in tests
+  global.console.warn = jest.fn() // console.warn are ignored in tests
+}
 
 if (!process.env.CI) {
   // set a longer timeout in dev for debugging
   // 100 seconds
   jest.setTimeout(100000)
 }
+
+testContainerUtils.setupEnv(env, coreEnv)

package/src/tests/utilities/TestConfiguration.ts

@@ -1,6 +1,16 @@
+import { generator, mocks, structures } from "@budibase/backend-core/tests"
+
+// init the licensing mock
+import * as pro from "@budibase/pro"
+mocks.licenses.init(pro)
+
+// use unlimited license by default
+mocks.licenses.useUnlimited()
+
 import { init as dbInit } from "../../db"
 dbInit()
 import env from "../../environment"
+import { env as coreEnv } from "@budibase/backend-core"
 import {
   basicTable,
   basicRow,
@@ -29,13 +39,16 @@ import { cleanup } from "../../utilities/fileSystem"
 import newid from "../../db/newid"
 import { generateUserMetadataID } from "../../db/utils"
 import { startup } from "../../startup"
+import { AuthToken, Database } from "@budibase/types"
 const supertest = require("supertest")
 
-const GLOBAL_USER_ID = "us_uuid1"
-const EMAIL = "babs@babs.com"
-const FIRSTNAME = "Barbara"
-const LASTNAME = "Barbington"
-const CSRF_TOKEN = "e3727778-7af0-4226-b5eb-f43cbe60a306"
+type DefaultUserValues = {
+  globalUserId: string
+  email: string
+  firstName: string
+  lastName: string
+  csrfToken: string
+}
 
 class TestConfiguration {
   server: any
@@ -53,12 +66,14 @@ class TestConfiguration {
   linkedTable: any
   automation: any
   datasource: any
+  tenantId: string | null
+  defaultUserValues: DefaultUserValues
 
   constructor(openServer = true) {
     if (openServer) {
       // use a random port because it doesn't matter
       env.PORT = "0"
-      this.server = require("../../app")
+      this.server = require("../../app").default
       // we need the request for logging in, involves cookies, hard to fake
       this.request = supertest(this.server)
       this.started = true
@@ -67,6 +82,18 @@ class TestConfiguration {
     }
     this.appId = null
     this.allApps = []
+    this.tenantId = null
+    this.defaultUserValues = this.populateDefaultUserValues()
+  }
+
+  populateDefaultUserValues(): DefaultUserValues {
+    return {
+      globalUserId: `us_${newid()}`,
+      email: generator.email(),
+      firstName: generator.first(),
+      lastName: generator.last(),
+      csrfToken: generator.hash(),
+    }
   }
 
   getRequest() {
@@ -91,10 +118,10 @@ class TestConfiguration {
 
   getUserDetails() {
     return {
-      globalId: GLOBAL_USER_ID,
-      email: EMAIL,
-      firstName: FIRSTNAME,
-      lastName: LASTNAME,
+      globalId: this.defaultUserValues.globalUserId,
+      email: this.defaultUserValues.email,
+      firstName: this.defaultUserValues.firstName,
+      lastName: this.defaultUserValues.lastName,
     }
   }
 
@@ -102,7 +129,9 @@ class TestConfiguration {
     if (!appId) {
       appId = this.appId
     }
-    return tenancy.doInTenant(TENANT_ID, () => {
+
+    const tenant = this.getTenantId()
+    return tenancy.doInTenant(tenant, () => {
       // check if already in a context
       if (context.getAppId() == null && appId !== null) {
         return context.doInAppContext(appId, async () => {
@@ -118,6 +147,11 @@ class TestConfiguration {
 
   // use a new id as the name to avoid name collisions
   async init(appName = newid()) {
+    this.defaultUserValues = this.populateDefaultUserValues()
+    if (context.isMultiTenant()) {
+      this.tenantId = structures.tenant.id()
+    }
+
     if (!this.started) {
       await startup()
     }
@@ -140,6 +174,27 @@ class TestConfiguration {
     }
   }
 
+  // MODES
+  #setMultiTenancy = (value: boolean) => {
+    env._set("MULTI_TENANCY", value)
+    coreEnv._set("MULTI_TENANCY", value)
+  }
+
+  #setSelfHosted = (value: boolean) => {
+    env._set("SELF_HOSTED", value)
+    coreEnv._set("SELF_HOSTED", value)
+  }
+
+  modeCloud = () => {
+    this.#setSelfHosted(false)
+    this.#setMultiTenancy(true)
+  }
+
+  modeSelf = () => {
+    this.#setSelfHosted(true)
+    this.#setMultiTenancy(false)
+  }
+
   // UTILS
 
   async _req(body: any, params: any, controlFunc: any) {
@@ -150,7 +205,7 @@ class TestConfiguration {
     // fake cookies, we don't need them
     request.cookies = { set: () => {}, get: () => {} }
     request.config = { jwtSecret: env.JWT_SECRET }
-    request.user = { appId, tenantId: TENANT_ID }
+    request.user = { appId, tenantId: this.getTenantId() }
     request.query = {}
     request.request = {
       body,
@@ -166,15 +221,15 @@ class TestConfiguration {
 
   // USER / AUTH
   async globalUser({
-    id = GLOBAL_USER_ID,
-    firstName = FIRSTNAME,
-    lastName = LASTNAME,
+    id = this.defaultUserValues.globalUserId,
+    firstName = this.defaultUserValues.firstName,
+    lastName = this.defaultUserValues.lastName,
     builder = true,
     admin = false,
-    email = EMAIL,
+    email = this.defaultUserValues.email,
     roles,
   }: any = {}) {
-    return tenancy.doWithGlobalDB(TENANT_ID, async (db: any) => {
+    return tenancy.doWithGlobalDB(this.getTenantId(), async (db: Database) => {
       let existing
       try {
         existing = await db.get(id)
@@ -185,14 +240,14 @@ class TestConfiguration {
         _id: id,
         ...existing,
         roles: roles || {},
-        tenantId: TENANT_ID,
+        tenantId: this.getTenantId(),
         firstName,
         lastName,
       }
       await sessions.createASession(id, {
         sessionId: "sessionid",
-        tenantId: TENANT_ID,
-        csrfToken: CSRF_TOKEN,
+        tenantId: this.getTenantId(),
+        csrfToken: this.defaultUserValues.csrfToken,
       })
       if (builder) {
         user.builder = { global: true }
@@ -206,7 +261,7 @@ class TestConfiguration {
       }
       const resp = await db.put(user)
       return {
-        _rev: resp._rev,
+        _rev: resp.rev,
         ...user,
       }
     })
@@ -214,9 +269,9 @@ class TestConfiguration {
 
   async createUser(
     id = null,
-    firstName = FIRSTNAME,
-    lastName = LASTNAME,
-    email = EMAIL,
+    firstName = this.defaultUserValues.firstName,
+    lastName = this.defaultUserValues.lastName,
+    email = this.defaultUserValues.email,
     builder = true,
     admin = false,
     roles = {}
@@ -255,13 +310,13 @@ class TestConfiguration {
       }
       await sessions.createASession(userId, {
         sessionId: "sessionid",
-        tenantId: TENANT_ID,
+        tenantId: this.getTenantId(),
       })
       // have to fake this
       const authObj = {
         userId,
         sessionId: "sessionid",
-        tenantId: TENANT_ID,
+        tenantId: this.getTenantId(),
       }
       const app = {
         roleId: roleId,
@@ -284,10 +339,11 @@ class TestConfiguration {
   }
 
   defaultHeaders(extras = {}) {
-    const authObj = {
-      userId: GLOBAL_USER_ID,
+    const tenantId = this.getTenantId()
+    const authObj: AuthToken = {
+      userId: this.defaultUserValues.globalUserId,
       sessionId: "sessionid",
-      tenantId: TENANT_ID,
+      tenantId,
     }
     const app = {
       roleId: roles.BUILTIN_ROLE_IDS.ADMIN,
@@ -301,7 +357,7 @@ class TestConfiguration {
         `${constants.Cookie.Auth}=${authToken}`,
         `${constants.Cookie.CurrentApp}=${appToken}`,
       ],
-      [constants.Header.CSRF_TOKEN]: CSRF_TOKEN,
+      [constants.Header.CSRF_TOKEN]: this.defaultUserValues.csrfToken,
       ...extras,
     }
     if (this.appId) {
@@ -310,6 +366,10 @@ class TestConfiguration {
     return headers
   }
 
+  getTenantId() {
+    return this.tenantId || TENANT_ID
+  }
+
   publicHeaders({ prodApp = true } = {}) {
     const appId = prodApp ? this.prodAppId : this.appId
 
@@ -319,11 +379,16 @@ class TestConfiguration {
     if (appId) {
       headers[constants.Header.APP_ID] = appId
     }
+
+    if (this.tenantId) {
+      headers[constants.Header.TENANT_ID] = this.tenantId
+    }
+
     return headers
   }
 
   async roleHeaders({
-    email = EMAIL,
+    email = this.defaultUserValues.email,
     roleId = roles.BUILTIN_ROLE_IDS.ADMIN,
     builder = false,
     prodApp = true,
@@ -333,8 +398,8 @@ class TestConfiguration {
 
   // API
 
-  async generateApiKey(userId = GLOBAL_USER_ID) {
-    return tenancy.doWithGlobalDB(TENANT_ID, async (db: any) => {
+  async generateApiKey(userId = this.defaultUserValues.globalUserId) {
+    return tenancy.doWithGlobalDB(this.getTenantId(), async (db: any) => {
       const id = dbCore.generateDevInfoID(userId)
       let devInfo
       try {
@@ -343,7 +408,7 @@ class TestConfiguration {
         devInfo = { _id: id, userId }
       }
       devInfo.apiKey = encryption.encrypt(
-        `${TENANT_ID}${dbCore.SEPARATOR}${newid()}`
+        `${this.getTenantId()}${dbCore.SEPARATOR}${newid()}`
       )
       await db.put(devInfo)
       return devInfo.apiKey
@@ -351,29 +416,31 @@ class TestConfiguration {
   }
 
   // APP
-
   async createApp(appName: string) {
     // create dev app
     // clear any old app
     this.appId = null
-    // @ts-ignore
-    await context.updateAppId(null)
-    this.app = await this._req({ name: appName }, null, controllers.app.create)
-    this.appId = this.app.appId
-    // @ts-ignore
-    await context.updateAppId(this.appId)
-
-    // create production app
-    this.prodApp = await this.deploy()
+    await context.doInAppContext(null, async () => {
+      this.app = await this._req(
+        { name: appName },
+        null,
+        controllers.app.create
+      )
+      this.appId = this.app.appId
+    })
+    return await context.doInAppContext(this.appId, async () => {
+      // create production app
+      this.prodApp = await this.publish()
 
-    this.allApps.push(this.prodApp)
-    this.allApps.push(this.app)
+      this.allApps.push(this.prodApp)
+      this.allApps.push(this.app)
 
-    return this.app
+      return this.app
+    })
   }
 
-  async deploy() {
-    await this._req(null, null, controllers.deploy.deployApp)
+  async publish() {
+    await this._req(null, null, controllers.deploy.publishApp)
     // @ts-ignore
     const prodAppId = this.getAppId().replace("_dev", "")
     this.prodAppId = prodAppId
@@ -384,6 +451,17 @@ class TestConfiguration {
     })
   }
 
+  async unpublish() {
+    const response = await this._req(
+      null,
+      { appId: this.appId },
+      controllers.app.unpublish
+    )
+    this.prodAppId = null
+    this.prodApp = null
+    return response
+  }
+
   // TABLE
 
   async updateTable(config?: any) {

package/src/tests/utilities/structures.ts

@@ -1,8 +1,14 @@
-import { roles, permissions } from "@budibase/backend-core"
+import { permissions, roles } from "@budibase/backend-core"
 import { createHomeScreen } from "../../constants/screens"
 import { EMPTY_LAYOUT } from "../../constants/layouts"
 import { cloneDeep } from "lodash/fp"
-import { TRIGGER_DEFINITIONS, ACTION_DEFINITIONS } from "../../automations"
+import { ACTION_DEFINITIONS, TRIGGER_DEFINITIONS } from "../../automations"
+import {
+  Automation,
+  AutomationActionStepId,
+  AutomationTriggerStepId,
+} from "@budibase/types"
+
 const { v4: uuidv4 } = require("uuid")
 
 export const TENANT_ID = "default"
@@ -116,6 +122,63 @@ export function basicAutomation() {
   }
 }
 
+export function loopAutomation(tableId: string, loopOpts?: any): Automation {
+  if (!loopOpts) {
+    loopOpts = {
+      option: "Array",
+      binding: "{{ steps.1.rows }}",
+    }
+  }
+  const automation: any = {
+    name: "looping",
+    type: "automation",
+    definition: {
+      steps: [
+        {
+          id: "b",
+          type: "ACTION",
+          stepId: AutomationActionStepId.QUERY_ROWS,
+          internal: true,
+          inputs: {
+            tableId,
+          },
+          schema: ACTION_DEFINITIONS.QUERY_ROWS.schema,
+        },
+        {
+          id: "c",
+          type: "ACTION",
+          stepId: AutomationActionStepId.LOOP,
+          internal: true,
+          inputs: loopOpts,
+          blockToLoop: "d",
+          schema: ACTION_DEFINITIONS.LOOP.schema,
+        },
+        {
+          id: "d",
+          type: "ACTION",
+          internal: true,
+          stepId: AutomationActionStepId.SERVER_LOG,
+          inputs: {
+            text: "log statement",
+          },
+          schema: ACTION_DEFINITIONS.SERVER_LOG.schema,
+        },
+      ],
+      trigger: {
+        id: "a",
+        type: "TRIGGER",
+        event: "row:save",
+        stepId: AutomationTriggerStepId.ROW_SAVED,
+        inputs: {
+          tableId,
+        },
+        schema: TRIGGER_DEFINITIONS.ROW_SAVED.schema,
+      },
+    },
+  }
+  return automation as Automation
+}
+
 export function basicRow(tableId: string) {
   return {
     name: "Test Contact",