@budibase/server 2.2.12-alpha.5 → 2.2.12-alpha.51

package/src/integrations/rest.ts

@@ -6,21 +6,19 @@ import {
   IntegrationBase,
   PaginationValues,
   RestQueryFields as RestQuery,
-} from "@budibase/types"
-import {
   RestConfig,
-  AuthType,
-  BasicAuthConfig,
-  BearerAuthConfig,
-} from "../definitions/datasource"
+  RestAuthType,
+  RestBasicAuthConfig,
+  RestBearerAuthConfig,
+} from "@budibase/types"
 import { get } from "lodash"
 import * as https from "https"
 import qs from "querystring"
-const fetch = require("node-fetch")
-const { formatBytes } = require("../utilities")
-const { performance } = require("perf_hooks")
-const FormData = require("form-data")
-const { URLSearchParams } = require("url")
+import fetch from "node-fetch"
+import { formatBytes } from "../utilities"
+import { performance } from "perf_hooks"
+import FormData from "form-data"
+import { URLSearchParams } from "url"
 
 const BodyTypes = {
   NONE: "none",
@@ -204,12 +202,12 @@ class RestIntegration implements IntegrationBase {
 
       // Append page number or cursor param if configured
       if (pageParam && paginationValues.page != null) {
-        params.append(pageParam, paginationValues.page)
+        params.append(pageParam, paginationValues.page as string)
       }
 
       // Append page size param if configured
       if (sizeParam && paginationValues.limit != null) {
-        params.append(sizeParam, paginationValues.limit)
+        params.append(sizeParam, String(paginationValues.limit))
       }
 
       // Prepend query string with pagination params
@@ -280,7 +278,7 @@ class RestIntegration implements IntegrationBase {
       case BodyTypes.ENCODED:
         const params = new URLSearchParams()
         for (let [key, value] of Object.entries(object)) {
-          params.append(key, value)
+          params.append(key, value as string)
         }
         addPaginationToBody((key: string, value: any) => {
           params.append(key, value)
@@ -331,14 +329,14 @@ class RestIntegration implements IntegrationBase {
       if (authConfig) {
         let config
         switch (authConfig.type) {
-          case AuthType.BASIC:
-            config = authConfig.config as BasicAuthConfig
+          case RestAuthType.BASIC:
+            config = authConfig.config as RestBasicAuthConfig
             headers.Authorization = `Basic ${Buffer.from(
               `${config.username}:${config.password}`
             ).toString("base64")}`
             break
-          case AuthType.BEARER:
-            config = authConfig.config as BearerAuthConfig
+          case RestAuthType.BEARER:
+            config = authConfig.config as RestBearerAuthConfig
             headers.Authorization = `Bearer ${config.token}`
             break
         }
@@ -428,5 +426,4 @@ class RestIntegration implements IntegrationBase {
 export default {
   schema: SCHEMA,
   integration: RestIntegration,
-  AuthType,
 }

package/src/integrations/tests/rest.spec.ts

@@ -1,18 +1,21 @@
-const mockFetch = jest.fn(() => ({
-  headers: {
-    raw: () => {
-      return { "content-type": ["application/json"] }
+jest.mock("node-fetch", () => {
+  return jest.fn(() => ({
+    headers: {
+      raw: () => {
+        return { "content-type": ["application/json"] }
+      },
+      get: () => ["application/json"],
     },
-    get: () => ["application/json"],
-  },
-  json: jest.fn(() => ({
-    my_next_cursor: 123,
-  })),
-  text: jest.fn(),
-}))
-jest.mock("node-fetch", () => mockFetch)
+    json: jest.fn(() => ({
+      my_next_cursor: 123,
+    })),
+    text: jest.fn(),
+  }))
+})
+
 import fetch from "node-fetch"
 import { default as RestIntegration } from "../rest"
+import { RestAuthType } from "@budibase/types"
 const FormData = require("form-data")
 const { URLSearchParams } = require("url")
 
@@ -227,7 +230,7 @@ describe("REST Integration", () => {
     const basicAuth = {
       _id: "c59c14bd1898a43baa08da68959b24686",
       name: "basic-1",
-      type: RestIntegration.AuthType.BASIC,
+      type: RestAuthType.BASIC,
       config: {
         username: "user",
         password: "password",
@@ -237,7 +240,7 @@ describe("REST Integration", () => {
     const bearerAuth = {
       _id: "0d91d732f34e4befabeff50b392a8ff3",
       name: "bearer-1",
-      type: RestIntegration.AuthType.BEARER,
+      type: RestAuthType.BEARER,
       config: {
         token: "mytoken",
       },
@@ -579,7 +582,8 @@ describe("REST Integration", () => {
     })
     await config.integration.read({})
 
-    const calls: any = mockFetch.mock.calls[0]
+    // @ts-ignore
+    const calls: any = fetch.mock.calls[0]
     const url = calls[0]
     expect(url).toBe(`${BASE_URL}/`)
 

package/src/middleware/authorized.ts

@@ -63,7 +63,11 @@ const checkAuthorizedResource = async (
   }
 }
 
-export = (permType: any, permLevel: any = null, opts = { schema: false }) =>
+export default (
+    permType: any,
+    permLevel: any = null,
+    opts = { schema: false }
+  ) =>
   async (ctx: any, next: any) => {
     // webhooks don't need authentication, each webhook unique
     // also internal requests (between services) don't need authorized

package/src/middleware/builder.ts

@@ -64,7 +64,7 @@ async function updateAppUpdatedAt(ctx: BBContext) {
   })
 }
 
-export = async function builder(ctx: BBContext, permType: string) {
+export default async function builder(ctx: BBContext, permType: string) {
   const appId = ctx.appId
   // this only functions within an app context
   if (!appId) {

package/src/middleware/currentapp.ts

@@ -12,7 +12,7 @@ import env from "../environment"
 import { isWebhookEndpoint } from "./utils"
 import { BBContext } from "@budibase/types"
 
-export = async (ctx: BBContext, next: any) => {
+export default async (ctx: BBContext, next: any) => {
   // try to get the appID from the request
   let requestAppId = await utils.getAppIdFromCtx(ctx)
   // get app cookie if it exists
@@ -25,6 +25,7 @@ export = async (ctx: BBContext, next: any) => {
   if (!appCookie && !requestAppId) {
     return next()
   }
+
   // check the app exists referenced in cookie
   if (appCookie) {
     const appId = appCookie.appId
@@ -51,7 +52,7 @@ export = async (ctx: BBContext, next: any) => {
 
   let appId: string | undefined,
     roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
-  if (!ctx.user) {
+  if (!ctx.user?._id) {
     // not logged in, try to set a cookie for public apps
     appId = requestAppId
   } else if (requestAppId != null) {
@@ -96,7 +97,7 @@ export = async (ctx: BBContext, next: any) => {
     // need to judge this only based on the request app ID,
     if (
       env.MULTI_TENANCY &&
-      ctx.user &&
+      ctx.user?._id &&
       requestAppId &&
       !tenancy.isUserInAppTenant(requestAppId, ctx.user)
     ) {

package/src/middleware/publicApi.ts

@@ -1,7 +1,7 @@
 import { constants, utils } from "@budibase/backend-core"
 import { BBContext } from "@budibase/types"
 
-export = function ({ requiresAppId }: { requiresAppId?: boolean } = {}) {
+export default function ({ requiresAppId }: { requiresAppId?: boolean } = {}) {
   return async (ctx: BBContext, next: any) => {
     const appId = await utils.getAppIdFromCtx(ctx)
     if (requiresAppId && !appId) {

package/src/middleware/selfhost.ts

@@ -3,7 +3,7 @@ import { BBContext } from "@budibase/types"
 
 // if added as a middleware will stop requests unless builder is in self host mode
 // or cloud is in self host
-export = async (ctx: BBContext, next: any) => {
+export default async (ctx: BBContext, next: any) => {
   if (env.SELF_HOSTED) {
     await next()
     return

package/src/middleware/tests/authorized.spec.js

@@ -7,7 +7,7 @@ jest.mock("../../environment", () => ({
     }
   })
 )
-const authorizedMiddleware = require("../authorized")
+const authorizedMiddleware = require("../authorized").default
 const env = require("../../environment")
 const { permissions } = require("@budibase/backend-core")
 

package/src/middleware/tests/currentapp.spec.js

@@ -124,7 +124,7 @@ class TestConfiguration {
     // import as late as possible for mocks
     jest.resetModules()
     require("../../db").init()
-    const currentAppMiddleware = require("../currentapp")
+    const currentAppMiddleware = require("../currentapp").default
     return currentAppMiddleware(this.ctx, this.next)
   }
 }

package/src/middleware/tests/selfhost.spec.js

@@ -1,4 +1,4 @@
-const selfHostMiddleware = require("../selfhost")
+const selfHostMiddleware = require("../selfhost").default
 const env = require("../../environment")
 jest.mock("../../environment")
 

package/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts

@@ -2,7 +2,7 @@ import TestConfig from "../../../../tests/utilities/TestConfiguration"
 import * as syncRows from "../syncRows"
 import { quotas } from "@budibase/pro"
 import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
-const { db: dbCore } = require("@budibase/backend-core")
+import { db as dbCore, context } from "@budibase/backend-core"
 
 describe("syncRows", () => {
   let config = new TestConfig(false)
@@ -24,13 +24,17 @@ describe("syncRows", () => {
 
       // app 1
       const app1 = config.app
-      await config.createTable()
-      await config.createRow()
+      await context.doInAppContext(app1.appId, async () => {
+        await config.createTable()
+        await config.createRow()
+      })
       // app 2
       const app2 = await config.createApp("second-app")
-      await config.createTable()
-      await config.createRow()
-      await config.createRow()
+      await context.doInAppContext(app2.appId, async () => {
+        await config.createTable()
+        await config.createRow()
+        await config.createRow()
+      })
 
       // migrate
       await syncRows.run()

package/src/sdk/app/datasources/datasources.ts

@@ -0,0 +1,136 @@
+import { context } from "@budibase/backend-core"
+import { findHBSBlocks, processObjectSync } from "@budibase/string-templates"
+import {
+  Datasource,
+  DatasourceFieldType,
+  PASSWORD_REPLACEMENT,
+  RestAuthConfig,
+  RestAuthType,
+  RestBasicAuthConfig,
+  SourceName,
+} from "@budibase/types"
+import { cloneDeep } from "lodash/fp"
+import { getEnvironmentVariables } from "../../utils"
+import { getDefinitions } from "../../../integrations"
+
+const ENV_VAR_PREFIX = "env."
+
+async function enrichDatasourceWithValues(datasource: Datasource) {
+  const cloned = cloneDeep(datasource)
+  const env = await getEnvironmentVariables()
+  const processed = processObjectSync(cloned, { env }, { onlyFound: true })
+  return {
+    datasource: processed as Datasource,
+    envVars: env as Record<string, string>,
+  }
+}
+
+export async function enrich(datasource: Datasource) {
+  const { datasource: response } = await enrichDatasourceWithValues(datasource)
+  return response
+}
+
+export async function get(
+  datasourceId: string,
+  opts?: { enriched: boolean }
+): Promise<Datasource> {
+  const appDb = context.getAppDB()
+  const datasource = await appDb.get(datasourceId)
+  if (opts?.enriched) {
+    return (await enrichDatasourceWithValues(datasource)).datasource
+  } else {
+    return datasource
+  }
+}
+
+export async function getWithEnvVars(datasourceId: string) {
+  const appDb = context.getAppDB()
+  const datasource = await appDb.get(datasourceId)
+  return enrichDatasourceWithValues(datasource)
+}
+
+function hasAuthConfigs(datasource: Datasource) {
+  return datasource.source === SourceName.REST && datasource.config?.authConfigs
+}
+
+function useEnvVars(str: any) {
+  if (typeof str !== "string") {
+    return false
+  }
+  const blocks = findHBSBlocks(str)
+  return blocks.find(block => block.includes(ENV_VAR_PREFIX)) != null
+}
+
+export async function removeSecrets(datasources: Datasource[]) {
+  const definitions = await getDefinitions()
+  for (let datasource of datasources) {
+    const schema = definitions[datasource.source]
+    if (datasource.config) {
+      // strip secrets from response, so they don't show in the network request
+      if (datasource.config.auth) {
+        delete datasource.config.auth
+      }
+      // specific to REST datasources, contains passwords
+      if (hasAuthConfigs(datasource)) {
+        const configs = datasource.config.authConfigs as RestAuthConfig[]
+        for (let config of configs) {
+          if (config.type !== RestAuthType.BASIC) {
+            continue
+          }
+          const basic = config.config as RestBasicAuthConfig
+          if (!useEnvVars(basic.password)) {
+            basic.password = PASSWORD_REPLACEMENT
+          }
+        }
+      }
+      // remove general passwords
+      for (let key of Object.keys(datasource.config)) {
+        if (
+          schema.datasource?.[key]?.type === DatasourceFieldType.PASSWORD &&
+          !useEnvVars(datasource.config[key])
+        ) {
+          datasource.config[key] = PASSWORD_REPLACEMENT
+        }
+      }
+    }
+  }
+  return datasources
+}
+
+export async function removeSecretSingle(datasource: Datasource) {
+  return (await removeSecrets([datasource]))[0]
+}
+
+export function mergeConfigs(update: Datasource, old: Datasource) {
+  if (!update.config) {
+    return update
+  }
+  // specific to REST datasources, fix the auth configs again if required
+  if (hasAuthConfigs(update)) {
+    const configs = update.config.authConfigs as RestAuthConfig[]
+    const oldConfigs = old.config?.authConfigs as RestAuthConfig[]
+    for (let config of configs) {
+      if (config.type !== RestAuthType.BASIC) {
+        continue
+      }
+      const basic = config.config as RestBasicAuthConfig
+      const oldBasic = oldConfigs.find(old => old.name === config.name)
+        ?.config as RestBasicAuthConfig
+      if (basic.password === PASSWORD_REPLACEMENT) {
+        basic.password = oldBasic.password
+      }
+    }
+  }
+  // update back to actual passwords for everything else
+  for (let [key, value] of Object.entries(update.config)) {
+    if (value !== PASSWORD_REPLACEMENT) {
+      continue
+    }
+    if (old.config?.[key]) {
+      update.config[key] = old.config?.[key]
+    } else {
+      delete update.config[key]
+    }
+  }
+  return update
+}

package/src/sdk/app/datasources/index.ts

@@ -0,0 +1,5 @@
+import * as datasources from "./datasources"
+
+export default {
+  ...datasources,
+}

package/src/sdk/app/queries/index.ts

@@ -0,0 +1,5 @@
+import * as queries from "./queries"
+
+export default {
+  ...queries,
+}

package/src/sdk/app/queries/queries.ts

@@ -0,0 +1,50 @@
+import { getEnvironmentVariables } from "../../utils"
+import { processStringSync } from "@budibase/string-templates"
+
+export async function enrichContext(
+  fields: Record<string, any>,
+  inputs = {}
+): Promise<Record<string, any>> {
+  const enrichedQuery: Record<string, any> = Array.isArray(fields) ? [] : {}
+  if (!fields || !inputs) {
+    return enrichedQuery
+  }
+  const env = await getEnvironmentVariables()
+  const parameters = { ...inputs, env }
+  // enrich the fields with dynamic parameters
+  for (let key of Object.keys(fields)) {
+    if (fields[key] == null) {
+      continue
+    }
+    if (typeof fields[key] === "object") {
+      // enrich nested fields object
+      enrichedQuery[key] = await enrichContext(fields[key], parameters)
+    } else if (typeof fields[key] === "string") {
+      // enrich string value as normal
+      enrichedQuery[key] = processStringSync(fields[key], parameters, {
+        noEscaping: true,
+        noHelpers: true,
+        escapeNewlines: true,
+      })
+    } else {
+      enrichedQuery[key] = fields[key]
+    }
+  }
+  if (
+    enrichedQuery.json ||
+    enrichedQuery.customData ||
+    enrichedQuery.requestBody
+  ) {
+    try {
+      enrichedQuery.json = JSON.parse(
+        enrichedQuery.json ||
+          enrichedQuery.customData ||
+          enrichedQuery.requestBody
+      )
+    } catch (err) {
+      // no json found, ignore
+    }
+    delete enrichedQuery.customData
+  }
+  return enrichedQuery
+}

package/src/sdk/app/tables/index.ts

@@ -6,6 +6,7 @@ import {
   isSQL,
 } from "../../../integrations/utils"
 import { Table, Database } from "@budibase/types"
+import datasources from "../datasources"
 
 async function getAllInternalTables(db?: Database): Promise<Table[]> {
   if (!db) {
@@ -23,9 +24,11 @@ async function getAllInternalTables(db?: Database): Promise<Table[]> {
   }))
 }
 
-async function getAllExternalTables(datasourceId: any): Promise<Table[]> {
+async function getAllExternalTables(
+  datasourceId: any
+): Promise<Record<string, Table>> {
   const db = context.getAppDB()
-  const datasource = await db.get(datasourceId)
+  const datasource = await datasources.get(datasourceId, { enriched: true })
   if (!datasource || !datasource.entities) {
     throw "Datasource is not configured fully."
   }
@@ -44,7 +47,7 @@ async function getTable(tableId: any): Promise<Table> {
   const db = context.getAppDB()
   if (isExternalTable(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
-    const datasource = await db.get(datasourceId)
+    const datasource = await datasources.get(datasourceId!)
     const table = await getExternalTable(datasourceId, tableName)
     return { ...table, sql: isSQL(datasource) }
   } else {

package/src/sdk/index.ts

@@ -2,6 +2,8 @@ import { default as backups } from "./app/backups"
 import { default as tables } from "./app/tables"
 import { default as automations } from "./app/automations"
 import { default as applications } from "./app/applications"
+import { default as datasources } from "./app/datasources"
+import { default as queries } from "./app/queries"
 import { default as rows } from "./app/rows"
 import { default as users } from "./users"
 
@@ -12,6 +14,8 @@ const sdk = {
   applications,
   rows,
   users,
+  datasources,
+  queries,
 }
 
 // default export for TS

package/src/sdk/utils/index.ts

@@ -0,0 +1,16 @@
+import { environmentVariables } from "@budibase/pro"
+import { context, db as dbCore } from "@budibase/backend-core"
+import { AppEnvironment } from "@budibase/types"
+
+export async function getEnvironmentVariables() {
+  let envVars = context.getEnvironmentVariables()
+  if (!envVars) {
+    const appId = context.getAppId()
+    const appEnv = dbCore.isDevAppID(appId)
+      ? AppEnvironment.DEVELOPMENT
+      : AppEnvironment.PRODUCTION
+
+    envVars = await environmentVariables.fetchValues(appEnv)
+  }
+  return envVars
+}

package/src/startup.ts

@@ -1,16 +1,11 @@
-import * as env from "./environment"
+import env from "./environment"
 import * as redis from "./utilities/redis"
 import {
   createAdminUser,
   generateApiKey,
   getChecklist,
 } from "./utilities/workerRequests"
-import {
-  installation,
-  pinoSettings,
-  tenancy,
-  logging,
-} from "@budibase/backend-core"
+import { installation, tenancy, logging } from "@budibase/backend-core"
 import fs from "fs"
 import { watch } from "./watch"
 import * as automations from "./automations"
@@ -26,7 +21,7 @@ const pino = require("koa-pino-logger")
 let STARTUP_RAN = false
 
 async function initRoutes(app: any) {
-  app.use(pino(pinoSettings()))
+  app.use(pino(logging.pinoSettings()))
 
   if (!env.isTest()) {
     const plugin = await bullboard.init()

package/src/tests/jestEnv.ts

@@ -0,0 +1,13 @@
+import env from "../environment"
+import { tmpdir } from "os"
+
+env._set("SELF_HOSTED", "1")
+env._set("NODE_ENV", "jest")
+env._set("JWT_SECRET", "test-jwtsecret")
+env._set("CLIENT_ID", "test-client-id")
+env._set("BUDIBASE_DIR", tmpdir("budibase-unittests"))
+env._set("LOG_LEVEL", "silent")
+env._set("PORT", 0)
+env._set("MINIO_URL", "http://localhost")
+env._set("MINIO_ACCESS_KEY", "test")
+env._set("MINIO_SECRET_KEY", "test")

package/src/tests/jestSetup.ts

@@ -1,17 +1,3 @@
-const { tmpdir } = require("os")
-import env from "../environment"
-
-env._set("SELF_HOSTED", "1")
-env._set("NODE_ENV", "jest")
-env._set("JWT_SECRET", "test-jwtsecret")
-env._set("CLIENT_ID", "test-client-id")
-env._set("BUDIBASE_DIR", tmpdir("budibase-unittests"))
-env._set("LOG_LEVEL", "silent")
-env._set("PORT", 0)
-env._set("MINIO_URL", "http://localhost")
-env._set("MINIO_ACCESS_KEY", "test")
-env._set("MINIO_SECRET_KEY", "test")
-
 import { mocks } from "@budibase/backend-core/tests"
 
 // mock all dates to 2020-01-01T00:00:00.000Z

package/src/tests/utilities/TestConfiguration.ts

@@ -67,7 +67,7 @@ class TestConfiguration {
     if (openServer) {
       // use a random port because it doesn't matter
       env.PORT = "0"
-      this.server = require("../../app")
+      this.server = require("../../app").default
       // we need the request for logging in, involves cookies, hard to fake
       this.request = supertest(this.server)
       this.started = true
@@ -364,20 +364,23 @@ class TestConfiguration {
     // create dev app
     // clear any old app
     this.appId = null
-    // @ts-ignore
-    await context.updateAppId(null)
-    this.app = await this._req({ name: appName }, null, controllers.app.create)
-    this.appId = this.app.appId
-    // @ts-ignore
-    await context.updateAppId(this.appId)
-
-    // create production app
-    this.prodApp = await this.publish()
+    await context.doInAppContext(null, async () => {
+      this.app = await this._req(
+        { name: appName },
+        null,
+        controllers.app.create
+      )
+      this.appId = this.app.appId
+    })
+    return await context.doInAppContext(this.appId, async () => {
+      // create production app
+      this.prodApp = await this.publish()
 
-    this.allApps.push(this.prodApp)
-    this.allApps.push(this.app)
+      this.allApps.push(this.prodApp)
+      this.allApps.push(this.app)
 
-    return this.app
+      return this.app
+    })
   }
 
   async publish() {