@budibase/server 2.2.25 → 2.2.27-alpha.0

package/src/integrations/rest.ts

@@ -6,21 +6,19 @@ import {
   IntegrationBase,
   PaginationValues,
   RestQueryFields as RestQuery,
-} from "@budibase/types"
-import {
   RestConfig,
-  AuthType,
-  BasicAuthConfig,
-  BearerAuthConfig,
-} from "../definitions/datasource"
+  RestAuthType,
+  RestBasicAuthConfig,
+  RestBearerAuthConfig,
+} from "@budibase/types"
 import { get } from "lodash"
 import * as https from "https"
 import qs from "querystring"
-const fetch = require("node-fetch")
-const { formatBytes } = require("../utilities")
-const { performance } = require("perf_hooks")
-const FormData = require("form-data")
-const { URLSearchParams } = require("url")
+import fetch from "node-fetch"
+import { formatBytes } from "../utilities"
+import { performance } from "perf_hooks"
+import FormData from "form-data"
+import { URLSearchParams } from "url"
 
 const BodyTypes = {
   NONE: "none",
@@ -204,12 +202,12 @@ class RestIntegration implements IntegrationBase {
 
       // Append page number or cursor param if configured
       if (pageParam && paginationValues.page != null) {
-        params.append(pageParam, paginationValues.page)
+        params.append(pageParam, paginationValues.page as string)
       }
 
       // Append page size param if configured
       if (sizeParam && paginationValues.limit != null) {
-        params.append(sizeParam, paginationValues.limit)
+        params.append(sizeParam, String(paginationValues.limit))
       }
 
       // Prepend query string with pagination params
@@ -280,7 +278,7 @@ class RestIntegration implements IntegrationBase {
       case BodyTypes.ENCODED:
         const params = new URLSearchParams()
         for (let [key, value] of Object.entries(object)) {
-          params.append(key, value)
+          params.append(key, value as string)
         }
         addPaginationToBody((key: string, value: any) => {
           params.append(key, value)
@@ -331,14 +329,14 @@ class RestIntegration implements IntegrationBase {
       if (authConfig) {
         let config
         switch (authConfig.type) {
-          case AuthType.BASIC:
-            config = authConfig.config as BasicAuthConfig
+          case RestAuthType.BASIC:
+            config = authConfig.config as RestBasicAuthConfig
             headers.Authorization = `Basic ${Buffer.from(
               `${config.username}:${config.password}`
             ).toString("base64")}`
             break
-          case AuthType.BEARER:
-            config = authConfig.config as BearerAuthConfig
+          case RestAuthType.BEARER:
+            config = authConfig.config as RestBearerAuthConfig
             headers.Authorization = `Bearer ${config.token}`
             break
         }
@@ -428,5 +426,4 @@ class RestIntegration implements IntegrationBase {
 export default {
   schema: SCHEMA,
   integration: RestIntegration,
-  AuthType,
 }

package/src/integrations/tests/couchdb.spec.ts

@@ -1,23 +1,32 @@
-jest.mock(
-  "pouchdb",
-  () =>
-    function CouchDBMock(this: any) {
-      this.post = jest.fn()
-      this.allDocs = jest.fn(() => ({ rows: [] }))
-      this.put = jest.fn()
-      this.get = jest.fn()
-      this.remove = jest.fn()
-      this.plugin = jest.fn()
-      this.close = jest.fn()
-    }
-)
+import { DatabaseWithConnection } from "@budibase/backend-core/src/db"
+
+jest.mock("@budibase/backend-core", () => {
+  const core = jest.requireActual("@budibase/backend-core")
+  return {
+    ...core,
+    db: {
+      ...core.db,
+      DatabaseWithConnection: function () {
+        return {
+          post: jest.fn(),
+          allDocs: jest.fn().mockReturnValue({ rows: [] }),
+          put: jest.fn(),
+          get: jest.fn().mockReturnValue({ _rev: "a" }),
+          remove: jest.fn(),
+        }
+      },
+    },
+  }
+})
 
 import { default as CouchDBIntegration } from "../couchdb"
 
 class TestConfiguration {
   integration: any
 
-  constructor(config: any = {}) {
+  constructor(
+    config: any = { url: "http://somewhere", database: "something" }
+  ) {
     this.integration = new CouchDBIntegration.integration(config)
   }
 }
@@ -33,8 +42,8 @@ describe("CouchDB Integration", () => {
     const doc = {
       test: 1,
     }
-    const response = await config.integration.create({
-      json: doc,
+    await config.integration.create({
+      json: JSON.stringify(doc),
     })
     expect(config.integration.client.post).toHaveBeenCalledWith(doc)
   })
@@ -44,8 +53,8 @@ describe("CouchDB Integration", () => {
       name: "search",
     }
 
-    const response = await config.integration.read({
-      json: doc,
+    await config.integration.read({
+      json: JSON.stringify(doc),
     })
 
     expect(config.integration.client.allDocs).toHaveBeenCalledWith({
@@ -60,11 +69,14 @@ describe("CouchDB Integration", () => {
       name: "search",
     }
 
-    const response = await config.integration.update({
-      json: doc,
+    await config.integration.update({
+      json: JSON.stringify(doc),
     })
 
-    expect(config.integration.client.put).toHaveBeenCalledWith(doc)
+    expect(config.integration.client.put).toHaveBeenCalledWith({
+      ...doc,
+      _rev: "a",
+    })
   })
 
   it("calls the delete method with the correct params", async () => {

package/src/integrations/tests/rest.spec.ts

@@ -1,18 +1,21 @@
-const mockFetch = jest.fn(() => ({
-  headers: {
-    raw: () => {
-      return { "content-type": ["application/json"] }
+jest.mock("node-fetch", () => {
+  return jest.fn(() => ({
+    headers: {
+      raw: () => {
+        return { "content-type": ["application/json"] }
+      },
+      get: () => ["application/json"],
     },
-    get: () => ["application/json"],
-  },
-  json: jest.fn(() => ({
-    my_next_cursor: 123,
-  })),
-  text: jest.fn(),
-}))
-jest.mock("node-fetch", () => mockFetch)
+    json: jest.fn(() => ({
+      my_next_cursor: 123,
+    })),
+    text: jest.fn(),
+  }))
+})
+
 import fetch from "node-fetch"
 import { default as RestIntegration } from "../rest"
+import { RestAuthType } from "@budibase/types"
 const FormData = require("form-data")
 const { URLSearchParams } = require("url")
 
@@ -227,7 +230,7 @@ describe("REST Integration", () => {
     const basicAuth = {
       _id: "c59c14bd1898a43baa08da68959b24686",
       name: "basic-1",
-      type: RestIntegration.AuthType.BASIC,
+      type: RestAuthType.BASIC,
       config: {
         username: "user",
         password: "password",
@@ -237,7 +240,7 @@ describe("REST Integration", () => {
     const bearerAuth = {
       _id: "0d91d732f34e4befabeff50b392a8ff3",
       name: "bearer-1",
-      type: RestIntegration.AuthType.BEARER,
+      type: RestAuthType.BEARER,
       config: {
         token: "mytoken",
       },
@@ -579,7 +582,8 @@ describe("REST Integration", () => {
     })
     await config.integration.read({})
 
-    const calls: any = mockFetch.mock.calls[0]
+    // @ts-ignore
+    const calls: any = fetch.mock.calls[0]
     const url = calls[0]
     expect(url).toBe(`${BASE_URL}/`)
 

package/src/middleware/authorized.ts

@@ -63,7 +63,11 @@ const checkAuthorizedResource = async (
   }
 }
 
-export = (permType: any, permLevel: any = null, opts = { schema: false }) =>
+export default (
+    permType: any,
+    permLevel: any = null,
+    opts = { schema: false }
+  ) =>
   async (ctx: any, next: any) => {
     // webhooks don't need authentication, each webhook unique
     // also internal requests (between services) don't need authorized

package/src/middleware/builder.ts

@@ -64,7 +64,7 @@ async function updateAppUpdatedAt(ctx: BBContext) {
   })
 }
 
-export = async function builder(ctx: BBContext, permType: string) {
+export default async function builder(ctx: BBContext, permType: string) {
   const appId = ctx.appId
   // this only functions within an app context
   if (!appId) {

package/src/middleware/currentapp.ts

@@ -12,7 +12,7 @@ import env from "../environment"
 import { isWebhookEndpoint } from "./utils"
 import { BBContext } from "@budibase/types"
 
-export = async (ctx: BBContext, next: any) => {
+export default async (ctx: BBContext, next: any) => {
   // try to get the appID from the request
   let requestAppId = await utils.getAppIdFromCtx(ctx)
   // get app cookie if it exists
@@ -25,6 +25,7 @@ export = async (ctx: BBContext, next: any) => {
   if (!appCookie && !requestAppId) {
     return next()
   }
+
   // check the app exists referenced in cookie
   if (appCookie) {
     const appId = appCookie.appId
@@ -51,7 +52,7 @@ export = async (ctx: BBContext, next: any) => {
 
   let appId: string | undefined,
     roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
-  if (!ctx.user) {
+  if (!ctx.user?._id) {
     // not logged in, try to set a cookie for public apps
     appId = requestAppId
   } else if (requestAppId != null) {
@@ -96,7 +97,7 @@ export = async (ctx: BBContext, next: any) => {
     // need to judge this only based on the request app ID,
     if (
       env.MULTI_TENANCY &&
-      ctx.user &&
+      ctx.user?._id &&
       requestAppId &&
       !tenancy.isUserInAppTenant(requestAppId, ctx.user)
     ) {

package/src/middleware/publicApi.ts

@@ -1,7 +1,7 @@
 import { constants, utils } from "@budibase/backend-core"
 import { BBContext } from "@budibase/types"
 
-export = function ({ requiresAppId }: { requiresAppId?: boolean } = {}) {
+export default function ({ requiresAppId }: { requiresAppId?: boolean } = {}) {
   return async (ctx: BBContext, next: any) => {
     const appId = await utils.getAppIdFromCtx(ctx)
     if (requiresAppId && !appId) {

package/src/middleware/selfhost.ts

@@ -3,7 +3,7 @@ import { BBContext } from "@budibase/types"
 
 // if added as a middleware will stop requests unless builder is in self host mode
 // or cloud is in self host
-export = async (ctx: BBContext, next: any) => {
+export default async (ctx: BBContext, next: any) => {
   if (env.SELF_HOSTED) {
     await next()
     return

package/src/middleware/tests/authorized.spec.js

@@ -7,7 +7,7 @@ jest.mock("../../environment", () => ({
     }
   })
 )
-const authorizedMiddleware = require("../authorized")
+const authorizedMiddleware = require("../authorized").default
 const env = require("../../environment")
 const { permissions } = require("@budibase/backend-core")
 

package/src/middleware/tests/currentapp.spec.js

@@ -124,7 +124,7 @@ class TestConfiguration {
     // import as late as possible for mocks
     jest.resetModules()
     require("../../db").init()
-    const currentAppMiddleware = require("../currentapp")
+    const currentAppMiddleware = require("../currentapp").default
     return currentAppMiddleware(this.ctx, this.next)
   }
 }

package/src/middleware/tests/selfhost.spec.js

@@ -1,4 +1,4 @@
-const selfHostMiddleware = require("../selfhost")
+const selfHostMiddleware = require("../selfhost").default
 const env = require("../../environment")
 jest.mock("../../environment")
 

package/src/migrations/functions/backfill/global/configs.ts

@@ -5,6 +5,7 @@ import {
   isGoogleConfig,
   isOIDCConfig,
   isSettingsConfig,
+  ConfigType,
 } from "@budibase/types"
 import env from "./../../../../environment"
 
@@ -31,15 +32,15 @@ export const backfill = async (
       await events.email.SMTPCreated(timestamp)
     }
     if (isGoogleConfig(config)) {
-      await events.auth.SSOCreated("google", timestamp)
+      await events.auth.SSOCreated(ConfigType.GOOGLE, timestamp)
       if (config.config.activated) {
-        await events.auth.SSOActivated("google", timestamp)
+        await events.auth.SSOActivated(ConfigType.GOOGLE, timestamp)
       }
     }
     if (isOIDCConfig(config)) {
-      await events.auth.SSOCreated("oidc", timestamp)
+      await events.auth.SSOCreated(ConfigType.OIDC, timestamp)
       if (config.config.configs[0].activated) {
-        await events.auth.SSOActivated("oidc", timestamp)
+        await events.auth.SSOActivated(ConfigType.OIDC, timestamp)
       }
     }
     if (isSettingsConfig(config)) {

package/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts

@@ -2,7 +2,7 @@ import TestConfig from "../../../../tests/utilities/TestConfiguration"
 import * as syncRows from "../syncRows"
 import { quotas } from "@budibase/pro"
 import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
-const { db: dbCore } = require("@budibase/backend-core")
+import { db as dbCore, context } from "@budibase/backend-core"
 
 describe("syncRows", () => {
   let config = new TestConfig(false)
@@ -24,13 +24,17 @@ describe("syncRows", () => {
 
       // app 1
       const app1 = config.app
-      await config.createTable()
-      await config.createRow()
+      await context.doInAppContext(app1.appId, async () => {
+        await config.createTable()
+        await config.createRow()
+      })
       // app 2
       const app2 = await config.createApp("second-app")
-      await config.createTable()
-      await config.createRow()
-      await config.createRow()
+      await context.doInAppContext(app2.appId, async () => {
+        await config.createTable()
+        await config.createRow()
+        await config.createRow()
+      })
 
       // migrate
       await syncRows.run()

package/src/sdk/app/backups/imports.ts

@@ -26,11 +26,7 @@ type TemplateType = {
 
 function rewriteAttachmentUrl(appId: string, attachment: RowAttachment) {
   // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv
-  const urlParts = attachment.url.split("/")
-  // drop the first empty element
-  urlParts.shift()
-  // get the prefix
-  const prefix = urlParts.shift()
+  const urlParts = attachment.key.split("/")
   // remove the app ID
   urlParts.shift()
   // add new app ID
@@ -39,7 +35,7 @@ function rewriteAttachmentUrl(appId: string, attachment: RowAttachment) {
   return {
     ...attachment,
     key,
-    url: `/${prefix}/${key}`,
+    url: "", // calculated on retrieval using key
   }
 }
 

package/src/sdk/app/datasources/datasources.ts

@@ -0,0 +1,136 @@
+import { context } from "@budibase/backend-core"
+import { findHBSBlocks, processObjectSync } from "@budibase/string-templates"
+import {
+  Datasource,
+  DatasourceFieldType,
+  PASSWORD_REPLACEMENT,
+  RestAuthConfig,
+  RestAuthType,
+  RestBasicAuthConfig,
+  SourceName,
+} from "@budibase/types"
+import { cloneDeep } from "lodash/fp"
+import { getEnvironmentVariables } from "../../utils"
+import { getDefinitions } from "../../../integrations"
+
+const ENV_VAR_PREFIX = "env."
+
+async function enrichDatasourceWithValues(datasource: Datasource) {
+  const cloned = cloneDeep(datasource)
+  const env = await getEnvironmentVariables()
+  const processed = processObjectSync(cloned, { env }, { onlyFound: true })
+  return {
+    datasource: processed as Datasource,
+    envVars: env as Record<string, string>,
+  }
+}
+
+export async function enrich(datasource: Datasource) {
+  const { datasource: response } = await enrichDatasourceWithValues(datasource)
+  return response
+}
+
+export async function get(
+  datasourceId: string,
+  opts?: { enriched: boolean }
+): Promise<Datasource> {
+  const appDb = context.getAppDB()
+  const datasource = await appDb.get(datasourceId)
+  if (opts?.enriched) {
+    return (await enrichDatasourceWithValues(datasource)).datasource
+  } else {
+    return datasource
+  }
+}
+
+export async function getWithEnvVars(datasourceId: string) {
+  const appDb = context.getAppDB()
+  const datasource = await appDb.get(datasourceId)
+  return enrichDatasourceWithValues(datasource)
+}
+
+function hasAuthConfigs(datasource: Datasource) {
+  return datasource.source === SourceName.REST && datasource.config?.authConfigs
+}
+
+function useEnvVars(str: any) {
+  if (typeof str !== "string") {
+    return false
+  }
+  const blocks = findHBSBlocks(str)
+  return blocks.find(block => block.includes(ENV_VAR_PREFIX)) != null
+}
+
+export async function removeSecrets(datasources: Datasource[]) {
+  const definitions = await getDefinitions()
+  for (let datasource of datasources) {
+    const schema = definitions[datasource.source]
+    if (datasource.config) {
+      // strip secrets from response, so they don't show in the network request
+      if (datasource.config.auth) {
+        delete datasource.config.auth
+      }
+      // specific to REST datasources, contains passwords
+      if (hasAuthConfigs(datasource)) {
+        const configs = datasource.config.authConfigs as RestAuthConfig[]
+        for (let config of configs) {
+          if (config.type !== RestAuthType.BASIC) {
+            continue
+          }
+          const basic = config.config as RestBasicAuthConfig
+          if (!useEnvVars(basic.password)) {
+            basic.password = PASSWORD_REPLACEMENT
+          }
+        }
+      }
+      // remove general passwords
+      for (let key of Object.keys(datasource.config)) {
+        if (
+          schema.datasource?.[key]?.type === DatasourceFieldType.PASSWORD &&
+          !useEnvVars(datasource.config[key])
+        ) {
+          datasource.config[key] = PASSWORD_REPLACEMENT
+        }
+      }
+    }
+  }
+  return datasources
+}
+
+export async function removeSecretSingle(datasource: Datasource) {
+  return (await removeSecrets([datasource]))[0]
+}
+
+export function mergeConfigs(update: Datasource, old: Datasource) {
+  if (!update.config) {
+    return update
+  }
+  // specific to REST datasources, fix the auth configs again if required
+  if (hasAuthConfigs(update)) {
+    const configs = update.config.authConfigs as RestAuthConfig[]
+    const oldConfigs = old.config?.authConfigs as RestAuthConfig[]
+    for (let config of configs) {
+      if (config.type !== RestAuthType.BASIC) {
+        continue
+      }
+      const basic = config.config as RestBasicAuthConfig
+      const oldBasic = oldConfigs.find(old => old.name === config.name)
+        ?.config as RestBasicAuthConfig
+      if (basic.password === PASSWORD_REPLACEMENT) {
+        basic.password = oldBasic.password
+      }
+    }
+  }
+  // update back to actual passwords for everything else
+  for (let [key, value] of Object.entries(update.config)) {
+    if (value !== PASSWORD_REPLACEMENT) {
+      continue
+    }
+    if (old.config?.[key]) {
+      update.config[key] = old.config?.[key]
+    } else {
+      delete update.config[key]
+    }
+  }
+  return update
+}

package/src/sdk/app/datasources/index.ts

@@ -0,0 +1,5 @@
+import * as datasources from "./datasources"
+
+export default {
+  ...datasources,
+}

package/src/sdk/app/queries/index.ts

@@ -0,0 +1,5 @@
+import * as queries from "./queries"
+
+export default {
+  ...queries,
+}

package/src/sdk/app/queries/queries.ts

@@ -0,0 +1,50 @@
+import { getEnvironmentVariables } from "../../utils"
+import { processStringSync } from "@budibase/string-templates"
+
+export async function enrichContext(
+  fields: Record<string, any>,
+  inputs = {}
+): Promise<Record<string, any>> {
+  const enrichedQuery: Record<string, any> = Array.isArray(fields) ? [] : {}
+  if (!fields || !inputs) {
+    return enrichedQuery
+  }
+  const env = await getEnvironmentVariables()
+  const parameters = { ...inputs, env }
+  // enrich the fields with dynamic parameters
+  for (let key of Object.keys(fields)) {
+    if (fields[key] == null) {
+      continue
+    }
+    if (typeof fields[key] === "object") {
+      // enrich nested fields object
+      enrichedQuery[key] = await enrichContext(fields[key], parameters)
+    } else if (typeof fields[key] === "string") {
+      // enrich string value as normal
+      enrichedQuery[key] = processStringSync(fields[key], parameters, {
+        noEscaping: true,
+        noHelpers: true,
+        escapeNewlines: true,
+      })
+    } else {
+      enrichedQuery[key] = fields[key]
+    }
+  }
+  if (
+    enrichedQuery.json ||
+    enrichedQuery.customData ||
+    enrichedQuery.requestBody
+  ) {
+    try {
+      enrichedQuery.json = JSON.parse(
+        enrichedQuery.json ||
+          enrichedQuery.customData ||
+          enrichedQuery.requestBody
+      )
+    } catch (err) {
+      // no json found, ignore
+    }
+    delete enrichedQuery.customData
+  }
+  return enrichedQuery
+}

package/src/sdk/app/tables/index.ts

@@ -6,6 +6,7 @@ import {
   isSQL,
 } from "../../../integrations/utils"
 import { Table, Database } from "@budibase/types"
+import datasources from "../datasources"
 
 async function getAllInternalTables(db?: Database): Promise<Table[]> {
   if (!db) {
@@ -23,9 +24,11 @@ async function getAllInternalTables(db?: Database): Promise<Table[]> {
   }))
 }
 
-async function getAllExternalTables(datasourceId: any): Promise<Table[]> {
+async function getAllExternalTables(
+  datasourceId: any
+): Promise<Record<string, Table>> {
   const db = context.getAppDB()
-  const datasource = await db.get(datasourceId)
+  const datasource = await datasources.get(datasourceId, { enriched: true })
   if (!datasource || !datasource.entities) {
     throw "Datasource is not configured fully."
   }
@@ -44,7 +47,7 @@ async function getTable(tableId: any): Promise<Table> {
   const db = context.getAppDB()
   if (isExternalTable(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
-    const datasource = await db.get(datasourceId)
+    const datasource = await datasources.get(datasourceId!)
     const table = await getExternalTable(datasourceId, tableName)
     return { ...table, sql: isSQL(datasource) }
   } else {

package/src/sdk/index.ts

@@ -2,6 +2,8 @@ import { default as backups } from "./app/backups"
 import { default as tables } from "./app/tables"
 import { default as automations } from "./app/automations"
 import { default as applications } from "./app/applications"
+import { default as datasources } from "./app/datasources"
+import { default as queries } from "./app/queries"
 import { default as rows } from "./app/rows"
 import { default as users } from "./users"
 
@@ -12,6 +14,8 @@ const sdk = {
   applications,
   rows,
   users,
+  datasources,
+  queries,
 }
 
 // default export for TS