@budibase/server 2.2.12-alpha.43 → 2.2.12-alpha.45

package/src/integrations/queries/sql.ts

@@ -1,55 +1,10 @@
-import { findHBSBlocks, processStringSync } from "@budibase/string-templates"
+import { findHBSBlocks } from "@budibase/string-templates"
 import { DatasourcePlus } from "@budibase/types"
+import sdk from "../../sdk"
 
 const CONST_CHAR_REGEX = new RegExp("'[^']*'", "g")
 
-export function enrichQueryFields(
-  fields: { [key: string]: any },
-  parameters = {}
-) {
-  const enrichedQuery: { [key: string]: any } = Array.isArray(fields) ? [] : {}
-  if (!fields || !parameters) {
-    return enrichedQuery
-  }
-  // enrich the fields with dynamic parameters
-  for (let key of Object.keys(fields)) {
-    if (fields[key] == null) {
-      continue
-    }
-    if (typeof fields[key] === "object") {
-      // enrich nested fields object
-      enrichedQuery[key] = enrichQueryFields(fields[key], parameters)
-    } else if (typeof fields[key] === "string") {
-      // enrich string value as normal
-      enrichedQuery[key] = processStringSync(fields[key], parameters, {
-        noEscaping: true,
-        noHelpers: true,
-        escapeNewlines: true,
-      })
-    } else {
-      enrichedQuery[key] = fields[key]
-    }
-  }
-  if (
-    enrichedQuery.json ||
-    enrichedQuery.customData ||
-    enrichedQuery.requestBody
-  ) {
-    try {
-      enrichedQuery.json = JSON.parse(
-        enrichedQuery.json ||
-          enrichedQuery.customData ||
-          enrichedQuery.requestBody
-      )
-    } catch (err) {
-      // no json found, ignore
-    }
-    delete enrichedQuery.customData
-  }
-  return enrichedQuery
-}
-
-export function interpolateSQL(
+export async function interpolateSQL(
   fields: { [key: string]: any },
   parameters: { [key: string]: any },
   integration: DatasourcePlus
@@ -90,7 +45,7 @@ export function interpolateSQL(
     else if (listRegexMatch) {
       arrays.push(binding)
       // determine the length of the array
-      const value = enrichQueryFields([binding], parameters)[0]
+      const value = (await sdk.queries.enrichContext([binding], parameters))[0]
         .split(",")
         .map((val: string) => val.trim())
       // build a string like ($1, $2, $3)
@@ -109,7 +64,7 @@ export function interpolateSQL(
   }
   // replicate the knex structure
   fields.sql = sql
-  fields.bindings = enrichQueryFields(variables, parameters)
+  fields.bindings = await sdk.queries.enrichContext(variables, parameters)
   // check for arrays in the data
   let updated: string[] = []
   for (let i = 0; i < variables.length; i++) {

package/src/integrations/rest.ts

@@ -16,11 +16,11 @@ import {
 import { get } from "lodash"
 import * as https from "https"
 import qs from "querystring"
-const fetch = require("node-fetch")
-const { formatBytes } = require("../utilities")
-const { performance } = require("perf_hooks")
-const FormData = require("form-data")
-const { URLSearchParams } = require("url")
+import fetch from "node-fetch"
+import { formatBytes } from "../utilities"
+import { performance } from "perf_hooks"
+import FormData from "form-data"
+import { URLSearchParams } from "url"
 
 const BodyTypes = {
   NONE: "none",
@@ -204,12 +204,12 @@ class RestIntegration implements IntegrationBase {
 
       // Append page number or cursor param if configured
       if (pageParam && paginationValues.page != null) {
-        params.append(pageParam, paginationValues.page)
+        params.append(pageParam, paginationValues.page as string)
       }
 
       // Append page size param if configured
       if (sizeParam && paginationValues.limit != null) {
-        params.append(sizeParam, paginationValues.limit)
+        params.append(sizeParam, String(paginationValues.limit))
       }
 
       // Prepend query string with pagination params
@@ -280,7 +280,7 @@ class RestIntegration implements IntegrationBase {
       case BodyTypes.ENCODED:
         const params = new URLSearchParams()
         for (let [key, value] of Object.entries(object)) {
-          params.append(key, value)
+          params.append(key, value as string)
         }
         addPaginationToBody((key: string, value: any) => {
           params.append(key, value)

package/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts

@@ -2,7 +2,7 @@ import TestConfig from "../../../../tests/utilities/TestConfiguration"
 import * as syncRows from "../syncRows"
 import { quotas } from "@budibase/pro"
 import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
-const { db: dbCore } = require("@budibase/backend-core")
+import { db as dbCore, context } from "@budibase/backend-core"
 
 describe("syncRows", () => {
   let config = new TestConfig(false)
@@ -24,13 +24,17 @@ describe("syncRows", () => {
 
       // app 1
       const app1 = config.app
-      await config.createTable()
-      await config.createRow()
+      await context.doInAppContext(app1.appId, async () => {
+        await config.createTable()
+        await config.createRow()
+      })
       // app 2
       const app2 = await config.createApp("second-app")
-      await config.createTable()
-      await config.createRow()
-      await config.createRow()
+      await context.doInAppContext(app2.appId, async () => {
+        await config.createTable()
+        await config.createRow()
+        await config.createRow()
+      })
 
       // migrate
       await syncRows.run()

package/src/sdk/app/datasources/datasources.ts

@@ -0,0 +1,98 @@
+import { context } from "@budibase/backend-core"
+import { processObjectSync, findHBSBlocks } from "@budibase/string-templates"
+import {
+  Datasource,
+  DatasourceFieldType,
+  Integration,
+  PASSWORD_REPLACEMENT,
+} from "@budibase/types"
+import { cloneDeep } from "lodash/fp"
+import { getEnvironmentVariables } from "../../utils"
+import { getDefinitions } from "../../../integrations"
+
+const ENV_VAR_PREFIX = "env."
+const USER_PREFIX = "user"
+
+async function enrichDatasourceWithValues(datasource: Datasource) {
+  const cloned = cloneDeep(datasource)
+  const env = await getEnvironmentVariables()
+  const processed = processObjectSync(cloned, { env }, { onlyFound: true })
+  return {
+    datasource: processed as Datasource,
+    envVars: env as Record<string, string>,
+  }
+}
+
+export async function enrich(datasource: Datasource) {
+  const { datasource: response } = await enrichDatasourceWithValues(datasource)
+  return response
+}
+
+export async function get(
+  datasourceId: string,
+  opts?: { enriched: boolean }
+): Promise<Datasource> {
+  const appDb = context.getAppDB()
+  const datasource = await appDb.get(datasourceId)
+  if (opts?.enriched) {
+    return (await enrichDatasourceWithValues(datasource)).datasource
+  } else {
+    return datasource
+  }
+}
+
+export async function getWithEnvVars(datasourceId: string) {
+  const appDb = context.getAppDB()
+  const datasource = await appDb.get(datasourceId)
+  return enrichDatasourceWithValues(datasource)
+}
+
+export async function removeSecrets(datasources: Datasource[]) {
+  const definitions = await getDefinitions()
+  for (let datasource of datasources) {
+    const schema = definitions[datasource.source]
+    if (datasource.config) {
+      // strip secrets from response, so they don't show in the network request
+      if (datasource.config.auth) {
+        delete datasource.config.auth
+      }
+      // remove passwords
+      for (let key of Object.keys(datasource.config)) {
+        if (typeof datasource.config[key] !== "string") {
+          continue
+        }
+        const blocks = findHBSBlocks(datasource.config[key] as string)
+        const usesEnvVars =
+          blocks.find(block => block.includes(ENV_VAR_PREFIX)) != null
+        if (
+          !usesEnvVars &&
+          schema.datasource?.[key]?.type === DatasourceFieldType.PASSWORD
+        ) {
+          datasource.config[key] = PASSWORD_REPLACEMENT
+        }
+      }
+    }
+  }
+  return datasources
+}
+
+export async function removeSecretSingle(datasource: Datasource) {
+  return (await removeSecrets([datasource]))[0]
+}
+
+export function mergeConfigs(update: Datasource, old: Datasource) {
+  if (!update.config) {
+    return update
+  }
+  for (let [key, value] of Object.entries(update.config)) {
+    if (value !== PASSWORD_REPLACEMENT) {
+      continue
+    }
+    if (old.config?.[key]) {
+      update.config[key] = old.config?.[key]
+    } else {
+      delete update.config[key]
+    }
+  }
+  return update
+}

package/src/sdk/app/datasources/index.ts

@@ -0,0 +1,5 @@
+import * as datasources from "./datasources"
+
+export default {
+  ...datasources,
+}

package/src/sdk/app/queries/index.ts

@@ -0,0 +1,5 @@
+import * as queries from "./queries"
+
+export default {
+  ...queries,
+}

package/src/sdk/app/queries/queries.ts

@@ -0,0 +1,50 @@
+import { getEnvironmentVariables } from "../../utils"
+import { processStringSync } from "@budibase/string-templates"
+
+export async function enrichContext(
+  fields: Record<string, any>,
+  inputs = {}
+): Promise<Record<string, any>> {
+  const enrichedQuery: Record<string, any> = Array.isArray(fields) ? [] : {}
+  if (!fields || !inputs) {
+    return enrichedQuery
+  }
+  const env = await getEnvironmentVariables()
+  const parameters = { ...inputs, env }
+  // enrich the fields with dynamic parameters
+  for (let key of Object.keys(fields)) {
+    if (fields[key] == null) {
+      continue
+    }
+    if (typeof fields[key] === "object") {
+      // enrich nested fields object
+      enrichedQuery[key] = await enrichContext(fields[key], parameters)
+    } else if (typeof fields[key] === "string") {
+      // enrich string value as normal
+      enrichedQuery[key] = processStringSync(fields[key], parameters, {
+        noEscaping: true,
+        noHelpers: true,
+        escapeNewlines: true,
+      })
+    } else {
+      enrichedQuery[key] = fields[key]
+    }
+  }
+  if (
+    enrichedQuery.json ||
+    enrichedQuery.customData ||
+    enrichedQuery.requestBody
+  ) {
+    try {
+      enrichedQuery.json = JSON.parse(
+        enrichedQuery.json ||
+          enrichedQuery.customData ||
+          enrichedQuery.requestBody
+      )
+    } catch (err) {
+      // no json found, ignore
+    }
+    delete enrichedQuery.customData
+  }
+  return enrichedQuery
+}

package/src/sdk/app/tables/index.ts

@@ -6,6 +6,7 @@ import {
   isSQL,
 } from "../../../integrations/utils"
 import { Table, Database } from "@budibase/types"
+import datasources from "../datasources"
 
 async function getAllInternalTables(db?: Database): Promise<Table[]> {
   if (!db) {
@@ -23,9 +24,11 @@ async function getAllInternalTables(db?: Database): Promise<Table[]> {
   }))
 }
 
-async function getAllExternalTables(datasourceId: any): Promise<Table[]> {
+async function getAllExternalTables(
+  datasourceId: any
+): Promise<Record<string, Table>> {
   const db = context.getAppDB()
-  const datasource = await db.get(datasourceId)
+  const datasource = await datasources.get(datasourceId, { enriched: true })
   if (!datasource || !datasource.entities) {
     throw "Datasource is not configured fully."
   }
@@ -44,7 +47,7 @@ async function getTable(tableId: any): Promise<Table> {
   const db = context.getAppDB()
   if (isExternalTable(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
-    const datasource = await db.get(datasourceId)
+    const datasource = await datasources.get(datasourceId!)
     const table = await getExternalTable(datasourceId, tableName)
     return { ...table, sql: isSQL(datasource) }
   } else {

package/src/sdk/index.ts

@@ -2,6 +2,8 @@ import { default as backups } from "./app/backups"
 import { default as tables } from "./app/tables"
 import { default as automations } from "./app/automations"
 import { default as applications } from "./app/applications"
+import { default as datasources } from "./app/datasources"
+import { default as queries } from "./app/queries"
 import { default as rows } from "./app/rows"
 import { default as users } from "./users"
 
@@ -12,6 +14,8 @@ const sdk = {
   applications,
   rows,
   users,
+  datasources,
+  queries,
 }
 
 // default export for TS

package/src/sdk/utils/index.ts

@@ -0,0 +1,16 @@
+import { environmentVariables } from "@budibase/pro"
+import { context, db as dbCore } from "@budibase/backend-core"
+import { AppEnvironment } from "@budibase/types"
+
+export async function getEnvironmentVariables() {
+  let envVars = context.getEnvironmentVariables()
+  if (!envVars) {
+    const appId = context.getAppId()
+    const appEnv = dbCore.isDevAppID(appId)
+      ? AppEnvironment.DEVELOPMENT
+      : AppEnvironment.PRODUCTION
+
+    envVars = await environmentVariables.fetchValues(appEnv)
+  }
+  return envVars
+}

package/src/tests/utilities/TestConfiguration.ts

@@ -364,20 +364,23 @@ class TestConfiguration {
     // create dev app
     // clear any old app
     this.appId = null
-    // @ts-ignore
-    await context.updateAppId(null)
-    this.app = await this._req({ name: appName }, null, controllers.app.create)
-    this.appId = this.app.appId
-    // @ts-ignore
-    await context.updateAppId(this.appId)
-
-    // create production app
-    this.prodApp = await this.publish()
+    await context.doInAppContext(null, async () => {
+      this.app = await this._req(
+        { name: appName },
+        null,
+        controllers.app.create
+      )
+      this.appId = this.app.appId
+    })
+    return await context.doInAppContext(this.appId, async () => {
+      // create production app
+      this.prodApp = await this.publish()
 
-    this.allApps.push(this.prodApp)
-    this.allApps.push(this.app)
+      this.allApps.push(this.prodApp)
+      this.allApps.push(this.app)
 
-    return this.app
+      return this.app
+    })
   }
 
   async publish() {

package/src/threads/automation.ts

@@ -16,7 +16,6 @@ import { storeLog } from "../automations/logging"
 import { Automation, AutomationStep, AutomationStatus } from "@budibase/types"
 import {
   LoopStep,
-  LoopStepType,
   LoopInput,
   TriggerOutput,
   AutomationContext,
@@ -26,6 +25,7 @@ import { WorkerCallback } from "./definitions"
 import { context, logging } from "@budibase/backend-core"
 import { processObject } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
+import * as sdkUtils from "../sdk/utils"
 import env from "../environment"
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
 const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
@@ -225,6 +225,8 @@ class Orchestrator {
   }
 
   async execute() {
+    // this will retrieve from context created at start of thread
+    this._context.env = await sdkUtils.getEnvironmentVariables()
     let automation = this._automation
     let stopped = false
     let loopStep: AutomationStep | undefined = undefined
@@ -478,7 +480,11 @@ export const removeStalled = async (job: Job) => {
     throw new Error("Unable to execute, event doesn't contain app ID.")
   }
   await context.doInAppContext(appId, async () => {
-    const automationOrchestrator = new Orchestrator(job)
-    await automationOrchestrator.stopCron("stalled")
+    const envVars = await sdkUtils.getEnvironmentVariables()
+    // put into automation thread for whole context
+    await context.doInEnvironmentContext(envVars, async () => {
+      const automationOrchestrator = new Orchestrator(job)
+      await automationOrchestrator.stopCron("stalled")
+    })
   })
 }

package/src/threads/definitions.ts

@@ -1,3 +1,5 @@
+import { EnvironmentVariablesDecrypted } from "@budibase/types"
+
 export type WorkerCallback = (error: any, response?: any) => void
 
 export interface QueryEvent {
@@ -9,6 +11,7 @@ export interface QueryEvent {
   pagination?: any
   transformer: any
   queryId: string
+  environmentVariables?: Record<string, string>
   ctx?: any
 }
 

package/src/threads/query.ts

@@ -6,13 +6,11 @@ import { getIntegration } from "../integrations"
 import { processStringSync } from "@budibase/string-templates"
 import { context, cache, auth } from "@budibase/backend-core"
 import { getGlobalIDFromUserMetadataID } from "../db/utils"
+import sdk from "../sdk"
 import { cloneDeep } from "lodash/fp"
 
-const { isSQL } = require("../integrations/utils")
-const {
-  enrichQueryFields,
-  interpolateSQL,
-} = require("../integrations/queries/sql")
+import { isSQL } from "../integrations/utils"
+import { interpolateSQL } from "../integrations/queries/sql"
 
 class QueryRunner {
   datasource: any
@@ -62,10 +60,11 @@ class QueryRunner {
     }
 
     if (datasourceClone.config.authConfigs) {
-      datasourceClone.config.authConfigs =
-        datasourceClone.config.authConfigs.map((config: any) => {
-          return enrichQueryFields(config, this.ctx)
-        })
+      const updatedConfigs = []
+      for (let config of datasourceClone.config.authConfigs) {
+        updatedConfigs.push(await sdk.queries.enrichContext(config, this.ctx))
+      }
+      datasourceClone.config.authConfigs = updatedConfigs
     }
 
     const integration = new Integration(datasourceClone.config)
@@ -75,12 +74,15 @@ class QueryRunner {
 
     // Enrich the parameters with the addition context items.
     // 'user' is now a reserved variable key in mapping parameters
-    const enrichedParameters = enrichQueryFields(parameters, this.ctx)
+    const enrichedParameters = await sdk.queries.enrichContext(
+      parameters,
+      this.ctx
+    )
     const enrichedContext = { ...enrichedParameters, ...this.ctx }
 
     // Parse global headers
     if (datasourceClone.config.defaultHeaders) {
-      datasourceClone.config.defaultHeaders = enrichQueryFields(
+      datasourceClone.config.defaultHeaders = await sdk.queries.enrichContext(
         datasourceClone.config.defaultHeaders,
         enrichedContext
       )
@@ -89,9 +91,9 @@ class QueryRunner {
     let query
     // handle SQL injections by interpolating the variables
     if (isSQL(datasourceClone)) {
-      query = interpolateSQL(fieldsClone, enrichedParameters, integration)
+      query = await interpolateSQL(fieldsClone, enrichedParameters, integration)
     } else {
-      query = enrichQueryFields(fieldsClone, enrichedContext)
+      query = await sdk.queries.enrichContext(fieldsClone, enrichedContext)
     }
 
     // Add pagination values for REST queries
@@ -166,7 +168,9 @@ class QueryRunner {
   async runAnotherQuery(queryId: string, parameters: any) {
     const db = context.getAppDB()
     const query = await db.get(queryId)
-    const datasource = await db.get(query.datasourceId)
+    const datasource = await sdk.datasources.get(query.datasourceId, {
+      enriched: true,
+    })
     return new QueryRunner(
       {
         datasource,
@@ -280,7 +284,7 @@ class QueryRunner {
 }
 
 export function execute(input: QueryEvent, callback: WorkerCallback) {
-  context.doInAppContext(input.appId!, async () => {
+  const run = async () => {
     const Runner = new QueryRunner(input)
     try {
       const response = await Runner.execute()
@@ -288,5 +292,14 @@ export function execute(input: QueryEvent, callback: WorkerCallback) {
     } catch (err) {
       callback(err)
     }
+  }
+  context.doInAppContext(input.appId!, async () => {
+    if (input.environmentVariables) {
+      return context.doInEnvironmentContext(input.environmentVariables, () => {
+        return run()
+      })
+    } else {
+      return run()
+    }
   })
 }

package/src/utilities/fileSystem/filesystem.ts

@@ -1,10 +1,10 @@
 import { PathLike } from "fs"
-const { budibaseTempDir } = require("../budibaseDir")
-const fs = require("fs")
-const { join } = require("path")
-const uuid = require("uuid/v4")
+import fs from "fs"
+import { budibaseTempDir } from "../budibaseDir"
+import { join } from "path"
 import env from "../../environment"
 import tar from "tar"
+const uuid = require("uuid/v4")
 
 export const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
 
@@ -112,6 +112,7 @@ export const sendTempFile = (fileContents: any) => {
  * allows a centralised location to check logic is all good.
  */
 export const readFileSync = (filepath: PathLike, options = "utf8") => {
+  // @ts-ignore
   return fs.readFileSync(filepath, options)
 }
 
@@ -147,6 +148,7 @@ export const findFileRec = (startPath: PathLike, filter: string): any => {
 
   const files = fs.readdirSync(startPath)
   for (let i = 0, len = files.length; i < len; i++) {
+    // @ts-ignore
     const filename = join(startPath, files[i])
     const stat = fs.lstatSync(filename)
 

package/src/utilities/fileSystem/plugin.ts

@@ -1,8 +1,7 @@
 import { Plugin } from "@budibase/types"
-
-const { budibaseTempDir } = require("../budibaseDir")
-const fs = require("fs")
-const { join } = require("path")
+import { budibaseTempDir } from "../budibaseDir"
+import fs from "fs"
+import { join } from "path"
 import { objectStore } from "@budibase/backend-core"
 
 const DATASOURCE_PATH = join(budibaseTempDir(), "datasource")

package/src/utilities/fileSystem/template.ts

@@ -1,5 +1,5 @@
-const fs = require("fs")
-const { join } = require("path")
+import fs from "fs"
+import { join } from "path"
 import { ObjectStoreBuckets } from "../../constants"
 import { objectStore } from "@budibase/backend-core"