@budibase/server 2.2.26 → 2.2.27-alpha.1

package/src/threads/automation.ts

@@ -16,7 +16,6 @@ import { storeLog } from "../automations/logging"
 import { Automation, AutomationStep, AutomationStatus } from "@budibase/types"
 import {
   LoopStep,
-  LoopStepType,
   LoopInput,
   TriggerOutput,
   AutomationContext,
@@ -26,43 +25,25 @@ import { WorkerCallback } from "./definitions"
 import { context, logging } from "@budibase/backend-core"
 import { processObject } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
+import * as sdkUtils from "../sdk/utils"
 import env from "../environment"
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
 const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
 const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED }
 
-function typecastForLooping(loopStep: LoopStep, input: LoopInput) {
-  if (!input || !input.binding) {
-    return null
-  }
-  try {
-    switch (loopStep.inputs.option) {
-      case LoopStepType.ARRAY:
-        if (typeof input.binding === "string") {
-          return JSON.parse(input.binding)
-        }
-        break
-      case LoopStepType.STRING:
-        if (Array.isArray(input.binding)) {
-          return input.binding.join(",")
-        }
-        break
-    }
-  } catch (err) {
-    throw new Error("Unable to cast to correct type")
-  }
-  return input.binding
-}
-
 function getLoopIterations(loopStep: LoopStep, input: LoopInput) {
-  const binding = typecastForLooping(loopStep, input)
+  const binding = automationUtils.typecastForLooping(loopStep, input)
   if (!loopStep || !binding) {
     return 1
   }
-  return Array.isArray(binding)
-    ? binding.length
-    : automationUtils.stringSplit(binding).length
+  if (Array.isArray(binding)) {
+    return binding.length
+  }
+  if (typeof binding === "string") {
+    return automationUtils.stringSplit(binding).length
+  }
+  return 1
 }
 
 /**
@@ -244,6 +225,8 @@ class Orchestrator {
   }
 
   async execute() {
+    // this will retrieve from context created at start of thread
+    this._context.env = await sdkUtils.getEnvironmentVariables()
     let automation = this._automation
     let stopped = false
     let loopStep: AutomationStep | undefined = undefined
@@ -289,7 +272,7 @@ class Orchestrator {
 
           let tempOutput = { items: loopSteps, iterations: iterationCount }
           try {
-            newInput.binding = typecastForLooping(
+            newInput.binding = automationUtils.typecastForLooping(
               loopStep as LoopStep,
               newInput
             )
@@ -303,13 +286,13 @@ class Orchestrator {
             break
           }
 
-          let item
+          let item = []
           if (
             typeof loopStep.inputs.binding === "string" &&
             loopStep.inputs.option === "String"
           ) {
             item = automationUtils.stringSplit(newInput.binding)
-          } else {
+          } else if (Array.isArray(loopStep.inputs.binding)) {
             item = loopStep.inputs.binding
           }
           this._context.steps[loopStepNumber] = {
@@ -442,7 +425,7 @@ class Orchestrator {
 
       // Delete the step after the loop step as it's irrelevant, since information is included
       // in the loop step
-      if (wasLoopStep) {
+      if (wasLoopStep && !loopStep) {
         this._context.steps.splice(loopStepNumber + 1, 1)
         wasLoopStep = false
       }
@@ -461,8 +444,8 @@ class Orchestrator {
         })
         this._context.steps[loopStepNumber] = tempOutput
 
-        loopSteps = undefined
         wasLoopStep = true
+        loopSteps = []
       }
     }
 
@@ -497,7 +480,11 @@ export const removeStalled = async (job: Job) => {
     throw new Error("Unable to execute, event doesn't contain app ID.")
   }
   await context.doInAppContext(appId, async () => {
-    const automationOrchestrator = new Orchestrator(job)
-    await automationOrchestrator.stopCron("stalled")
+    const envVars = await sdkUtils.getEnvironmentVariables()
+    // put into automation thread for whole context
+    await context.doInEnvironmentContext(envVars, async () => {
+      const automationOrchestrator = new Orchestrator(job)
+      await automationOrchestrator.stopCron("stalled")
+    })
   })
 }

package/src/threads/definitions.ts

@@ -1,3 +1,5 @@
+import { EnvironmentVariablesDecrypted } from "@budibase/types"
+
 export type WorkerCallback = (error: any, response?: any) => void
 
 export interface QueryEvent {
@@ -9,6 +11,7 @@ export interface QueryEvent {
   pagination?: any
   transformer: any
   queryId: string
+  environmentVariables?: Record<string, string>
   ctx?: any
 }
 

package/src/threads/index.ts

@@ -1,5 +1,5 @@
 import workerFarm from "worker-farm"
-import * as env from "../environment"
+import env from "../environment"
 
 export const ThreadType = {
   QUERY: "query",

package/src/threads/query.ts

@@ -6,13 +6,11 @@ import { getIntegration } from "../integrations"
 import { processStringSync } from "@budibase/string-templates"
 import { context, cache, auth } from "@budibase/backend-core"
 import { getGlobalIDFromUserMetadataID } from "../db/utils"
+import sdk from "../sdk"
 import { cloneDeep } from "lodash/fp"
 
-const { isSQL } = require("../integrations/utils")
-const {
-  enrichQueryFields,
-  interpolateSQL,
-} = require("../integrations/queries/sql")
+import { isSQL } from "../integrations/utils"
+import { interpolateSQL } from "../integrations/queries/sql"
 
 class QueryRunner {
   datasource: any
@@ -62,10 +60,11 @@ class QueryRunner {
     }
 
     if (datasourceClone.config.authConfigs) {
-      datasourceClone.config.authConfigs =
-        datasourceClone.config.authConfigs.map((config: any) => {
-          return enrichQueryFields(config, this.ctx)
-        })
+      const updatedConfigs = []
+      for (let config of datasourceClone.config.authConfigs) {
+        updatedConfigs.push(await sdk.queries.enrichContext(config, this.ctx))
+      }
+      datasourceClone.config.authConfigs = updatedConfigs
     }
 
     const integration = new Integration(datasourceClone.config)
@@ -75,12 +74,15 @@ class QueryRunner {
 
     // Enrich the parameters with the addition context items.
     // 'user' is now a reserved variable key in mapping parameters
-    const enrichedParameters = enrichQueryFields(parameters, this.ctx)
+    const enrichedParameters = await sdk.queries.enrichContext(
+      parameters,
+      this.ctx
+    )
     const enrichedContext = { ...enrichedParameters, ...this.ctx }
 
     // Parse global headers
     if (datasourceClone.config.defaultHeaders) {
-      datasourceClone.config.defaultHeaders = enrichQueryFields(
+      datasourceClone.config.defaultHeaders = await sdk.queries.enrichContext(
         datasourceClone.config.defaultHeaders,
         enrichedContext
       )
@@ -89,9 +91,9 @@ class QueryRunner {
     let query
     // handle SQL injections by interpolating the variables
     if (isSQL(datasourceClone)) {
-      query = interpolateSQL(fieldsClone, enrichedContext, integration)
+      query = await interpolateSQL(fieldsClone, enrichedContext, integration)
     } else {
-      query = enrichQueryFields(fieldsClone, enrichedContext)
+      query = await sdk.queries.enrichContext(fieldsClone, enrichedContext)
     }
 
     // Add pagination values for REST queries
@@ -166,7 +168,9 @@ class QueryRunner {
   async runAnotherQuery(queryId: string, parameters: any) {
     const db = context.getAppDB()
     const query = await db.get(queryId)
-    const datasource = await db.get(query.datasourceId)
+    const datasource = await sdk.datasources.get(query.datasourceId, {
+      enriched: true,
+    })
     return new QueryRunner(
       {
         datasource,
@@ -280,7 +284,7 @@ class QueryRunner {
 }
 
 export function execute(input: QueryEvent, callback: WorkerCallback) {
-  context.doInAppContext(input.appId!, async () => {
+  const run = async () => {
     const Runner = new QueryRunner(input)
     try {
       const response = await Runner.execute()
@@ -288,5 +292,14 @@ export function execute(input: QueryEvent, callback: WorkerCallback) {
     } catch (err) {
       callback(err)
     }
+  }
+  context.doInAppContext(input.appId!, async () => {
+    if (input.environmentVariables) {
+      return context.doInEnvironmentContext(input.environmentVariables, () => {
+        return run()
+      })
+    } else {
+      return run()
+    }
   })
 }

package/src/utilities/fileSystem/app.ts

@@ -0,0 +1,86 @@
+import { budibaseTempDir } from "../budibaseDir"
+import fs from "fs"
+import { join } from "path"
+import { ObjectStoreBuckets } from "../../constants"
+import { updateClientLibrary } from "./clientLibrary"
+import env from "../../environment"
+import { objectStore, context } from "@budibase/backend-core"
+import { TOP_LEVEL_PATH } from "./filesystem"
+
+export const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
+
+/**
+ * Uploads the latest client library to the object store.
+ * @param {string} appId The ID of the app which is being created.
+ * @return {Promise<void>} once promise completes app resources should be ready in object store.
+ */
+export const createApp = async (appId: string) => {
+  await updateClientLibrary(appId)
+}
+
+/**
+ * Removes all of the assets created for an app in the object store.
+ * @param {string} appId The ID of the app which is being deleted.
+ * @return {Promise<void>} once promise completes the app resources will be removed from object store.
+ */
+export const deleteApp = async (appId: string) => {
+  await objectStore.deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`)
+}
+
+/**
+ * Retrieves component libraries from object store (or tmp symlink if in local)
+ */
+export const getComponentLibraryManifest = async (library: string) => {
+  const appId = context.getAppId()
+  const filename = "manifest.json"
+  /* istanbul ignore next */
+  // when testing in cypress and so on we need to get the package
+  // as the environment may not be fully fleshed out for dev or prod
+  if (env.isTest()) {
+    library = library.replace("standard-components", "client")
+    const lib = library.split("/")[1]
+    const path = require.resolve(library).split(lib)[0]
+    return require(join(path, lib, filename))
+  } else if (env.isDev()) {
+    const path = join(NODE_MODULES_PATH, "@budibase", "client", filename)
+    // always load from new so that updates are refreshed
+    delete require.cache[require.resolve(path)]
+    return require(path)
+  }
+
+  if (!appId) {
+    throw new Error("No app ID found - cannot get component libraries")
+  }
+
+  let resp
+  let path
+  try {
+    // Try to load the manifest from the new file location
+    path = join(appId, filename)
+    resp = await objectStore.retrieve(ObjectStoreBuckets.APPS, path)
+  } catch (error) {
+    console.error(
+      `component-manifest-objectstore=failed appId=${appId} path=${path}`,
+      error
+    )
+    // Fallback to loading it from the old location for old apps
+    path = join(appId, "node_modules", library, "package", filename)
+    resp = await objectStore.retrieve(ObjectStoreBuckets.APPS, path)
+  }
+  if (typeof resp !== "string") {
+    resp = resp.toString("utf8")
+  }
+  return JSON.parse(resp)
+}
+
+/**
+ * Given a set of app IDs makes sure file system is cleared of any of their temp info.
+ */
+export const cleanup = (appIds: string[]) => {
+  for (let appId of appIds) {
+    const path = join(budibaseTempDir(), appId)
+    if (fs.existsSync(path)) {
+      fs.rmdirSync(path, { recursive: true })
+    }
+  }
+}

package/src/utilities/fileSystem/clientLibrary.ts

@@ -4,7 +4,7 @@ import fs from "fs"
 import { objectStore } from "@budibase/backend-core"
 import { resolve } from "../centralPath"
 import env from "../../environment"
-const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
+import { TOP_LEVEL_PATH } from "./filesystem"
 
 /**
  * Client library paths in the object store:

package/src/utilities/fileSystem/filesystem.ts

@@ -0,0 +1,172 @@
+import { PathLike } from "fs"
+import fs from "fs"
+import { budibaseTempDir } from "../budibaseDir"
+import { join } from "path"
+import env from "../../environment"
+import tar from "tar"
+const uuid = require("uuid/v4")
+
+export const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
+
+/**
+ * Upon first startup of instance there may not be everything we need in tmp directory, set it up.
+ */
+export const init = () => {
+  const tempDir = budibaseTempDir()
+  if (!fs.existsSync(tempDir)) {
+    // some test cases fire this quickly enough that
+    // synchronous cases can end up here at the same time
+    try {
+      fs.mkdirSync(tempDir)
+    } catch (err: any) {
+      if (!err || err.code !== "EEXIST") {
+        throw err
+      }
+    }
+  }
+  const clientLibPath = join(budibaseTempDir(), "budibase-client.js")
+  if (env.isTest() && !fs.existsSync(clientLibPath)) {
+    fs.copyFileSync(require.resolve("@budibase/client"), clientLibPath)
+  }
+}
+
+/**
+ * Checks if the system is currently in development mode and if it is makes sure
+ * everything required to function is ready.
+ */
+export const checkDevelopmentEnvironment = () => {
+  if (!env.isDev() || env.isTest()) {
+    return
+  }
+  if (!fs.existsSync(budibaseTempDir())) {
+    fs.mkdirSync(budibaseTempDir())
+  }
+  let error
+  if (!fs.existsSync(join(process.cwd(), ".env"))) {
+    error = "Must run via yarn once to generate environment."
+  }
+  if (error) {
+    console.error(error)
+    process.exit(-1)
+  }
+}
+
+/**
+ * Used to retrieve a handlebars file from the system which will be used as a template.
+ * This is allowable as the template handlebars files should be static and identical across
+ * the cluster.
+ * @param {string} path The path to the handlebars file which is to be loaded.
+ * @returns {string} The loaded handlebars file as a string - loaded as utf8.
+ */
+export const loadHandlebarsFile = (path: PathLike) => {
+  return fs.readFileSync(path, "utf8")
+}
+
+/**
+ * When return a file from the API need to write the file to the system temporarily so we
+ * can create a read stream to send.
+ * @param {string} contents the contents of the file which is to be returned from the API.
+ * @return {Object} the read stream which can be put into the koa context body.
+ */
+export const apiFileReturn = (contents: any) => {
+  const path = join(budibaseTempDir(), uuid())
+  fs.writeFileSync(path, contents)
+  return fs.createReadStream(path)
+}
+
+export const streamFile = (path: string) => {
+  return fs.createReadStream(path)
+}
+
+/**
+ * Writes the provided contents to a temporary file, which can be used briefly.
+ * @param {string} fileContents contents which will be written to a temp file.
+ * @return {string} the path to the temp file.
+ */
+export const storeTempFile = (fileContents: any) => {
+  const path = join(budibaseTempDir(), uuid())
+  fs.writeFileSync(path, fileContents)
+  return path
+}
+
+/**
+ * Utility function for getting a file read stream - a simple in memory buffered read
+ * stream doesn't work for pouchdb.
+ */
+export const stringToFileStream = (contents: any) => {
+  const path = storeTempFile(contents)
+  return fs.createReadStream(path)
+}
+
+/**
+ * Creates a temp file and returns it from the API.
+ * @param {string} fileContents the contents to be returned in file.
+ */
+export const sendTempFile = (fileContents: any) => {
+  const path = storeTempFile(fileContents)
+  return fs.createReadStream(path)
+}
+
+/**
+ * All file reads come through here just to make sure all of them make sense
+ * allows a centralised location to check logic is all good.
+ */
+export const readFileSync = (filepath: PathLike, options = "utf8") => {
+  // @ts-ignore
+  return fs.readFileSync(filepath, options)
+}
+
+export const createTempFolder = (item: any) => {
+  const path = join(budibaseTempDir(), item)
+  try {
+    // remove old tmp directories automatically - don't combine
+    if (fs.existsSync(path)) {
+      fs.rmSync(path, { recursive: true, force: true })
+    }
+    fs.mkdirSync(path)
+  } catch (err: any) {
+    throw new Error(`Path cannot be created: ${err.message}`)
+  }
+
+  return path
+}
+
+export const extractTarball = async (fromFilePath: string, toPath: string) => {
+  await tar.extract({
+    file: fromFilePath,
+    C: toPath,
+  })
+}
+
+/**
+ * Find for a file recursively from start path applying filter, return first match
+ */
+export const findFileRec = (startPath: PathLike, filter: string): any => {
+  if (!fs.existsSync(startPath)) {
+    return
+  }
+
+  const files = fs.readdirSync(startPath)
+  for (let i = 0, len = files.length; i < len; i++) {
+    // @ts-ignore
+    const filename = join(startPath, files[i])
+    const stat = fs.lstatSync(filename)
+
+    if (stat.isDirectory()) {
+      return findFileRec(filename, filter)
+    } else if (filename.endsWith(filter)) {
+      return filename
+    }
+  }
+}
+
+/**
+ * Remove a folder which is not empty from the file system
+ */
+export const deleteFolderFileSystem = (path: PathLike) => {
+  if (!fs.existsSync(path)) {
+    return
+  }
+
+  fs.rmSync(path, { recursive: true, force: true })
+}