@budibase/server 2.2.10 → 2.2.11-alpha.1

package/src/sdk/app/backups/imports.ts

@@ -55,12 +55,8 @@ async function updateAttachmentColumns(prodAppId: string, db: Database) {
           continue
         }
         row[column] = row[column].map((attachment: RowAttachment) => {
-          // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv
-          const urlParts = attachment.url.split("/")
-          // drop the first empty element
-          urlParts.shift()
-          // get the prefix
-          const prefix = urlParts.shift()
+          // Key looks like: appId/attachments/file.csv
+          const urlParts = attachment.key.split("/")
           // remove the app ID
           urlParts.shift()
           // add new app ID
@@ -69,7 +65,7 @@ async function updateAttachmentColumns(prodAppId: string, db: Database) {
           return {
             ...attachment,
             key,
-            url: `/${prefix}/${key}`,
+            url: "", // calculated on retrieval using key
           }
         })
       }

package/src/tests/jestSetup.ts

@@ -19,7 +19,10 @@ import { mocks } from "@budibase/backend-core/tests"
 const tk = require("timekeeper")
 tk.freeze(mocks.date.MOCK_DATE)
 
-global.console.log = jest.fn() // console.log are ignored in tests
+if (!process.env.DEBUG) {
+  global.console.log = jest.fn() // console.log are ignored in tests
+  global.console.warn = jest.fn() // console.warn are ignored in tests
+}
 
 if (!process.env.CI) {
   // set a longer timeout in dev for debugging

package/src/tests/utilities/TestConfiguration.ts

@@ -1,3 +1,12 @@
+import { mocks } from "@budibase/backend-core/tests"
+
+// init the licensing mock
+import * as pro from "@budibase/pro"
+mocks.licenses.init(pro)
+
+// use unlimited license by default
+mocks.licenses.useUnlimited()
+
 import { init as dbInit } from "../../db"
 dbInit()
 import env from "../../environment"
@@ -351,7 +360,6 @@ class TestConfiguration {
   }
 
   // APP
-
   async createApp(appName: string) {
     // create dev app
     // clear any old app
@@ -364,7 +372,7 @@ class TestConfiguration {
     await context.updateAppId(this.appId)
 
     // create production app
-    this.prodApp = await this.deploy()
+    this.prodApp = await this.publish()
 
     this.allApps.push(this.prodApp)
     this.allApps.push(this.app)
@@ -372,8 +380,8 @@ class TestConfiguration {
     return this.app
   }
 
-  async deploy() {
-    await this._req(null, null, controllers.deploy.deployApp)
+  async publish() {
+    await this._req(null, null, controllers.deploy.publishApp)
     // @ts-ignore
     const prodAppId = this.getAppId().replace("_dev", "")
     this.prodAppId = prodAppId
@@ -384,6 +392,17 @@ class TestConfiguration {
     })
   }
 
+  async unpublish() {
+    const response = await this._req(
+      null,
+      { appId: this.appId },
+      controllers.app.unpublish
+    )
+    this.prodAppId = null
+    this.prodApp = null
+    return response
+  }
+
   // TABLE
 
   async updateTable(config?: any) {

package/src/threads/automation.ts

@@ -32,31 +32,8 @@ const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
 const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED }
 
-function typecastForLooping(loopStep: LoopStep, input: LoopInput) {
-  if (!input || !input.binding) {
-    return null
-  }
-  try {
-    switch (loopStep.inputs.option) {
-      case LoopStepType.ARRAY:
-        if (typeof input.binding === "string") {
-          return JSON.parse(input.binding)
-        }
-        break
-      case LoopStepType.STRING:
-        if (Array.isArray(input.binding)) {
-          return input.binding.join(",")
-        }
-        break
-    }
-  } catch (err) {
-    throw new Error("Unable to cast to correct type")
-  }
-  return input.binding
-}
-
 function getLoopIterations(loopStep: LoopStep, input: LoopInput) {
-  const binding = typecastForLooping(loopStep, input)
+  const binding = automationUtils.typecastForLooping(loopStep, input)
   if (!loopStep || !binding) {
     return 1
   }
@@ -289,7 +266,7 @@ class Orchestrator {
 
           let tempOutput = { items: loopSteps, iterations: iterationCount }
           try {
-            newInput.binding = typecastForLooping(
+            newInput.binding = automationUtils.typecastForLooping(
               loopStep as LoopStep,
               newInput
             )
@@ -442,7 +419,7 @@ class Orchestrator {
 
       // Delete the step after the loop step as it's irrelevant, since information is included
       // in the loop step
-      if (wasLoopStep) {
+      if (wasLoopStep && !loopStep) {
         this._context.steps.splice(loopStepNumber + 1, 1)
         wasLoopStep = false
       }
@@ -461,8 +438,8 @@ class Orchestrator {
         })
         this._context.steps[loopStepNumber] = tempOutput
 
-        loopSteps = undefined
         wasLoopStep = true
+        loopSteps = []
       }
     }
 

package/src/utilities/fileSystem/app.ts

@@ -0,0 +1,86 @@
+import { budibaseTempDir } from "../budibaseDir"
+import fs from "fs"
+import { join } from "path"
+import { ObjectStoreBuckets } from "../../constants"
+import { updateClientLibrary } from "./clientLibrary"
+import env from "../../environment"
+import { objectStore, context } from "@budibase/backend-core"
+import { TOP_LEVEL_PATH } from "./filesystem"
+
+export const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
+
+/**
+ * Uploads the latest client library to the object store.
+ * @param {string} appId The ID of the app which is being created.
+ * @return {Promise<void>} once promise completes app resources should be ready in object store.
+ */
+export const createApp = async (appId: string) => {
+  await updateClientLibrary(appId)
+}
+
+/**
+ * Removes all of the assets created for an app in the object store.
+ * @param {string} appId The ID of the app which is being deleted.
+ * @return {Promise<void>} once promise completes the app resources will be removed from object store.
+ */
+export const deleteApp = async (appId: string) => {
+  await objectStore.deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`)
+}
+
+/**
+ * Retrieves component libraries from object store (or tmp symlink if in local)
+ */
+export const getComponentLibraryManifest = async (library: string) => {
+  const appId = context.getAppId()
+  const filename = "manifest.json"
+  /* istanbul ignore next */
+  // when testing in cypress and so on we need to get the package
+  // as the environment may not be fully fleshed out for dev or prod
+  if (env.isTest()) {
+    library = library.replace("standard-components", "client")
+    const lib = library.split("/")[1]
+    const path = require.resolve(library).split(lib)[0]
+    return require(join(path, lib, filename))
+  } else if (env.isDev()) {
+    const path = join(NODE_MODULES_PATH, "@budibase", "client", filename)
+    // always load from new so that updates are refreshed
+    delete require.cache[require.resolve(path)]
+    return require(path)
+  }
+
+  if (!appId) {
+    throw new Error("No app ID found - cannot get component libraries")
+  }
+
+  let resp
+  let path
+  try {
+    // Try to load the manifest from the new file location
+    path = join(appId, filename)
+    resp = await objectStore.retrieve(ObjectStoreBuckets.APPS, path)
+  } catch (error) {
+    console.error(
+      `component-manifest-objectstore=failed appId=${appId} path=${path}`,
+      error
+    )
+    // Fallback to loading it from the old location for old apps
+    path = join(appId, "node_modules", library, "package", filename)
+    resp = await objectStore.retrieve(ObjectStoreBuckets.APPS, path)
+  }
+  if (typeof resp !== "string") {
+    resp = resp.toString("utf8")
+  }
+  return JSON.parse(resp)
+}
+
+/**
+ * Given a set of app IDs makes sure file system is cleared of any of their temp info.
+ */
+export const cleanup = (appIds: string[]) => {
+  for (let appId of appIds) {
+    const path = join(budibaseTempDir(), appId)
+    if (fs.existsSync(path)) {
+      fs.rmdirSync(path, { recursive: true })
+    }
+  }
+}

package/src/utilities/fileSystem/clientLibrary.ts

@@ -4,7 +4,7 @@ import fs from "fs"
 import { objectStore } from "@budibase/backend-core"
 import { resolve } from "../centralPath"
 import env from "../../environment"
-const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
+import { TOP_LEVEL_PATH } from "./filesystem"
 
 /**
  * Client library paths in the object store:

package/src/utilities/fileSystem/filesystem.ts

@@ -0,0 +1,170 @@
+import { PathLike } from "fs"
+const { budibaseTempDir } = require("../budibaseDir")
+const fs = require("fs")
+const { join } = require("path")
+const uuid = require("uuid/v4")
+const env = require("../../environment")
+import tar from "tar"
+
+export const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
+
+/**
+ * Upon first startup of instance there may not be everything we need in tmp directory, set it up.
+ */
+export const init = () => {
+  const tempDir = budibaseTempDir()
+  if (!fs.existsSync(tempDir)) {
+    // some test cases fire this quickly enough that
+    // synchronous cases can end up here at the same time
+    try {
+      fs.mkdirSync(tempDir)
+    } catch (err: any) {
+      if (!err || err.code !== "EEXIST") {
+        throw err
+      }
+    }
+  }
+  const clientLibPath = join(budibaseTempDir(), "budibase-client.js")
+  if (env.isTest() && !fs.existsSync(clientLibPath)) {
+    fs.copyFileSync(require.resolve("@budibase/client"), clientLibPath)
+  }
+}
+
+/**
+ * Checks if the system is currently in development mode and if it is makes sure
+ * everything required to function is ready.
+ */
+export const checkDevelopmentEnvironment = () => {
+  if (!env.isDev() || env.isTest()) {
+    return
+  }
+  if (!fs.existsSync(budibaseTempDir())) {
+    fs.mkdirSync(budibaseTempDir())
+  }
+  let error
+  if (!fs.existsSync(join(process.cwd(), ".env"))) {
+    error = "Must run via yarn once to generate environment."
+  }
+  if (error) {
+    console.error(error)
+    process.exit(-1)
+  }
+}
+
+/**
+ * Used to retrieve a handlebars file from the system which will be used as a template.
+ * This is allowable as the template handlebars files should be static and identical across
+ * the cluster.
+ * @param {string} path The path to the handlebars file which is to be loaded.
+ * @returns {string} The loaded handlebars file as a string - loaded as utf8.
+ */
+export const loadHandlebarsFile = (path: PathLike) => {
+  return fs.readFileSync(path, "utf8")
+}
+
+/**
+ * When return a file from the API need to write the file to the system temporarily so we
+ * can create a read stream to send.
+ * @param {string} contents the contents of the file which is to be returned from the API.
+ * @return {Object} the read stream which can be put into the koa context body.
+ */
+export const apiFileReturn = (contents: any) => {
+  const path = join(budibaseTempDir(), uuid())
+  fs.writeFileSync(path, contents)
+  return fs.createReadStream(path)
+}
+
+export const streamFile = (path: string) => {
+  return fs.createReadStream(path)
+}
+
+/**
+ * Writes the provided contents to a temporary file, which can be used briefly.
+ * @param {string} fileContents contents which will be written to a temp file.
+ * @return {string} the path to the temp file.
+ */
+export const storeTempFile = (fileContents: any) => {
+  const path = join(budibaseTempDir(), uuid())
+  fs.writeFileSync(path, fileContents)
+  return path
+}
+
+/**
+ * Utility function for getting a file read stream - a simple in memory buffered read
+ * stream doesn't work for pouchdb.
+ */
+export const stringToFileStream = (contents: any) => {
+  const path = storeTempFile(contents)
+  return fs.createReadStream(path)
+}
+
+/**
+ * Creates a temp file and returns it from the API.
+ * @param {string} fileContents the contents to be returned in file.
+ */
+export const sendTempFile = (fileContents: any) => {
+  const path = storeTempFile(fileContents)
+  return fs.createReadStream(path)
+}
+
+/**
+ * All file reads come through here just to make sure all of them make sense
+ * allows a centralised location to check logic is all good.
+ */
+export const readFileSync = (filepath: PathLike, options = "utf8") => {
+  return fs.readFileSync(filepath, options)
+}
+
+export const createTempFolder = (item: any) => {
+  const path = join(budibaseTempDir(), item)
+  try {
+    // remove old tmp directories automatically - don't combine
+    if (fs.existsSync(path)) {
+      fs.rmSync(path, { recursive: true, force: true })
+    }
+    fs.mkdirSync(path)
+  } catch (err: any) {
+    throw new Error(`Path cannot be created: ${err.message}`)
+  }
+
+  return path
+}
+
+export const extractTarball = async (fromFilePath: string, toPath: string) => {
+  await tar.extract({
+    file: fromFilePath,
+    C: toPath,
+  })
+}
+
+/**
+ * Find for a file recursively from start path applying filter, return first match
+ */
+export const findFileRec = (startPath: PathLike, filter: string): any => {
+  if (!fs.existsSync(startPath)) {
+    return
+  }
+
+  const files = fs.readdirSync(startPath)
+  for (let i = 0, len = files.length; i < len; i++) {
+    const filename = join(startPath, files[i])
+    const stat = fs.lstatSync(filename)
+
+    if (stat.isDirectory()) {
+      return findFileRec(filename, filter)
+    } else if (filename.endsWith(filter)) {
+      return filename
+    }
+  }
+}
+
+/**
+ * Remove a folder which is not empty from the file system
+ */
+export const deleteFolderFileSystem = (path: PathLike) => {
+  if (!fs.existsSync(path)) {
+    return
+  }
+
+  fs.rmSync(path, { recursive: true, force: true })
+}