@budibase/server 2.2.12-alpha.5 → 2.2.13

package/src/tests/utilities/TestConfiguration.ts

@@ -1,12 +1,3 @@
-import { mocks } from "@budibase/backend-core/tests"
-
-// init the licensing mock
-import * as pro from "@budibase/pro"
-mocks.licenses.init(pro)
-
-// use unlimited license by default
-mocks.licenses.useUnlimited()
-
 import { init as dbInit } from "../../db"
 dbInit()
 import env from "../../environment"
@@ -360,6 +351,7 @@ class TestConfiguration {
   }
 
   // APP
+
   async createApp(appName: string) {
     // create dev app
     // clear any old app
@@ -372,7 +364,7 @@ class TestConfiguration {
     await context.updateAppId(this.appId)
 
     // create production app
-    this.prodApp = await this.publish()
+    this.prodApp = await this.deploy()
 
     this.allApps.push(this.prodApp)
     this.allApps.push(this.app)
@@ -380,8 +372,8 @@ class TestConfiguration {
     return this.app
   }
 
-  async publish() {
-    await this._req(null, null, controllers.deploy.publishApp)
+  async deploy() {
+    await this._req(null, null, controllers.deploy.deployApp)
     // @ts-ignore
     const prodAppId = this.getAppId().replace("_dev", "")
     this.prodAppId = prodAppId
@@ -392,17 +384,6 @@ class TestConfiguration {
     })
   }
 
-  async unpublish() {
-    const response = await this._req(
-      null,
-      { appId: this.appId },
-      controllers.app.unpublish
-    )
-    this.prodAppId = null
-    this.prodApp = null
-    return response
-  }
-
   // TABLE
 
   async updateTable(config?: any) {

package/src/threads/automation.ts

@@ -32,8 +32,31 @@ const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
 const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED }
 
+function typecastForLooping(loopStep: LoopStep, input: LoopInput) {
+  if (!input || !input.binding) {
+    return null
+  }
+  try {
+    switch (loopStep.inputs.option) {
+      case LoopStepType.ARRAY:
+        if (typeof input.binding === "string") {
+          return JSON.parse(input.binding)
+        }
+        break
+      case LoopStepType.STRING:
+        if (Array.isArray(input.binding)) {
+          return input.binding.join(",")
+        }
+        break
+    }
+  } catch (err) {
+    throw new Error("Unable to cast to correct type")
+  }
+  return input.binding
+}
+
 function getLoopIterations(loopStep: LoopStep, input: LoopInput) {
-  const binding = automationUtils.typecastForLooping(loopStep, input)
+  const binding = typecastForLooping(loopStep, input)
   if (!loopStep || !binding) {
     return 1
   }
@@ -266,7 +289,7 @@ class Orchestrator {
 
           let tempOutput = { items: loopSteps, iterations: iterationCount }
           try {
-            newInput.binding = automationUtils.typecastForLooping(
+            newInput.binding = typecastForLooping(
               loopStep as LoopStep,
               newInput
             )
@@ -419,7 +442,7 @@ class Orchestrator {
 
       // Delete the step after the loop step as it's irrelevant, since information is included
       // in the loop step
-      if (wasLoopStep && !loopStep) {
+      if (wasLoopStep) {
         this._context.steps.splice(loopStepNumber + 1, 1)
         wasLoopStep = false
       }
@@ -438,8 +461,8 @@ class Orchestrator {
         })
         this._context.steps[loopStepNumber] = tempOutput
 
+        loopSteps = undefined
         wasLoopStep = true
-        loopSteps = []
       }
     }
 

package/src/utilities/fileSystem/clientLibrary.ts

@@ -4,7 +4,7 @@ import fs from "fs"
 import { objectStore } from "@budibase/backend-core"
 import { resolve } from "../centralPath"
 import env from "../../environment"
-import { TOP_LEVEL_PATH } from "./filesystem"
+const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
 
 /**
  * Client library paths in the object store:

package/src/utilities/fileSystem/index.ts

@@ -1,5 +1,348 @@
-export * from "./app"
-export * from "./clientLibrary"
-export * from "./filesystem"
-export * from "./plugin"
-export * from "./template"
+import { budibaseTempDir } from "../budibaseDir"
+import fs from "fs"
+import { join } from "path"
+import { context, objectStore } from "@budibase/backend-core"
+import { ObjectStoreBuckets } from "../../constants"
+import { updateClientLibrary } from "./clientLibrary"
+import { checkSlashesInUrl } from "../"
+import env from "../../environment"
+import fetch from "node-fetch"
+const uuid = require("uuid/v4")
+const tar = require("tar")
+
+export const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
+export const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
+const DATASOURCE_PATH = join(budibaseTempDir(), "datasource")
+
+/**
+ * The single stack system (Cloud and Builder) should not make use of the file system where possible,
+ * this file handles all of the file access for the system with the intention of limiting it all to one
+ * place. Keeping all of this logic in one place means that when we need to do file system access (like
+ * downloading a package or opening a temporary file) in can be done in way that we can confirm it shouldn't
+ * be done through an object store instead.
+ */
+
+/**
+ * Upon first startup of instance there may not be everything we need in tmp directory, set it up.
+ */
+export function init() {
+  const tempDir = budibaseTempDir()
+  if (!fs.existsSync(tempDir)) {
+    // some test cases fire this quickly enough that
+    // synchronous cases can end up here at the same time
+    try {
+      fs.mkdirSync(tempDir)
+    } catch (err: any) {
+      if (!err || err.code !== "EEXIST") {
+        throw err
+      }
+    }
+  }
+  const clientLibPath = join(budibaseTempDir(), "budibase-client.js")
+  if (env.isTest() && !fs.existsSync(clientLibPath)) {
+    fs.copyFileSync(require.resolve("@budibase/client"), clientLibPath)
+  }
+}
+
+/**
+ * Checks if the system is currently in development mode and if it is makes sure
+ * everything required to function is ready.
+ */
+exports.checkDevelopmentEnvironment = () => {
+  if (!env.isDev() || env.isTest()) {
+    return
+  }
+  if (!fs.existsSync(budibaseTempDir())) {
+    fs.mkdirSync(budibaseTempDir())
+  }
+  let error
+  if (!fs.existsSync(join(process.cwd(), ".env"))) {
+    error = "Must run via yarn once to generate environment."
+  }
+  if (error) {
+    console.error(error)
+    process.exit(-1)
+  }
+}
+
+/**
+ * Used to retrieve a handlebars file from the system which will be used as a template.
+ * This is allowable as the template handlebars files should be static and identical across
+ * the cluster.
+ * @param {string} path The path to the handlebars file which is to be loaded.
+ * @returns {string} The loaded handlebars file as a string - loaded as utf8.
+ */
+export function loadHandlebarsFile(path: string) {
+  return fs.readFileSync(path, "utf8")
+}
+
+/**
+ * When return a file from the API need to write the file to the system temporarily so we
+ * can create a read stream to send.
+ * @param {string} contents the contents of the file which is to be returned from the API.
+ * @param {string} encoding the encoding of the file to return (utf8 default)
+ * @return {Object} the read stream which can be put into the koa context body.
+ */
+export function apiFileReturn(
+  contents: string,
+  encoding: BufferEncoding = "utf8"
+) {
+  const path = join(budibaseTempDir(), uuid())
+  fs.writeFileSync(path, contents, { encoding })
+  return fs.createReadStream(path, { encoding })
+}
+
+export function streamFile(path: string) {
+  return fs.createReadStream(path)
+}
+
+/**
+ * Writes the provided contents to a temporary file, which can be used briefly.
+ * @param {string} fileContents contents which will be written to a temp file.
+ * @return {string} the path to the temp file.
+ */
+export function storeTempFile(fileContents: string) {
+  const path = join(budibaseTempDir(), uuid())
+  fs.writeFileSync(path, fileContents)
+  return path
+}
+
+/**
+ * Utility function for getting a file read stream - a simple in memory buffered read
+ * stream doesn't work for pouchdb.
+ */
+export function stringToFileStream(contents: string) {
+  const path = exports.storeTempFile(contents)
+  return fs.createReadStream(path)
+}
+
+/**
+ * Creates a temp file and returns it from the API.
+ * @param {string} fileContents the contents to be returned in file.
+ */
+export function sendTempFile(fileContents: string) {
+  const path = exports.storeTempFile(fileContents)
+  return fs.createReadStream(path)
+}
+
+/**
+ * Uploads the latest client library to the object store.
+ * @param {string} appId The ID of the app which is being created.
+ * @return {Promise<void>} once promise completes app resources should be ready in object store.
+ */
+export async function createApp(appId: string) {
+  await updateClientLibrary(appId)
+}
+
+/**
+ * Removes all of the assets created for an app in the object store.
+ * @param {string} appId The ID of the app which is being deleted.
+ * @return {Promise<void>} once promise completes the app resources will be removed from object store.
+ */
+export async function deleteApp(appId: string) {
+  await objectStore.deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`)
+}
+
+/**
+ * Retrieves a template and pipes it to minio as well as making it available temporarily.
+ * @param {string} type The type of template which is to be retrieved.
+ * @param name
+ * @return {Promise<*>}
+ */
+export async function downloadTemplate(type: string, name: string) {
+  const DEFAULT_TEMPLATES_BUCKET =
+    "prod-budi-templates.s3-eu-west-1.amazonaws.com"
+  const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz`
+  return objectStore.downloadTarball(
+    templateUrl,
+    ObjectStoreBuckets.TEMPLATES,
+    type
+  )
+}
+
+/**
+ * Retrieves component libraries from object store (or tmp symlink if in local)
+ */
+export async function getComponentLibraryManifest(library: string) {
+  const appId = context.getAppId()
+  const filename = "manifest.json"
+  /* istanbul ignore next */
+  // when testing in cypress and so on we need to get the package
+  // as the environment may not be fully fleshed out for dev or prod
+  if (env.isTest()) {
+    library = library.replace("standard-components", "client")
+    const lib = library.split("/")[1]
+    const path = require.resolve(library).split(lib)[0]
+    return require(join(path, lib, filename))
+  } else if (env.isDev()) {
+    const path = join(NODE_MODULES_PATH, "@budibase", "client", filename)
+    // always load from new so that updates are refreshed
+    delete require.cache[require.resolve(path)]
+    return require(path)
+  }
+
+  if (!appId) {
+    throw new Error("No app ID found - cannot get component libraries")
+  }
+
+  let resp
+  let path
+  try {
+    // Try to load the manifest from the new file location
+    path = join(appId, filename)
+    resp = await objectStore.retrieve(ObjectStoreBuckets.APPS, path)
+  } catch (error) {
+    console.error(
+      `component-manifest-objectstore=failed appId=${appId} path=${path}`,
+      error
+    )
+    // Fallback to loading it from the old location for old apps
+    path = join(appId, "node_modules", library, "package", filename)
+    resp = await objectStore.retrieve(ObjectStoreBuckets.APPS, path)
+  }
+  if (typeof resp !== "string") {
+    resp = resp.toString("utf8")
+  }
+  return JSON.parse(resp)
+}
+
+/**
+ * All file reads come through here just to make sure all of them make sense
+ * allows a centralised location to check logic is all good.
+ */
+export function readFileSync(
+  filepath: string,
+  options: BufferEncoding = "utf8"
+) {
+  return fs.readFileSync(filepath, { encoding: options })
+}
+
+/**
+ * Given a set of app IDs makes sure file system is cleared of any of their temp info.
+ */
+export function cleanup(appIds: string[]) {
+  for (let appId of appIds) {
+    const path = join(budibaseTempDir(), appId)
+    if (fs.existsSync(path)) {
+      fs.rmdirSync(path, { recursive: true })
+    }
+  }
+}
+
+export function createTempFolder(item: string) {
+  const path = join(budibaseTempDir(), item)
+  try {
+    // remove old tmp directories automatically - don't combine
+    if (fs.existsSync(path)) {
+      fs.rmSync(path, { recursive: true, force: true })
+    }
+    fs.mkdirSync(path)
+  } catch (err: any) {
+    throw new Error(`Path cannot be created: ${err.message}`)
+  }
+
+  return path
+}
+
+export async function extractTarball(fromFilePath: string, toPath: string) {
+  await tar.extract({
+    file: fromFilePath,
+    C: toPath,
+  })
+}
+
+export async function getPluginMetadata(path: string) {
+  let metadata: { schema?: any; package?: any } = {}
+  try {
+    const pkg = fs.readFileSync(join(path, "package.json"), "utf8")
+    const schema = fs.readFileSync(join(path, "schema.json"), "utf8")
+
+    metadata.schema = JSON.parse(schema)
+    metadata.package = JSON.parse(pkg)
+
+    if (
+      !metadata.package.name ||
+      !metadata.package.version ||
+      !metadata.package.description
+    ) {
+      throw new Error(
+        "package.json is missing one of 'name', 'version' or 'description'."
+      )
+    }
+  } catch (err: any) {
+    throw new Error(
+      `Unable to process schema.json/package.json in plugin. ${err.message}`
+    )
+  }
+
+  return { metadata, directory: path }
+}
+
+export async function getDatasourcePlugin(
+  name: string,
+  url: string,
+  hash: string
+) {
+  if (!fs.existsSync(DATASOURCE_PATH)) {
+    fs.mkdirSync(DATASOURCE_PATH)
+  }
+  const filename = join(DATASOURCE_PATH, name)
+  const metadataName = `${filename}.bbmetadata`
+  if (fs.existsSync(filename)) {
+    const currentHash = fs.readFileSync(metadataName, "utf8")
+    // if hash is the same return the file, otherwise remove it and re-download
+    if (currentHash === hash) {
+      return require(filename)
+    } else {
+      console.log(`Updating plugin: ${name}`)
+      delete require.cache[require.resolve(filename)]
+      fs.unlinkSync(filename)
+    }
+  }
+  const fullUrl = checkSlashesInUrl(
+    `${env.MINIO_URL}/${ObjectStoreBuckets.PLUGINS}/${url}`
+  )
+  const response = await fetch(fullUrl)
+  if (response.status === 200) {
+    const content = await response.text()
+    fs.writeFileSync(filename, content)
+    fs.writeFileSync(metadataName, hash)
+    return require(filename)
+  } else {
+    throw new Error(
+      `Unable to retrieve plugin - reason: ${await response.text()}`
+    )
+  }
+}
+
+/**
+ * Find for a file recursively from start path applying filter, return first match
+ */
+export function findFileRec(startPath: string, filter: any) {
+  if (!fs.existsSync(startPath)) {
+    return
+  }
+
+  const files = fs.readdirSync(startPath)
+  for (let i = 0, len = files.length; i < len; i++) {
+    const filename = join(startPath, files[i])
+    const stat = fs.lstatSync(filename)
+
+    if (stat.isDirectory()) {
+      return exports.findFileRec(filename, filter)
+    } else if (filename.endsWith(filter)) {
+      return filename
+    }
+  }
+}
+
+/**
+ * Remove a folder which is not empty from the file system
+ */
+export function deleteFolderFileSystem(path: string) {
+  if (!fs.existsSync(path)) {
+    return
+  }
+
+  fs.rmSync(path, { recursive: true, force: true })
+}

package/src/utilities/index.ts

@@ -1,5 +1,6 @@
 import env from "../environment"
-import { context } from "@budibase/backend-core"
+import { OBJ_STORE_DIRECTORY } from "../constants"
+import { objectStore, context } from "@budibase/backend-core"
 import { generateMetadataID } from "../db/utils"
 import { Document } from "@budibase/types"
 import stream from "stream"
@@ -31,6 +32,49 @@ export function checkSlashesInUrl(url: string) {
   return url.replace(/(https?:\/\/)|(\/)+/g, "$1$2")
 }
 
+/**
+ * Gets the address of the object store, depending on whether self hosted or in cloud.
+ * @return {string} The base URL of the object store (MinIO or S3).
+ */
+export function objectStoreUrl() {
+  if (env.SELF_HOSTED || env.MINIO_URL) {
+    // can use a relative url for this as all goes through the proxy (this is hosted in minio)
+    return OBJ_STORE_DIRECTORY
+  } else {
+    return env.CDN_URL
+  }
+}
+
+/**
+ * In production the client library is stored in the object store, however in development
+ * we use the symlinked version produced by lerna, located in node modules. We link to this
+ * via a specific endpoint (under /api/assets/client).
+ * @param {string} appId In production we need the appId to look up the correct bucket, as the
+ * version of the client lib may differ between apps.
+ * @param {string} version The version to retrieve.
+ * @return {string} The URL to be inserted into appPackage response or server rendered
+ * app index file.
+ */
+export function clientLibraryPath(appId: string, version: string) {
+  if (env.isProd()) {
+    let url = `${objectStoreUrl()}/${objectStore.sanitizeKey(
+      appId
+    )}/budibase-client.js`
+
+    // append app version to bust the cache
+    if (version) {
+      url += `?v=${version}`
+    }
+    return url
+  } else {
+    return `/api/assets/client`
+  }
+}
+
+export function attachmentsRelativeURL(attachmentKey: string) {
+  return checkSlashesInUrl(`${objectStoreUrl()}/${attachmentKey}`)
+}
+
 export async function updateEntityMetadata(
   type: string,
   entityId: string,

package/src/utilities/plugins.ts

@@ -0,0 +1,22 @@
+import env from "../environment"
+import { plugins as ProPlugins } from "@budibase/pro"
+import { objectStore } from "@budibase/backend-core"
+import { Plugin } from "@budibase/types"
+
+export function enrichPluginURLs(plugins: Plugin[]) {
+  if (!plugins || !plugins.length) {
+    return []
+  }
+  return plugins.map(plugin => {
+    const cloud = !env.SELF_HOSTED
+    const bucket = objectStore.ObjectStoreBuckets.PLUGINS
+    const jsFileName = "plugin.min.js"
+
+    // In self host we need to prefix the path, as the bucket name is not part
+    // of the bucket path. In cloud, it's already part of the bucket path.
+    let jsUrl = cloud ? `${env.CDN_URL}/` : `/${bucket}/`
+    jsUrl += ProPlugins.getBucketPath(plugin.name)
+    jsUrl += jsFileName
+    return { ...plugin, jsUrl }
+  })
+}

package/src/utilities/rowProcessor/index.ts

@@ -1,11 +1,12 @@
 import * as linkRows from "../../db/linkedRows"
 import { FieldTypes, AutoFieldSubTypes } from "../../constants"
+import { attachmentsRelativeURL } from "../index"
 import { processFormulas, fixAutoColumnSubType } from "./utils"
 import { ObjectStoreBuckets } from "../../constants"
 import { context, db as dbCore, objectStore } from "@budibase/backend-core"
 import { InternalTables } from "../../db/utils"
 import { TYPE_TRANSFORM_MAP } from "./map"
-import { Row, RowAttachment, Table, ContextUser } from "@budibase/types"
+import { Row, Table, ContextUser } from "@budibase/types"
 const { cloneDeep } = require("lodash/fp")
 export * from "./utils"
 
@@ -34,7 +35,7 @@ function getRemovedAttachmentKeys(
     return oldKeys
   }
   const newKeys = row[attachmentKey].map((attachment: any) => attachment.key)
-  return oldKeys.filter((key: string) => newKeys.indexOf(key) === -1)
+  return oldKeys.filter((key: any) => newKeys.indexOf(key) === -1)
 }
 
 /**
@@ -104,7 +105,7 @@ export function processAutoColumn(
  * @param {object} type The type fo coerce to
  * @returns {object} The coerced value
  */
-export function coerce(row: any, type: string) {
+export function coerce(row: any, type: any) {
   // no coercion specified for type, skip it
   if (!TYPE_TRANSFORM_MAP[type]) {
     return row
@@ -157,16 +158,6 @@ export function inputProcessing(
     else {
       clonedRow[key] = coerce(value, field.type)
     }
-
-    // remove any attachment urls, they are generated on read
-    if (field.type === FieldTypes.ATTACHMENT) {
-      const attachments = clonedRow[key]
-      if (attachments?.length) {
-        attachments.forEach((attachment: RowAttachment) => {
-          delete attachment.url
-        })
-      }
-    }
   }
 
   if (!clonedRow._id || !clonedRow._rev) {
@@ -203,15 +194,15 @@ export async function outputProcessing(
   // process formulas
   enriched = processFormulas(table, enriched, { dynamic: true }) as Row[]
 
-  // set the attachments URLs
+  // update the attachments URL depending on hosting
   for (let [property, column] of Object.entries(table.schema)) {
     if (column.type === FieldTypes.ATTACHMENT) {
       for (let row of enriched) {
         if (row[property] == null || !Array.isArray(row[property])) {
           continue
         }
-        row[property].forEach((attachment: RowAttachment) => {
-          attachment.url = objectStore.getAppFileUrl(attachment.key)
+        row[property].forEach((attachment: any) => {
+          attachment.url = attachmentsRelativeURL(attachment.key)
         })
       }
     }
@@ -274,6 +265,6 @@ export async function cleanupAttachments(
     }
   }
   if (files.length > 0) {
-    await objectStore.deleteFiles(ObjectStoreBuckets.APPS, files)
+    return objectStore.deleteFiles(ObjectStoreBuckets.APPS, files)
   }
 }

package/tsconfig.build.json

@@ -19,8 +19,6 @@
     "node_modules",
     "dist",
     "src/tests",
-    "src/api/routes/tests/utilities",
-    "src/automations/tests/utilities",
     "**/*.spec.ts",
     "**/*.spec.js"
   ]