@budibase/server 2.2.26 → 2.2.27-alpha.1

package/src/utilities/fileSystem/index.ts

@@ -1,348 +1,5 @@
-import { budibaseTempDir } from "../budibaseDir"
-import fs from "fs"
-import { join } from "path"
-import { context, objectStore } from "@budibase/backend-core"
-import { ObjectStoreBuckets } from "../../constants"
-import { updateClientLibrary } from "./clientLibrary"
-import { checkSlashesInUrl } from "../"
-import env from "../../environment"
-import fetch from "node-fetch"
-const uuid = require("uuid/v4")
-const tar = require("tar")
-
-export const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
-export const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
-const DATASOURCE_PATH = join(budibaseTempDir(), "datasource")
-
-/**
- * The single stack system (Cloud and Builder) should not make use of the file system where possible,
- * this file handles all of the file access for the system with the intention of limiting it all to one
- * place. Keeping all of this logic in one place means that when we need to do file system access (like
- * downloading a package or opening a temporary file) in can be done in way that we can confirm it shouldn't
- * be done through an object store instead.
- */
-
-/**
- * Upon first startup of instance there may not be everything we need in tmp directory, set it up.
- */
-export function init() {
-  const tempDir = budibaseTempDir()
-  if (!fs.existsSync(tempDir)) {
-    // some test cases fire this quickly enough that
-    // synchronous cases can end up here at the same time
-    try {
-      fs.mkdirSync(tempDir)
-    } catch (err: any) {
-      if (!err || err.code !== "EEXIST") {
-        throw err
-      }
-    }
-  }
-  const clientLibPath = join(budibaseTempDir(), "budibase-client.js")
-  if (env.isTest() && !fs.existsSync(clientLibPath)) {
-    fs.copyFileSync(require.resolve("@budibase/client"), clientLibPath)
-  }
-}
-
-/**
- * Checks if the system is currently in development mode and if it is makes sure
- * everything required to function is ready.
- */
-exports.checkDevelopmentEnvironment = () => {
-  if (!env.isDev() || env.isTest()) {
-    return
-  }
-  if (!fs.existsSync(budibaseTempDir())) {
-    fs.mkdirSync(budibaseTempDir())
-  }
-  let error
-  if (!fs.existsSync(join(process.cwd(), ".env"))) {
-    error = "Must run via yarn once to generate environment."
-  }
-  if (error) {
-    console.error(error)
-    process.exit(-1)
-  }
-}
-
-/**
- * Used to retrieve a handlebars file from the system which will be used as a template.
- * This is allowable as the template handlebars files should be static and identical across
- * the cluster.
- * @param {string} path The path to the handlebars file which is to be loaded.
- * @returns {string} The loaded handlebars file as a string - loaded as utf8.
- */
-export function loadHandlebarsFile(path: string) {
-  return fs.readFileSync(path, "utf8")
-}
-
-/**
- * When return a file from the API need to write the file to the system temporarily so we
- * can create a read stream to send.
- * @param {string} contents the contents of the file which is to be returned from the API.
- * @param {string} encoding the encoding of the file to return (utf8 default)
- * @return {Object} the read stream which can be put into the koa context body.
- */
-export function apiFileReturn(
-  contents: string,
-  encoding: BufferEncoding = "utf8"
-) {
-  const path = join(budibaseTempDir(), uuid())
-  fs.writeFileSync(path, contents, { encoding })
-  return fs.createReadStream(path, { encoding })
-}
-
-export function streamFile(path: string) {
-  return fs.createReadStream(path)
-}
-
-/**
- * Writes the provided contents to a temporary file, which can be used briefly.
- * @param {string} fileContents contents which will be written to a temp file.
- * @return {string} the path to the temp file.
- */
-export function storeTempFile(fileContents: string) {
-  const path = join(budibaseTempDir(), uuid())
-  fs.writeFileSync(path, fileContents)
-  return path
-}
-
-/**
- * Utility function for getting a file read stream - a simple in memory buffered read
- * stream doesn't work for pouchdb.
- */
-export function stringToFileStream(contents: string) {
-  const path = exports.storeTempFile(contents)
-  return fs.createReadStream(path)
-}
-
-/**
- * Creates a temp file and returns it from the API.
- * @param {string} fileContents the contents to be returned in file.
- */
-export function sendTempFile(fileContents: string) {
-  const path = exports.storeTempFile(fileContents)
-  return fs.createReadStream(path)
-}
-
-/**
- * Uploads the latest client library to the object store.
- * @param {string} appId The ID of the app which is being created.
- * @return {Promise<void>} once promise completes app resources should be ready in object store.
- */
-export async function createApp(appId: string) {
-  await updateClientLibrary(appId)
-}
-
-/**
- * Removes all of the assets created for an app in the object store.
- * @param {string} appId The ID of the app which is being deleted.
- * @return {Promise<void>} once promise completes the app resources will be removed from object store.
- */
-export async function deleteApp(appId: string) {
-  await objectStore.deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`)
-}
-
-/**
- * Retrieves a template and pipes it to minio as well as making it available temporarily.
- * @param {string} type The type of template which is to be retrieved.
- * @param name
- * @return {Promise<*>}
- */
-export async function downloadTemplate(type: string, name: string) {
-  const DEFAULT_TEMPLATES_BUCKET =
-    "prod-budi-templates.s3-eu-west-1.amazonaws.com"
-  const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz`
-  return objectStore.downloadTarball(
-    templateUrl,
-    ObjectStoreBuckets.TEMPLATES,
-    type
-  )
-}
-
-/**
- * Retrieves component libraries from object store (or tmp symlink if in local)
- */
-export async function getComponentLibraryManifest(library: string) {
-  const appId = context.getAppId()
-  const filename = "manifest.json"
-  /* istanbul ignore next */
-  // when testing in cypress and so on we need to get the package
-  // as the environment may not be fully fleshed out for dev or prod
-  if (env.isTest()) {
-    library = library.replace("standard-components", "client")
-    const lib = library.split("/")[1]
-    const path = require.resolve(library).split(lib)[0]
-    return require(join(path, lib, filename))
-  } else if (env.isDev()) {
-    const path = join(NODE_MODULES_PATH, "@budibase", "client", filename)
-    // always load from new so that updates are refreshed
-    delete require.cache[require.resolve(path)]
-    return require(path)
-  }
-
-  if (!appId) {
-    throw new Error("No app ID found - cannot get component libraries")
-  }
-
-  let resp
-  let path
-  try {
-    // Try to load the manifest from the new file location
-    path = join(appId, filename)
-    resp = await objectStore.retrieve(ObjectStoreBuckets.APPS, path)
-  } catch (error) {
-    console.error(
-      `component-manifest-objectstore=failed appId=${appId} path=${path}`,
-      error
-    )
-    // Fallback to loading it from the old location for old apps
-    path = join(appId, "node_modules", library, "package", filename)
-    resp = await objectStore.retrieve(ObjectStoreBuckets.APPS, path)
-  }
-  if (typeof resp !== "string") {
-    resp = resp.toString("utf8")
-  }
-  return JSON.parse(resp)
-}
-
-/**
- * All file reads come through here just to make sure all of them make sense
- * allows a centralised location to check logic is all good.
- */
-export function readFileSync(
-  filepath: string,
-  options: BufferEncoding = "utf8"
-) {
-  return fs.readFileSync(filepath, { encoding: options })
-}
-
-/**
- * Given a set of app IDs makes sure file system is cleared of any of their temp info.
- */
-export function cleanup(appIds: string[]) {
-  for (let appId of appIds) {
-    const path = join(budibaseTempDir(), appId)
-    if (fs.existsSync(path)) {
-      fs.rmdirSync(path, { recursive: true })
-    }
-  }
-}
-
-export function createTempFolder(item: string) {
-  const path = join(budibaseTempDir(), item)
-  try {
-    // remove old tmp directories automatically - don't combine
-    if (fs.existsSync(path)) {
-      fs.rmSync(path, { recursive: true, force: true })
-    }
-    fs.mkdirSync(path)
-  } catch (err: any) {
-    throw new Error(`Path cannot be created: ${err.message}`)
-  }
-
-  return path
-}
-
-export async function extractTarball(fromFilePath: string, toPath: string) {
-  await tar.extract({
-    file: fromFilePath,
-    C: toPath,
-  })
-}
-
-export async function getPluginMetadata(path: string) {
-  let metadata: { schema?: any; package?: any } = {}
-  try {
-    const pkg = fs.readFileSync(join(path, "package.json"), "utf8")
-    const schema = fs.readFileSync(join(path, "schema.json"), "utf8")
-
-    metadata.schema = JSON.parse(schema)
-    metadata.package = JSON.parse(pkg)
-
-    if (
-      !metadata.package.name ||
-      !metadata.package.version ||
-      !metadata.package.description
-    ) {
-      throw new Error(
-        "package.json is missing one of 'name', 'version' or 'description'."
-      )
-    }
-  } catch (err: any) {
-    throw new Error(
-      `Unable to process schema.json/package.json in plugin. ${err.message}`
-    )
-  }
-
-  return { metadata, directory: path }
-}
-
-export async function getDatasourcePlugin(
-  name: string,
-  url: string,
-  hash: string
-) {
-  if (!fs.existsSync(DATASOURCE_PATH)) {
-    fs.mkdirSync(DATASOURCE_PATH)
-  }
-  const filename = join(DATASOURCE_PATH, name)
-  const metadataName = `${filename}.bbmetadata`
-  if (fs.existsSync(filename)) {
-    const currentHash = fs.readFileSync(metadataName, "utf8")
-    // if hash is the same return the file, otherwise remove it and re-download
-    if (currentHash === hash) {
-      return require(filename)
-    } else {
-      console.log(`Updating plugin: ${name}`)
-      delete require.cache[require.resolve(filename)]
-      fs.unlinkSync(filename)
-    }
-  }
-  const fullUrl = checkSlashesInUrl(
-    `${env.MINIO_URL}/${ObjectStoreBuckets.PLUGINS}/${url}`
-  )
-  const response = await fetch(fullUrl)
-  if (response.status === 200) {
-    const content = await response.text()
-    fs.writeFileSync(filename, content)
-    fs.writeFileSync(metadataName, hash)
-    return require(filename)
-  } else {
-    throw new Error(
-      `Unable to retrieve plugin - reason: ${await response.text()}`
-    )
-  }
-}
-
-/**
- * Find for a file recursively from start path applying filter, return first match
- */
-export function findFileRec(startPath: string, filter: any) {
-  if (!fs.existsSync(startPath)) {
-    return
-  }
-
-  const files = fs.readdirSync(startPath)
-  for (let i = 0, len = files.length; i < len; i++) {
-    const filename = join(startPath, files[i])
-    const stat = fs.lstatSync(filename)
-
-    if (stat.isDirectory()) {
-      return exports.findFileRec(filename, filter)
-    } else if (filename.endsWith(filter)) {
-      return filename
-    }
-  }
-}
-
-/**
- * Remove a folder which is not empty from the file system
- */
-export function deleteFolderFileSystem(path: string) {
-  if (!fs.existsSync(path)) {
-    return
-  }
-
-  fs.rmSync(path, { recursive: true, force: true })
-}
+export * from "./app"
+export * from "./clientLibrary"
+export * from "./filesystem"
+export * from "./plugin"
+export * from "./template"

package/src/utilities/fileSystem/plugin.ts

@@ -0,0 +1,64 @@
+import { Plugin } from "@budibase/types"
+import { budibaseTempDir } from "../budibaseDir"
+import fs from "fs"
+import { join } from "path"
+import { objectStore } from "@budibase/backend-core"
+
+const DATASOURCE_PATH = join(budibaseTempDir(), "datasource")
+
+export const getPluginMetadata = async (path: string) => {
+  let metadata: any = {}
+  try {
+    const pkg = fs.readFileSync(join(path, "package.json"), "utf8")
+    const schema = fs.readFileSync(join(path, "schema.json"), "utf8")
+
+    metadata.schema = JSON.parse(schema)
+    metadata.package = JSON.parse(pkg)
+
+    if (
+      !metadata.package.name ||
+      !metadata.package.version ||
+      !metadata.package.description
+    ) {
+      throw new Error(
+        "package.json is missing one of 'name', 'version' or 'description'."
+      )
+    }
+  } catch (err: any) {
+    throw new Error(
+      `Unable to process schema.json/package.json in plugin. ${err.message}`
+    )
+  }
+
+  return { metadata, directory: path }
+}
+
+export const getDatasourcePlugin = async (plugin: Plugin) => {
+  const hash = plugin.schema?.hash
+  if (!fs.existsSync(DATASOURCE_PATH)) {
+    fs.mkdirSync(DATASOURCE_PATH)
+  }
+  const filename = join(DATASOURCE_PATH, plugin.name)
+  const metadataName = `${filename}.bbmetadata`
+  if (fs.existsSync(filename)) {
+    const currentHash = fs.readFileSync(metadataName, "utf8")
+    // if hash is the same return the file, otherwise remove it and re-download
+    if (currentHash === hash) {
+      return require(filename)
+    } else {
+      console.log(`Updating plugin: ${plugin.name}`)
+      delete require.cache[require.resolve(filename)]
+      fs.unlinkSync(filename)
+    }
+  }
+  const pluginKey = objectStore.getPluginJSKey(plugin)
+  const pluginJs = await objectStore.retrieve(
+    objectStore.ObjectStoreBuckets.PLUGINS,
+    pluginKey
+  )
+
+  fs.writeFileSync(filename, pluginJs)
+  fs.writeFileSync(metadataName, hash)
+
+  return require(filename)
+}

package/src/utilities/fileSystem/template.ts

@@ -0,0 +1,36 @@
+import fs from "fs"
+import { join } from "path"
+import { ObjectStoreBuckets } from "../../constants"
+import { objectStore } from "@budibase/backend-core"
+
+/**
+ * This function manages temporary template files which are stored by Koa.
+ * @param {Object} template The template object retrieved from the Koa context object.
+ * @returns {Object} Returns an fs read stream which can be loaded into the database.
+ */
+export const getTemplateStream = async (template: any) => {
+  if (template.file) {
+    return fs.createReadStream(template.file.path)
+  } else {
+    const [type, name] = template.key.split("/")
+    const tmpPath = await downloadTemplate(type, name)
+    return fs.createReadStream(join(tmpPath, name, "db", "dump.txt"))
+  }
+}
+
+/**
+ * Retrieves a template and pipes it to minio as well as making it available temporarily.
+ * @param {string} type The type of template which is to be retrieved.
+ * @param name
+ * @return {Promise<*>}
+ */
+export const downloadTemplate = async (type: string, name: string) => {
+  const DEFAULT_TEMPLATES_BUCKET =
+    "prod-budi-templates.s3-eu-west-1.amazonaws.com"
+  const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz`
+  return objectStore.downloadTarball(
+    templateUrl,
+    ObjectStoreBuckets.TEMPLATES,
+    type
+  )
+}

package/src/utilities/index.ts

@@ -1,6 +1,5 @@
 import env from "../environment"
-import { OBJ_STORE_DIRECTORY } from "../constants"
-import { objectStore, context } from "@budibase/backend-core"
+import { context } from "@budibase/backend-core"
 import { generateMetadataID } from "../db/utils"
 import { Document } from "@budibase/types"
 import stream from "stream"
@@ -32,49 +31,6 @@ export function checkSlashesInUrl(url: string) {
   return url.replace(/(https?:\/\/)|(\/)+/g, "$1$2")
 }
 
-/**
- * Gets the address of the object store, depending on whether self hosted or in cloud.
- * @return {string} The base URL of the object store (MinIO or S3).
- */
-export function objectStoreUrl() {
-  if (env.SELF_HOSTED || env.MINIO_URL) {
-    // can use a relative url for this as all goes through the proxy (this is hosted in minio)
-    return OBJ_STORE_DIRECTORY
-  } else {
-    return env.CDN_URL
-  }
-}
-
-/**
- * In production the client library is stored in the object store, however in development
- * we use the symlinked version produced by lerna, located in node modules. We link to this
- * via a specific endpoint (under /api/assets/client).
- * @param {string} appId In production we need the appId to look up the correct bucket, as the
- * version of the client lib may differ between apps.
- * @param {string} version The version to retrieve.
- * @return {string} The URL to be inserted into appPackage response or server rendered
- * app index file.
- */
-export function clientLibraryPath(appId: string, version: string) {
-  if (env.isProd()) {
-    let url = `${objectStoreUrl()}/${objectStore.sanitizeKey(
-      appId
-    )}/budibase-client.js`
-
-    // append app version to bust the cache
-    if (version) {
-      url += `?v=${version}`
-    }
-    return url
-  } else {
-    return `/api/assets/client`
-  }
-}
-
-export function attachmentsRelativeURL(attachmentKey: string) {
-  return checkSlashesInUrl(`${objectStoreUrl()}/${attachmentKey}`)
-}
-
 export async function updateEntityMetadata(
   type: string,
   entityId: string,

package/src/utilities/retry.ts

@@ -0,0 +1,18 @@
+export async function retry<T extends (...arg0: any[]) => any>(
+  fn: T,
+  maxTry: number = 5,
+  retryCount = 1
+): Promise<Awaited<ReturnType<T>>> {
+  const currRetry = typeof retryCount === "number" ? retryCount : 1
+  try {
+    const result = await fn()
+    return result
+  } catch (e) {
+    console.log(`Retry ${currRetry} failed.`)
+    if (currRetry > maxTry) {
+      console.log(`All ${maxTry} retry attempts exhausted`)
+      throw e
+    }
+    return retry(fn, maxTry, currRetry + 1)
+  }
+}

package/src/utilities/rowProcessor/index.ts

@@ -1,12 +1,11 @@
 import * as linkRows from "../../db/linkedRows"
 import { FieldTypes, AutoFieldSubTypes } from "../../constants"
-import { attachmentsRelativeURL } from "../index"
 import { processFormulas, fixAutoColumnSubType } from "./utils"
 import { ObjectStoreBuckets } from "../../constants"
 import { context, db as dbCore, objectStore } from "@budibase/backend-core"
 import { InternalTables } from "../../db/utils"
 import { TYPE_TRANSFORM_MAP } from "./map"
-import { Row, Table, ContextUser } from "@budibase/types"
+import { Row, RowAttachment, Table, ContextUser } from "@budibase/types"
 const { cloneDeep } = require("lodash/fp")
 export * from "./utils"
 
@@ -35,7 +34,7 @@ function getRemovedAttachmentKeys(
     return oldKeys
   }
   const newKeys = row[attachmentKey].map((attachment: any) => attachment.key)
-  return oldKeys.filter((key: any) => newKeys.indexOf(key) === -1)
+  return oldKeys.filter((key: string) => newKeys.indexOf(key) === -1)
 }
 
 /**
@@ -105,7 +104,7 @@ export function processAutoColumn(
  * @param {object} type The type fo coerce to
  * @returns {object} The coerced value
  */
-export function coerce(row: any, type: any) {
+export function coerce(row: any, type: string) {
   // no coercion specified for type, skip it
   if (!TYPE_TRANSFORM_MAP[type]) {
     return row
@@ -158,6 +157,16 @@ export function inputProcessing(
     else {
       clonedRow[key] = coerce(value, field.type)
     }
+
+    // remove any attachment urls, they are generated on read
+    if (field.type === FieldTypes.ATTACHMENT) {
+      const attachments = clonedRow[key]
+      if (attachments?.length) {
+        attachments.forEach((attachment: RowAttachment) => {
+          delete attachment.url
+        })
+      }
+    }
   }
 
   if (!clonedRow._id || !clonedRow._rev) {
@@ -194,15 +203,15 @@ export async function outputProcessing(
   // process formulas
   enriched = processFormulas(table, enriched, { dynamic: true }) as Row[]
 
-  // update the attachments URL depending on hosting
+  // set the attachments URLs
   for (let [property, column] of Object.entries(table.schema)) {
     if (column.type === FieldTypes.ATTACHMENT) {
       for (let row of enriched) {
         if (row[property] == null || !Array.isArray(row[property])) {
           continue
         }
-        row[property].forEach((attachment: any) => {
-          attachment.url = attachmentsRelativeURL(attachment.key)
+        row[property].forEach((attachment: RowAttachment) => {
+          attachment.url = objectStore.getAppFileUrl(attachment.key)
         })
       }
     }
@@ -265,6 +274,6 @@ export async function cleanupAttachments(
     }
   }
   if (files.length > 0) {
-    return objectStore.deleteFiles(ObjectStoreBuckets.APPS, files)
+    await objectStore.deleteFiles(ObjectStoreBuckets.APPS, files)
   }
 }

package/src/utilities/rowProcessor/tests/utils.spec.ts

@@ -0,0 +1,57 @@
+import { fixAutoColumnSubType } from "../utils"
+import { AutoFieldDefaultNames, AutoFieldSubTypes } from "../../../constants"
+
+describe("rowProcessor utility", () => {
+  describe("fixAutoColumnSubType", () => {
+    let schema = {
+      name: "",
+      type: "link",
+      subtype: "", // missing subtype
+      icon: "ri-magic-line",
+      autocolumn: true,
+      constraints: { type: "array", presence: false },
+      tableId: "ta_users",
+      fieldName: "test-Updated By",
+      relationshipType: "many-to-many",
+      sortable: false,
+    }
+
+    it("updates the schema with the correct subtype", async () => {
+      schema.name = AutoFieldDefaultNames.CREATED_BY
+      expect(fixAutoColumnSubType(schema).subtype).toEqual(
+        AutoFieldSubTypes.CREATED_BY
+      )
+      schema.subtype = ""
+
+      schema.name = AutoFieldDefaultNames.UPDATED_BY
+      expect(fixAutoColumnSubType(schema).subtype).toEqual(
+        AutoFieldSubTypes.UPDATED_BY
+      )
+      schema.subtype = ""
+
+      schema.name = AutoFieldDefaultNames.CREATED_AT
+      expect(fixAutoColumnSubType(schema).subtype).toEqual(
+        AutoFieldSubTypes.CREATED_AT
+      )
+      schema.subtype = ""
+
+      schema.name = AutoFieldDefaultNames.UPDATED_AT
+      expect(fixAutoColumnSubType(schema).subtype).toEqual(
+        AutoFieldSubTypes.UPDATED_AT
+      )
+      schema.subtype = ""
+
+      schema.name = AutoFieldDefaultNames.AUTO_ID
+      expect(fixAutoColumnSubType(schema).subtype).toEqual(
+        AutoFieldSubTypes.AUTO_ID
+      )
+      schema.subtype = ""
+    })
+
+    it("returns the column if subtype exists", async () => {
+      schema.subtype = AutoFieldSubTypes.CREATED_BY
+      schema.name = AutoFieldDefaultNames.CREATED_AT
+      expect(fixAutoColumnSubType(schema)).toEqual(schema)
+    })
+  })
+})