@budibase/server 2.1.46-alpha.12 → 2.1.46-alpha.14

package/src/api/controllers/row/staticFormula.ts

@@ -16,7 +16,10 @@ const { cloneDeep } = require("lodash/fp")
  * updated.
  * NOTE: this will only for affect static formulas.
  */
-export async function updateRelatedFormula(table: Table, enrichedRows: Row[]) {
+export async function updateRelatedFormula(
+  table: Table,
+  enrichedRows: Row[] | Row
+) {
   const db = context.getAppDB()
   // no formula to update, we're done
   if (!table.relatedFormula) {
@@ -155,7 +158,7 @@ export async function finaliseRow(
   enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false })
   // this updates the related formulas in other rows based on the relations to this row
   if (updateFormula) {
-    await exports.updateRelatedFormula(table, enrichedRow)
+    await updateRelatedFormula(table, enrichedRow)
   }
   return { row: enrichedRow, table }
 }

package/src/api/controllers/row/utils.ts

@@ -7,6 +7,7 @@ import { BBContext, Row, Table } from "@budibase/types"
 export { removeKeyNumbering } from "../../../integrations/base/utils"
 const validateJs = require("validate.js")
 const { cloneDeep } = require("lodash/fp")
+import { Ctx } from "@budibase/types"
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value: string) {
@@ -25,7 +26,7 @@ export async function getDatasourceAndQuery(json: any) {
   return makeExternalQuery(datasource, json)
 }
 
-export async function findRow(ctx: BBContext, tableId: string, rowId: string) {
+export async function findRow(ctx: Ctx, tableId: string, rowId: string) {
   const db = context.getAppDB()
   let row
   // TODO remove special user case in future

package/src/api/controllers/static/index.ts

@@ -1,11 +1,9 @@
-import { enrichPluginURLs } from "../../../utilities/plugins"
-
 require("svelte/register")
 
 const send = require("koa-send")
 const { resolve, join } = require("../../../utilities/centralPath")
 const uuid = require("uuid")
-const { ObjectStoreBuckets } = require("../../../constants")
+import { ObjectStoreBuckets } from "../../../constants"
 const { processString } = require("@budibase/string-templates")
 const {
   loadHandlebarsFile,
@@ -13,8 +11,6 @@ const {
   TOP_LEVEL_PATH,
 } = require("../../../utilities/fileSystem")
 const env = require("../../../environment")
-const { clientLibraryPath } = require("../../../utilities")
-const { attachmentsRelativeURL } = require("../../../utilities")
 const { DocumentType } = require("../../../db/utils")
 const { context, objectStore, utils } = require("@budibase/backend-core")
 const AWS = require("aws-sdk")
@@ -33,7 +29,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }: any) {
   return {
     size: file.size,
     name: file.name,
-    url: attachmentsRelativeURL(response.Key),
+    url: objectStore.getAppFileUrl(s3Key),
     extension: [...file.name.split(".")].pop(),
     key: response.Key,
   }
@@ -85,7 +81,7 @@ export const uploadFile = async function (ctx: any) {
 
     return prepareUpload({
       file,
-      s3Key: `${ctx.appId}/attachments/${processedFileName}`,
+      s3Key: `${context.getProdAppId()}/attachments/${processedFileName}`,
       bucket: ObjectStoreBuckets.APPS,
     })
   })
@@ -107,14 +103,14 @@ export const serveApp = async function (ctx: any) {
 
   if (!env.isJest()) {
     const App = require("./templates/BudibaseApp.svelte").default
-    const plugins = enrichPluginURLs(appInfo.usedPlugins)
+    const plugins = objectStore.enrichPluginURLs(appInfo.usedPlugins)
     const { head, html, css } = App.render({
       metaImage:
         "https://res.cloudinary.com/daog6scxm/image/upload/v1666109324/meta-images/budibase-meta-image_uukc1m.png",
       title: appInfo.name,
       production: env.isProd(),
       appId,
-      clientLibPath: clientLibraryPath(appId, appInfo.version, ctx),
+      clientLibPath: objectStore.clientLibraryUrl(appId, appInfo.version),
       usedPlugins: plugins,
     })
 
@@ -139,7 +135,7 @@ export const serveBuilderPreview = async function (ctx: any) {
     let appId = context.getAppId()
     const previewHbs = loadHandlebarsFile(`${__dirname}/templates/preview.hbs`)
     ctx.body = await processString(previewHbs, {
-      clientLibPath: clientLibraryPath(appId, appInfo.version, ctx),
+      clientLibPath: objectStore.clientLibraryUrl(appId, appInfo.version),
     })
   } else {
     // just return the app info for jest to assert on

package/src/api/routes/index.ts

@@ -26,13 +26,14 @@ import cloudRoutes from "./cloud"
 import migrationRoutes from "./migrations"
 import pluginRoutes from "./plugin"
 import Router from "@koa/router"
-import { api } from "@budibase/pro"
+import { api as pro } from "@budibase/pro"
 
 export { default as staticRoutes } from "./static"
 export { default as publicRoutes } from "./public"
 
-const appBackupRoutes = api.appBackups
-const scheduleRoutes = api.schedules
+const appBackupRoutes = pro.appBackups
+const scheduleRoutes = pro.schedules
+
 export const mainRoutes: Router[] = [
   appBackupRoutes,
   backupRoutes,

package/src/api/routes/tests/row.spec.js

@@ -10,6 +10,7 @@ const {
   StaticQuotaName,
   MonthlyQuotaName,
 } = require("@budibase/types")
+const { structures } = require("@budibase/backend-core/tests");
 
 describe("/rows", () => {
   let request = setup.getRequest()
@@ -494,12 +495,13 @@ describe("/rows", () => {
   describe("attachments", () => {
     it("should allow enriching attachment rows", async () => {
       const table = await config.createAttachmentTable()
+      const attachmentId = `${structures.uuid()}.csv`
       const row = await config.createRow({
         name: "test",
         description: "test",
         attachment: [
           {
-            key: `${config.getAppId()}/attachments/test/thing.csv`,
+            key: `${config.getAppId()}/attachments/${attachmentId}`,
           },
         ],
         tableId: table._id,
@@ -509,7 +511,7 @@ describe("/rows", () => {
         context.doInAppContext(config.getAppId(), async () => {
           const enriched = await outputProcessing(table, [row])
           expect(enriched[0].attachment[0].url).toBe(
-            `/prod-budi-app-assets/${config.getAppId()}/attachments/test/thing.csv`
+            `/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
           )
         })
       })

package/src/api/routes/tests/static.spec.js

@@ -1,20 +1,5 @@
-jest.mock("node-fetch")
-jest.mock("aws-sdk", () => ({
-  config: {
-    update: jest.fn(),
-  },
-  DynamoDB: {
-    DocumentClient: jest.fn(),
-  },
-  S3: jest.fn(() => ({
-    getSignedUrl: jest.fn(() => {
-      return "my-url"
-    }),
-  })),
-}))
-
 const setup = require("./utilities")
-const { events, constants } = require("@budibase/backend-core")
+const { constants } = require("@budibase/backend-core")
 
 describe("/static", () => {
   let request = setup.getRequest()
@@ -102,7 +87,7 @@ describe("/static", () => {
           .set(config.defaultHeaders())
           .expect("Content-Type", /json/)
           .expect(200)
-        expect(res.body.signedUrl).toEqual("my-url")
+        expect(res.body.signedUrl).toEqual("http://test.com/foo/bar")
         expect(res.body.publicUrl).toEqual(
           `https://${bucket}.s3.eu-west-1.amazonaws.com/${key}`
         )

package/src/environment.ts

@@ -112,12 +112,17 @@ if (isDev() && module.exports.DISABLE_THREADING == null) {
 }
 
 // clean up any environment variable edge cases
-for (let [key, value] of Object.entries(module.exports)) {
+for (let [key, value] of Object.entries(environment)) {
   // handle the edge case of "0" to disable an environment variable
   if (value === "0") {
     // @ts-ignore
     environment[key] = 0
   }
+  // handle the edge case of "false" to disable an environment variable
+  if (value === "false") {
+    // @ts-ignore
+    environment[key] = 0
+  }
 }
 
 export = environment

package/src/integrations/index.ts

@@ -98,11 +98,7 @@ export async function getIntegration(integration: string) {
     for (let plugin of plugins) {
       if (plugin.name === integration) {
         // need to use commonJS require due to its dynamic runtime nature
-        const retrieved: any = await getDatasourcePlugin(
-          plugin.name,
-          plugin.jsUrl,
-          plugin.schema?.hash
-        )
+        const retrieved: any = await getDatasourcePlugin(plugin)
         if (retrieved.integration) {
           return retrieved.integration
         } else {

package/src/migrations/functions/backfill/global/configs.ts

@@ -5,6 +5,7 @@ import {
   isGoogleConfig,
   isOIDCConfig,
   isSettingsConfig,
+  ConfigType,
 } from "@budibase/types"
 import env from "./../../../../environment"
 
@@ -31,15 +32,15 @@ export const backfill = async (
       await events.email.SMTPCreated(timestamp)
     }
     if (isGoogleConfig(config)) {
-      await events.auth.SSOCreated("google", timestamp)
+      await events.auth.SSOCreated(ConfigType.GOOGLE, timestamp)
       if (config.config.activated) {
-        await events.auth.SSOActivated("google", timestamp)
+        await events.auth.SSOActivated(ConfigType.GOOGLE, timestamp)
       }
     }
     if (isOIDCConfig(config)) {
-      await events.auth.SSOCreated("oidc", timestamp)
+      await events.auth.SSOCreated(ConfigType.OIDC, timestamp)
       if (config.config.configs[0].activated) {
-        await events.auth.SSOActivated("oidc", timestamp)
+        await events.auth.SSOActivated(ConfigType.OIDC, timestamp)
       }
     }
     if (isSettingsConfig(config)) {

package/src/sdk/app/backups/imports.ts

@@ -55,12 +55,8 @@ async function updateAttachmentColumns(prodAppId: string, db: Database) {
           continue
         }
         row[column] = row[column].map((attachment: RowAttachment) => {
-          // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv
-          const urlParts = attachment.url.split("/")
-          // drop the first empty element
-          urlParts.shift()
-          // get the prefix
-          const prefix = urlParts.shift()
+          // Key looks like: appId/attachments/file.csv
+          const urlParts = attachment.key.split("/")
           // remove the app ID
           urlParts.shift()
           // add new app ID
@@ -69,7 +65,7 @@ async function updateAttachmentColumns(prodAppId: string, db: Database) {
           return {
             ...attachment,
             key,
-            url: `/${prefix}/${key}`,
+            url: "", // calculated on retrieval using key
           }
         })
       }

package/src/tests/jestSetup.ts

@@ -19,7 +19,10 @@ import { mocks } from "@budibase/backend-core/tests"
 const tk = require("timekeeper")
 tk.freeze(mocks.date.MOCK_DATE)
 
-global.console.log = jest.fn() // console.log are ignored in tests
+if (!process.env.DEBUG) {
+  global.console.log = jest.fn() // console.log are ignored in tests
+  global.console.warn = jest.fn() // console.warn are ignored in tests
+}
 
 if (!process.env.CI) {
   // set a longer timeout in dev for debugging

package/src/utilities/fileSystem/app.ts

@@ -0,0 +1,86 @@
+import { budibaseTempDir } from "../budibaseDir"
+import fs from "fs"
+import { join } from "path"
+import { ObjectStoreBuckets } from "../../constants"
+import { updateClientLibrary } from "./clientLibrary"
+import env from "../../environment"
+import { objectStore, context } from "@budibase/backend-core"
+import { TOP_LEVEL_PATH } from "./filesystem"
+
+export const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
+
+/**
+ * Uploads the latest client library to the object store.
+ * @param {string} appId The ID of the app which is being created.
+ * @return {Promise<void>} once promise completes app resources should be ready in object store.
+ */
+export const createApp = async (appId: string) => {
+  await updateClientLibrary(appId)
+}
+
+/**
+ * Removes all of the assets created for an app in the object store.
+ * @param {string} appId The ID of the app which is being deleted.
+ * @return {Promise<void>} once promise completes the app resources will be removed from object store.
+ */
+export const deleteApp = async (appId: string) => {
+  await objectStore.deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`)
+}
+
+/**
+ * Retrieves component libraries from object store (or tmp symlink if in local)
+ */
+export const getComponentLibraryManifest = async (library: string) => {
+  const appId = context.getAppId()
+  const filename = "manifest.json"
+  /* istanbul ignore next */
+  // when testing in cypress and so on we need to get the package
+  // as the environment may not be fully fleshed out for dev or prod
+  if (env.isTest()) {
+    library = library.replace("standard-components", "client")
+    const lib = library.split("/")[1]
+    const path = require.resolve(library).split(lib)[0]
+    return require(join(path, lib, filename))
+  } else if (env.isDev()) {
+    const path = join(NODE_MODULES_PATH, "@budibase", "client", filename)
+    // always load from new so that updates are refreshed
+    delete require.cache[require.resolve(path)]
+    return require(path)
+  }
+
+  if (!appId) {
+    throw new Error("No app ID found - cannot get component libraries")
+  }
+
+  let resp
+  let path
+  try {
+    // Try to load the manifest from the new file location
+    path = join(appId, filename)
+    resp = await objectStore.retrieve(ObjectStoreBuckets.APPS, path)
+  } catch (error) {
+    console.error(
+      `component-manifest-objectstore=failed appId=${appId} path=${path}`,
+      error
+    )
+    // Fallback to loading it from the old location for old apps
+    path = join(appId, "node_modules", library, "package", filename)
+    resp = await objectStore.retrieve(ObjectStoreBuckets.APPS, path)
+  }
+  if (typeof resp !== "string") {
+    resp = resp.toString("utf8")
+  }
+  return JSON.parse(resp)
+}
+
+/**
+ * Given a set of app IDs makes sure file system is cleared of any of their temp info.
+ */
+export const cleanup = (appIds: string[]) => {
+  for (let appId of appIds) {
+    const path = join(budibaseTempDir(), appId)
+    if (fs.existsSync(path)) {
+      fs.rmdirSync(path, { recursive: true })
+    }
+  }
+}

package/src/utilities/fileSystem/clientLibrary.ts

@@ -4,7 +4,7 @@ import fs from "fs"
 import { objectStore } from "@budibase/backend-core"
 import { resolve } from "../centralPath"
 import env from "../../environment"
-const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
+import { TOP_LEVEL_PATH } from "./filesystem"
 
 /**
  * Client library paths in the object store:

package/src/utilities/fileSystem/filesystem.ts

@@ -0,0 +1,170 @@
+import { PathLike } from "fs"
+const { budibaseTempDir } = require("../budibaseDir")
+const fs = require("fs")
+const { join } = require("path")
+const uuid = require("uuid/v4")
+const env = require("../../environment")
+import tar from "tar"
+
+export const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
+
+/**
+ * Upon first startup of instance there may not be everything we need in tmp directory, set it up.
+ */
+export const init = () => {
+  const tempDir = budibaseTempDir()
+  if (!fs.existsSync(tempDir)) {
+    // some test cases fire this quickly enough that
+    // synchronous cases can end up here at the same time
+    try {
+      fs.mkdirSync(tempDir)
+    } catch (err: any) {
+      if (!err || err.code !== "EEXIST") {
+        throw err
+      }
+    }
+  }
+  const clientLibPath = join(budibaseTempDir(), "budibase-client.js")
+  if (env.isTest() && !fs.existsSync(clientLibPath)) {
+    fs.copyFileSync(require.resolve("@budibase/client"), clientLibPath)
+  }
+}
+
+/**
+ * Checks if the system is currently in development mode and if it is makes sure
+ * everything required to function is ready.
+ */
+export const checkDevelopmentEnvironment = () => {
+  if (!env.isDev() || env.isTest()) {
+    return
+  }
+  if (!fs.existsSync(budibaseTempDir())) {
+    fs.mkdirSync(budibaseTempDir())
+  }
+  let error
+  if (!fs.existsSync(join(process.cwd(), ".env"))) {
+    error = "Must run via yarn once to generate environment."
+  }
+  if (error) {
+    console.error(error)
+    process.exit(-1)
+  }
+}
+
+/**
+ * Used to retrieve a handlebars file from the system which will be used as a template.
+ * This is allowable as the template handlebars files should be static and identical across
+ * the cluster.
+ * @param {string} path The path to the handlebars file which is to be loaded.
+ * @returns {string} The loaded handlebars file as a string - loaded as utf8.
+ */
+export const loadHandlebarsFile = (path: PathLike) => {
+  return fs.readFileSync(path, "utf8")
+}
+
+/**
+ * When return a file from the API need to write the file to the system temporarily so we
+ * can create a read stream to send.
+ * @param {string} contents the contents of the file which is to be returned from the API.
+ * @return {Object} the read stream which can be put into the koa context body.
+ */
+export const apiFileReturn = (contents: any) => {
+  const path = join(budibaseTempDir(), uuid())
+  fs.writeFileSync(path, contents)
+  return fs.createReadStream(path)
+}
+
+export const streamFile = (path: string) => {
+  return fs.createReadStream(path)
+}
+
+/**
+ * Writes the provided contents to a temporary file, which can be used briefly.
+ * @param {string} fileContents contents which will be written to a temp file.
+ * @return {string} the path to the temp file.
+ */
+export const storeTempFile = (fileContents: any) => {
+  const path = join(budibaseTempDir(), uuid())
+  fs.writeFileSync(path, fileContents)
+  return path
+}
+
+/**
+ * Utility function for getting a file read stream - a simple in memory buffered read
+ * stream doesn't work for pouchdb.
+ */
+export const stringToFileStream = (contents: any) => {
+  const path = storeTempFile(contents)
+  return fs.createReadStream(path)
+}
+
+/**
+ * Creates a temp file and returns it from the API.
+ * @param {string} fileContents the contents to be returned in file.
+ */
+export const sendTempFile = (fileContents: any) => {
+  const path = storeTempFile(fileContents)
+  return fs.createReadStream(path)
+}
+
+/**
+ * All file reads come through here just to make sure all of them make sense
+ * allows a centralised location to check logic is all good.
+ */
+export const readFileSync = (filepath: PathLike, options = "utf8") => {
+  return fs.readFileSync(filepath, options)
+}
+
+export const createTempFolder = (item: any) => {
+  const path = join(budibaseTempDir(), item)
+  try {
+    // remove old tmp directories automatically - don't combine
+    if (fs.existsSync(path)) {
+      fs.rmSync(path, { recursive: true, force: true })
+    }
+    fs.mkdirSync(path)
+  } catch (err: any) {
+    throw new Error(`Path cannot be created: ${err.message}`)
+  }
+
+  return path
+}
+
+export const extractTarball = async (fromFilePath: string, toPath: string) => {
+  await tar.extract({
+    file: fromFilePath,
+    C: toPath,
+  })
+}
+
+/**
+ * Find for a file recursively from start path applying filter, return first match
+ */
+export const findFileRec = (startPath: PathLike, filter: string): any => {
+  if (!fs.existsSync(startPath)) {
+    return
+  }
+
+  const files = fs.readdirSync(startPath)
+  for (let i = 0, len = files.length; i < len; i++) {
+    const filename = join(startPath, files[i])
+    const stat = fs.lstatSync(filename)
+
+    if (stat.isDirectory()) {
+      return findFileRec(filename, filter)
+    } else if (filename.endsWith(filter)) {
+      return filename
+    }
+  }
+}
+
+/**
+ * Remove a folder which is not empty from the file system
+ */
+export const deleteFolderFileSystem = (path: PathLike) => {
+  if (!fs.existsSync(path)) {
+    return
+  }
+
+  fs.rmSync(path, { recursive: true, force: true })
+}