@budibase/server 2.2.11-alpha.1 → 2.2.11

package/src/sdk/app/rows/attachments.ts

@@ -0,0 +1,60 @@
+import { CouchFindOptions, Table, Row } from "@budibase/types"
+import { db as dbCore } from "@budibase/backend-core"
+import { DocumentType, SEPARATOR } from "../../../db/utils"
+import { FieldTypes } from "../../../constants"
+
+// default limit - seems to work well for performance
+export const FIND_LIMIT = 25
+
+function generateAttachmentFindParams(
+  tableId: string,
+  attachmentCols: string[],
+  bookmark: null | string
+) {
+  const params: CouchFindOptions = {
+    selector: {
+      _id: {
+        $regex: `^${DocumentType.ROW}${SEPARATOR}${tableId}`,
+      },
+    },
+    limit: FIND_LIMIT,
+  }
+  attachmentCols.forEach(col => (params.selector[col] = { $exists: true }))
+  if (bookmark) {
+    params.bookmark = bookmark
+  }
+  return params
+}
+
+export async function getRowsWithAttachments(appId: string, table: Table) {
+  // iterate through attachment documents and update them
+  const db = dbCore.getDB(appId)
+  const attachmentCols: string[] = []
+  for (let [key, column] of Object.entries(table.schema)) {
+    if (column.type === FieldTypes.ATTACHMENT) {
+      attachmentCols.push(key)
+    }
+  }
+  // no attachment columns, nothing to do
+  if (attachmentCols.length === 0) {
+    return { rows: [], columns: [] }
+  }
+  let bookmark: null | string = null,
+    rowsLength = 0,
+    rowList: Row[] = []
+  do {
+    const params = generateAttachmentFindParams(
+      table._id!,
+      attachmentCols,
+      bookmark
+    )
+    // use the CouchDB Mango query API to lookup rows that have attachments
+    const resp = await dbCore.directCouchFind(db.name, params)
+    bookmark = resp.bookmark
+    rowsLength = resp.rows.length
+    const rows = resp.rows
+    rowList = rowList.concat(rows)
+  } while (rowsLength === FIND_LIMIT)
+  // write back the updated attachments
+  return { rows: rowList, columns: attachmentCols }
+}

package/src/sdk/app/rows/index.ts

@@ -0,0 +1,7 @@
+import * as attachments from "./attachments"
+import * as rows from "./rows"
+
+export default {
+  ...attachments,
+  ...rows,
+}

package/src/sdk/app/rows/rows.ts

@@ -0,0 +1,18 @@
+import { db as dbCore, context } from "@budibase/backend-core"
+import { Database, Row } from "@budibase/types"
+import { getRowParams } from "../../../db/utils"
+
+export async function getAllInternalRows(appId?: string) {
+  let db: Database
+  if (appId) {
+    db = dbCore.getDB(appId)
+  } else {
+    db = context.getAppDB()
+  }
+  const response = await db.allDocs(
+    getRowParams(null, null, {
+      include_docs: true,
+    })
+  )
+  return response.rows.map(row => row.doc) as Row[]
+}

package/src/sdk/index.ts

@@ -2,6 +2,7 @@ import { default as backups } from "./app/backups"
 import { default as tables } from "./app/tables"
 import { default as automations } from "./app/automations"
 import { default as applications } from "./app/applications"
+import { default as rows } from "./app/rows"
 import { default as users } from "./users"
 
 const sdk = {
@@ -9,6 +10,7 @@ const sdk = {
   tables,
   automations,
   applications,
+  rows,
   users,
 }
 

package/src/sdk/tests/attachments.spec.ts

@@ -0,0 +1,79 @@
+import newid from "../../db/newid"
+
+const attachment = {
+  size: 73479,
+  name: "2022-12-14 11_11_44-.png",
+  url: "/prod-budi-app-assets/app_bbb/attachments/a.png",
+  extension: "png",
+  key: "app_bbb/attachments/a.png",
+}
+
+const row = {
+  _id: "ro_ta_aaa",
+  photo: [attachment],
+  otherCol: "string",
+}
+
+const table = {
+  _id: "ta_aaa",
+  name: "photos",
+  schema: {
+    photo: {
+      type: "attachment",
+      name: "photo",
+    },
+    otherCol: {
+      type: "string",
+      name: "otherCol",
+    },
+  },
+}
+
+jest.mock("@budibase/backend-core", () => {
+  const core = jest.requireActual("@budibase/backend-core")
+  return {
+    ...core,
+    db: {
+      ...core.db,
+      directCouchFind: jest.fn(),
+    },
+  }
+})
+
+import { db as dbCore } from "@budibase/backend-core"
+import sdk from "../index"
+
+describe("should be able to re-write attachment URLs", () => {
+  it("it should update URLs on a number of rows over the limit", async () => {
+    const db = dbCore.getDB("app_aaa")
+    await db.put(table)
+    const limit = 30
+    let rows = []
+    for (let i = 0; i < limit; i++) {
+      const rowToWrite = {
+        ...row,
+        _id: `${row._id}_${newid()}`,
+      }
+      const { rev } = await db.put(rowToWrite)
+      rows.push({
+        ...rowToWrite,
+        _rev: rev,
+      })
+    }
+
+    dbCore.directCouchFind
+      // @ts-ignore
+      .mockReturnValueOnce({ rows: rows.slice(0, 25), bookmark: "aaa" })
+      .mockReturnValueOnce({ rows: rows.slice(25, limit), bookmark: "bbb" })
+    await sdk.backups.updateAttachmentColumns(db.name, db)
+    const finalRows = await sdk.rows.getAllInternalRows(db.name)
+    for (let rowToCheck of finalRows) {
+      expect(rowToCheck.otherCol).toBe(row.otherCol)
+      expect(rowToCheck.photo[0].url).not.toBe(row.photo[0].url)
+      expect(rowToCheck.photo[0].url).toBe(
+        `/prod-budi-app-assets/${db.name}/attachments/a.png`
+      )
+      expect(rowToCheck.photo[0].key).toBe(`${db.name}/attachments/a.png`)
+    }
+  })
+})

package/src/tests/jestSetup.ts

@@ -19,10 +19,7 @@ import { mocks } from "@budibase/backend-core/tests"
 const tk = require("timekeeper")
 tk.freeze(mocks.date.MOCK_DATE)
 
-if (!process.env.DEBUG) {
-  global.console.log = jest.fn() // console.log are ignored in tests
-  global.console.warn = jest.fn() // console.warn are ignored in tests
-}
+global.console.log = jest.fn() // console.log are ignored in tests
 
 if (!process.env.CI) {
   // set a longer timeout in dev for debugging

package/src/tests/utilities/TestConfiguration.ts

@@ -1,12 +1,3 @@
-import { mocks } from "@budibase/backend-core/tests"
-
-// init the licensing mock
-import * as pro from "@budibase/pro"
-mocks.licenses.init(pro)
-
-// use unlimited license by default
-mocks.licenses.useUnlimited()
-
 import { init as dbInit } from "../../db"
 dbInit()
 import env from "../../environment"
@@ -360,6 +351,7 @@ class TestConfiguration {
   }
 
   // APP
+
   async createApp(appName: string) {
     // create dev app
     // clear any old app
@@ -372,7 +364,7 @@ class TestConfiguration {
     await context.updateAppId(this.appId)
 
     // create production app
-    this.prodApp = await this.publish()
+    this.prodApp = await this.deploy()
 
     this.allApps.push(this.prodApp)
     this.allApps.push(this.app)
@@ -380,8 +372,8 @@ class TestConfiguration {
     return this.app
   }
 
-  async publish() {
-    await this._req(null, null, controllers.deploy.publishApp)
+  async deploy() {
+    await this._req(null, null, controllers.deploy.deployApp)
     // @ts-ignore
     const prodAppId = this.getAppId().replace("_dev", "")
     this.prodAppId = prodAppId
@@ -392,17 +384,6 @@ class TestConfiguration {
     })
   }
 
-  async unpublish() {
-    const response = await this._req(
-      null,
-      { appId: this.appId },
-      controllers.app.unpublish
-    )
-    this.prodAppId = null
-    this.prodApp = null
-    return response
-  }
-
   // TABLE
 
   async updateTable(config?: any) {

package/src/threads/automation.ts

@@ -32,8 +32,31 @@ const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
 const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED }
 
+function typecastForLooping(loopStep: LoopStep, input: LoopInput) {
+  if (!input || !input.binding) {
+    return null
+  }
+  try {
+    switch (loopStep.inputs.option) {
+      case LoopStepType.ARRAY:
+        if (typeof input.binding === "string") {
+          return JSON.parse(input.binding)
+        }
+        break
+      case LoopStepType.STRING:
+        if (Array.isArray(input.binding)) {
+          return input.binding.join(",")
+        }
+        break
+    }
+  } catch (err) {
+    throw new Error("Unable to cast to correct type")
+  }
+  return input.binding
+}
+
 function getLoopIterations(loopStep: LoopStep, input: LoopInput) {
-  const binding = automationUtils.typecastForLooping(loopStep, input)
+  const binding = typecastForLooping(loopStep, input)
   if (!loopStep || !binding) {
     return 1
   }
@@ -266,7 +289,7 @@ class Orchestrator {
 
           let tempOutput = { items: loopSteps, iterations: iterationCount }
           try {
-            newInput.binding = automationUtils.typecastForLooping(
+            newInput.binding = typecastForLooping(
               loopStep as LoopStep,
               newInput
             )
@@ -419,7 +442,7 @@ class Orchestrator {
 
       // Delete the step after the loop step as it's irrelevant, since information is included
       // in the loop step
-      if (wasLoopStep && !loopStep) {
+      if (wasLoopStep) {
         this._context.steps.splice(loopStepNumber + 1, 1)
         wasLoopStep = false
       }
@@ -438,8 +461,8 @@ class Orchestrator {
         })
         this._context.steps[loopStepNumber] = tempOutput
 
+        loopSteps = undefined
         wasLoopStep = true
-        loopSteps = []
       }
     }
 

package/src/utilities/fileSystem/clientLibrary.ts

@@ -4,7 +4,7 @@ import fs from "fs"
 import { objectStore } from "@budibase/backend-core"
 import { resolve } from "../centralPath"
 import env from "../../environment"
-import { TOP_LEVEL_PATH } from "./filesystem"
+const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
 
 /**
  * Client library paths in the object store:

package/src/utilities/fileSystem/index.ts

@@ -1,5 +1,348 @@
-export * from "./app"
-export * from "./clientLibrary"
-export * from "./filesystem"
-export * from "./plugin"
-export * from "./template"
+import { budibaseTempDir } from "../budibaseDir"
+import fs from "fs"
+import { join } from "path"
+import { context, objectStore } from "@budibase/backend-core"
+import { ObjectStoreBuckets } from "../../constants"
+import { updateClientLibrary } from "./clientLibrary"
+import { checkSlashesInUrl } from "../"
+import env from "../../environment"
+import fetch from "node-fetch"
+const uuid = require("uuid/v4")
+const tar = require("tar")
+
+export const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
+export const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
+const DATASOURCE_PATH = join(budibaseTempDir(), "datasource")
+
+/**
+ * The single stack system (Cloud and Builder) should not make use of the file system where possible,
+ * this file handles all of the file access for the system with the intention of limiting it all to one
+ * place. Keeping all of this logic in one place means that when we need to do file system access (like
+ * downloading a package or opening a temporary file) in can be done in way that we can confirm it shouldn't
+ * be done through an object store instead.
+ */
+
+/**
+ * Upon first startup of instance there may not be everything we need in tmp directory, set it up.
+ */
+export function init() {
+  const tempDir = budibaseTempDir()
+  if (!fs.existsSync(tempDir)) {
+    // some test cases fire this quickly enough that
+    // synchronous cases can end up here at the same time
+    try {
+      fs.mkdirSync(tempDir)
+    } catch (err: any) {
+      if (!err || err.code !== "EEXIST") {
+        throw err
+      }
+    }
+  }
+  const clientLibPath = join(budibaseTempDir(), "budibase-client.js")
+  if (env.isTest() && !fs.existsSync(clientLibPath)) {
+    fs.copyFileSync(require.resolve("@budibase/client"), clientLibPath)
+  }
+}
+
+/**
+ * Checks if the system is currently in development mode and if it is makes sure
+ * everything required to function is ready.
+ */
+exports.checkDevelopmentEnvironment = () => {
+  if (!env.isDev() || env.isTest()) {
+    return
+  }
+  if (!fs.existsSync(budibaseTempDir())) {
+    fs.mkdirSync(budibaseTempDir())
+  }
+  let error
+  if (!fs.existsSync(join(process.cwd(), ".env"))) {
+    error = "Must run via yarn once to generate environment."
+  }
+  if (error) {
+    console.error(error)
+    process.exit(-1)
+  }
+}
+
+/**
+ * Used to retrieve a handlebars file from the system which will be used as a template.
+ * This is allowable as the template handlebars files should be static and identical across
+ * the cluster.
+ * @param {string} path The path to the handlebars file which is to be loaded.
+ * @returns {string} The loaded handlebars file as a string - loaded as utf8.
+ */
+export function loadHandlebarsFile(path: string) {
+  return fs.readFileSync(path, "utf8")
+}
+
+/**
+ * When return a file from the API need to write the file to the system temporarily so we
+ * can create a read stream to send.
+ * @param {string} contents the contents of the file which is to be returned from the API.
+ * @param {string} encoding the encoding of the file to return (utf8 default)
+ * @return {Object} the read stream which can be put into the koa context body.
+ */
+export function apiFileReturn(
+  contents: string,
+  encoding: BufferEncoding = "utf8"
+) {
+  const path = join(budibaseTempDir(), uuid())
+  fs.writeFileSync(path, contents, { encoding })
+  return fs.createReadStream(path, { encoding })
+}
+
+export function streamFile(path: string) {
+  return fs.createReadStream(path)
+}
+
+/**
+ * Writes the provided contents to a temporary file, which can be used briefly.
+ * @param {string} fileContents contents which will be written to a temp file.
+ * @return {string} the path to the temp file.
+ */
+export function storeTempFile(fileContents: string) {
+  const path = join(budibaseTempDir(), uuid())
+  fs.writeFileSync(path, fileContents)
+  return path
+}
+
+/**
+ * Utility function for getting a file read stream - a simple in memory buffered read
+ * stream doesn't work for pouchdb.
+ */
+export function stringToFileStream(contents: string) {
+  const path = exports.storeTempFile(contents)
+  return fs.createReadStream(path)
+}
+
+/**
+ * Creates a temp file and returns it from the API.
+ * @param {string} fileContents the contents to be returned in file.
+ */
+export function sendTempFile(fileContents: string) {
+  const path = exports.storeTempFile(fileContents)
+  return fs.createReadStream(path)
+}
+
+/**
+ * Uploads the latest client library to the object store.
+ * @param {string} appId The ID of the app which is being created.
+ * @return {Promise<void>} once promise completes app resources should be ready in object store.
+ */
+export async function createApp(appId: string) {
+  await updateClientLibrary(appId)
+}
+
+/**
+ * Removes all of the assets created for an app in the object store.
+ * @param {string} appId The ID of the app which is being deleted.
+ * @return {Promise<void>} once promise completes the app resources will be removed from object store.
+ */
+export async function deleteApp(appId: string) {
+  await objectStore.deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`)
+}
+
+/**
+ * Retrieves a template and pipes it to minio as well as making it available temporarily.
+ * @param {string} type The type of template which is to be retrieved.
+ * @param name
+ * @return {Promise<*>}
+ */
+export async function downloadTemplate(type: string, name: string) {
+  const DEFAULT_TEMPLATES_BUCKET =
+    "prod-budi-templates.s3-eu-west-1.amazonaws.com"
+  const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz`
+  return objectStore.downloadTarball(
+    templateUrl,
+    ObjectStoreBuckets.TEMPLATES,
+    type
+  )
+}
+
+/**
+ * Retrieves component libraries from object store (or tmp symlink if in local)
+ */
+export async function getComponentLibraryManifest(library: string) {
+  const appId = context.getAppId()
+  const filename = "manifest.json"
+  /* istanbul ignore next */
+  // when testing in cypress and so on we need to get the package
+  // as the environment may not be fully fleshed out for dev or prod
+  if (env.isTest()) {
+    library = library.replace("standard-components", "client")
+    const lib = library.split("/")[1]
+    const path = require.resolve(library).split(lib)[0]
+    return require(join(path, lib, filename))
+  } else if (env.isDev()) {
+    const path = join(NODE_MODULES_PATH, "@budibase", "client", filename)
+    // always load from new so that updates are refreshed
+    delete require.cache[require.resolve(path)]
+    return require(path)
+  }
+
+  if (!appId) {
+    throw new Error("No app ID found - cannot get component libraries")
+  }
+
+  let resp
+  let path
+  try {
+    // Try to load the manifest from the new file location
+    path = join(appId, filename)
+    resp = await objectStore.retrieve(ObjectStoreBuckets.APPS, path)
+  } catch (error) {
+    console.error(
+      `component-manifest-objectstore=failed appId=${appId} path=${path}`,
+      error
+    )
+    // Fallback to loading it from the old location for old apps
+    path = join(appId, "node_modules", library, "package", filename)
+    resp = await objectStore.retrieve(ObjectStoreBuckets.APPS, path)
+  }
+  if (typeof resp !== "string") {
+    resp = resp.toString("utf8")
+  }
+  return JSON.parse(resp)
+}
+
+/**
+ * All file reads come through here just to make sure all of them make sense
+ * allows a centralised location to check logic is all good.
+ */
+export function readFileSync(
+  filepath: string,
+  options: BufferEncoding = "utf8"
+) {
+  return fs.readFileSync(filepath, { encoding: options })
+}
+
+/**
+ * Given a set of app IDs makes sure file system is cleared of any of their temp info.
+ */
+export function cleanup(appIds: string[]) {
+  for (let appId of appIds) {
+    const path = join(budibaseTempDir(), appId)
+    if (fs.existsSync(path)) {
+      fs.rmdirSync(path, { recursive: true })
+    }
+  }
+}
+
+export function createTempFolder(item: string) {
+  const path = join(budibaseTempDir(), item)
+  try {
+    // remove old tmp directories automatically - don't combine
+    if (fs.existsSync(path)) {
+      fs.rmSync(path, { recursive: true, force: true })
+    }
+    fs.mkdirSync(path)
+  } catch (err: any) {
+    throw new Error(`Path cannot be created: ${err.message}`)
+  }
+
+  return path
+}
+
+export async function extractTarball(fromFilePath: string, toPath: string) {
+  await tar.extract({
+    file: fromFilePath,
+    C: toPath,
+  })
+}
+
+export async function getPluginMetadata(path: string) {
+  let metadata: { schema?: any; package?: any } = {}
+  try {
+    const pkg = fs.readFileSync(join(path, "package.json"), "utf8")
+    const schema = fs.readFileSync(join(path, "schema.json"), "utf8")
+
+    metadata.schema = JSON.parse(schema)
+    metadata.package = JSON.parse(pkg)
+
+    if (
+      !metadata.package.name ||
+      !metadata.package.version ||
+      !metadata.package.description
+    ) {
+      throw new Error(
+        "package.json is missing one of 'name', 'version' or 'description'."
+      )
+    }
+  } catch (err: any) {
+    throw new Error(
+      `Unable to process schema.json/package.json in plugin. ${err.message}`
+    )
+  }
+
+  return { metadata, directory: path }
+}
+
+export async function getDatasourcePlugin(
+  name: string,
+  url: string,
+  hash: string
+) {
+  if (!fs.existsSync(DATASOURCE_PATH)) {
+    fs.mkdirSync(DATASOURCE_PATH)
+  }
+  const filename = join(DATASOURCE_PATH, name)
+  const metadataName = `${filename}.bbmetadata`
+  if (fs.existsSync(filename)) {
+    const currentHash = fs.readFileSync(metadataName, "utf8")
+    // if hash is the same return the file, otherwise remove it and re-download
+    if (currentHash === hash) {
+      return require(filename)
+    } else {
+      console.log(`Updating plugin: ${name}`)
+      delete require.cache[require.resolve(filename)]
+      fs.unlinkSync(filename)
+    }
+  }
+  const fullUrl = checkSlashesInUrl(
+    `${env.MINIO_URL}/${ObjectStoreBuckets.PLUGINS}/${url}`
+  )
+  const response = await fetch(fullUrl)
+  if (response.status === 200) {
+    const content = await response.text()
+    fs.writeFileSync(filename, content)
+    fs.writeFileSync(metadataName, hash)
+    return require(filename)
+  } else {
+    throw new Error(
+      `Unable to retrieve plugin - reason: ${await response.text()}`
+    )
+  }
+}
+
+/**
+ * Find for a file recursively from start path applying filter, return first match
+ */
+export function findFileRec(startPath: string, filter: any) {
+  if (!fs.existsSync(startPath)) {
+    return
+  }
+
+  const files = fs.readdirSync(startPath)
+  for (let i = 0, len = files.length; i < len; i++) {
+    const filename = join(startPath, files[i])
+    const stat = fs.lstatSync(filename)
+
+    if (stat.isDirectory()) {
+      return exports.findFileRec(filename, filter)
+    } else if (filename.endsWith(filter)) {
+      return filename
+    }
+  }
+}
+
+/**
+ * Remove a folder which is not empty from the file system
+ */
+export function deleteFolderFileSystem(path: string) {
+  if (!fs.existsSync(path)) {
+    return
+  }
+
+  fs.rmSync(path, { recursive: true, force: true })
+}