@budibase/server 1.3.13 → 1.3.15-alpha.1

package/src/utilities/fileSystem/index.js

@@ -15,9 +15,11 @@ const {
   streamUpload,
   deleteFolder,
   downloadTarball,
+  downloadTarballDirect,
   deleteFiles,
 } = require("./utilities")
 const { updateClientLibrary } = require("./clientLibrary")
+const { checkSlashesInUrl } = require("../")
 const env = require("../../environment")
 const {
   USER_METDATA_PREFIX,
@@ -26,9 +28,12 @@ const {
 } = require("../../db/utils")
 const MemoryStream = require("memorystream")
 const { getAppId } = require("@budibase/backend-core/context")
+const tar = require("tar")
+const fetch = require("node-fetch")
 
 const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
 const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
+const DATASOURCE_PATH = join(budibaseTempDir(), "datasource")
 
 /**
  * The single stack system (Cloud and Builder) should not make use of the file system where possible,
@@ -99,6 +104,13 @@ exports.loadHandlebarsFile = path => {
   return fs.readFileSync(path, "utf8")
 }
 
+/**
+ * Same as above just with a different name.
+ */
+exports.loadJSFile = (directory, name) => {
+  return fs.readFileSync(join(directory, name), "utf8")
+}
+
 /**
  * When return a file from the API need to write the file to the system temporarily so we
  * can create a read stream to send.
@@ -125,13 +137,13 @@ exports.defineFilter = excludeRows => {
  * data or user relationships.
  * @param {string} appId The app to backup
  * @param {object} config Config to send to export DB
- * @param {boolean} includeRows Flag to state whether the export should include data.
+ * @param {boolean} excludeRows Flag to state whether the export should include data.
  * @returns {*} either a string or a stream of the backup
  */
-const backupAppData = async (appId, config, includeRows) => {
+const backupAppData = async (appId, config, excludeRows) => {
   return await exports.exportDB(appId, {
     ...config,
-    filter: exports.defineFilter(includeRows),
+    filter: exports.defineFilter(excludeRows),
   })
 }
 
@@ -148,11 +160,11 @@ exports.performBackup = async (appId, backupName) => {
 /**
  * Streams a backup of the database state for an app
  * @param {string} appId The ID of the app which is to be backed up.
- * @param {boolean} includeRows Flag to state whether the export should include data.
+ * @param {boolean} excludeRows Flag to state whether the export should include data.
  * @returns {*} a readable stream of the backup which is written in real time
  */
-exports.streamBackup = async (appId, includeRows) => {
-  return await backupAppData(appId, { stream: true }, includeRows)
+exports.streamBackup = async (appId, excludeRows) => {
+  return await backupAppData(appId, { stream: true }, excludeRows)
 }
 
 /**
@@ -327,6 +339,122 @@ exports.cleanup = appIds => {
   }
 }
 
+const createTempFolder = item => {
+  const path = join(budibaseTempDir(), item)
+  try {
+    // remove old tmp directories automatically - don't combine
+    if (fs.existsSync(path)) {
+      fs.rmSync(path, { recursive: true, force: true })
+    }
+    fs.mkdirSync(path)
+  } catch (err) {
+    throw new Error(`Path cannot be created: ${err.message}`)
+  }
+
+  return path
+}
+exports.createTempFolder = createTempFolder
+
+const extractTarball = async (fromFilePath, toPath) => {
+  await tar.extract({
+    file: fromFilePath,
+    C: toPath,
+  })
+}
+exports.extractTarball = extractTarball
+
+const getPluginMetadata = async path => {
+  let metadata = {}
+  try {
+    const pkg = fs.readFileSync(join(path, "package.json"), "utf8")
+    const schema = fs.readFileSync(join(path, "schema.json"), "utf8")
+
+    metadata.schema = JSON.parse(schema)
+    metadata.package = JSON.parse(pkg)
+
+    if (
+      !metadata.package.name ||
+      !metadata.package.version ||
+      !metadata.package.description
+    ) {
+      throw new Error(
+        "package.json is missing one of 'name', 'version' or 'description'."
+      )
+    }
+  } catch (err) {
+    throw new Error(
+      `Unable to process schema.json/package.json in plugin. ${err.message}`
+    )
+  }
+
+  return { metadata, directory: path }
+}
+exports.getPluginMetadata = getPluginMetadata
+
+exports.getDatasourcePlugin = async (name, url, hash) => {
+  if (!fs.existsSync(DATASOURCE_PATH)) {
+    fs.mkdirSync(DATASOURCE_PATH)
+  }
+  const filename = join(DATASOURCE_PATH, name)
+  const metadataName = `${filename}.bbmetadata`
+  if (fs.existsSync(filename)) {
+    const currentHash = fs.readFileSync(metadataName, "utf8")
+    // if hash is the same return the file, otherwise remove it and re-download
+    if (currentHash === hash) {
+      return require(filename)
+    } else {
+      console.log(`Updating plugin: ${name}`)
+      fs.unlinkSync(filename)
+    }
+  }
+  const fullUrl = checkSlashesInUrl(
+    `${env.MINIO_URL}/${ObjectStoreBuckets.PLUGINS}/${url}`
+  )
+  const response = await fetch(fullUrl)
+  if (response.status === 200) {
+    const content = await response.text()
+    fs.writeFileSync(filename, content)
+    fs.writeFileSync(metadataName, hash)
+    require(filename)
+  } else {
+    throw new Error(
+      `Unable to retrieve plugin - reason: ${await response.text()}`
+    )
+  }
+}
+
+/**
+ * Find for a file recursively from start path applying filter, return first match
+ */
+exports.findFileRec = (startPath, filter) => {
+  if (!fs.existsSync(startPath)) {
+    return
+  }
+
+  const files = fs.readdirSync(startPath)
+  for (let i = 0, len = files.length; i < len; i++) {
+    const filename = join(startPath, files[i])
+    const stat = fs.lstatSync(filename)
+
+    if (stat.isDirectory()) {
+      return exports.findFileRec(filename, filter)
+    } else if (filename.endsWith(filter)) {
+      return filename
+    }
+  }
+}
+
+/**
+ * Remove a folder which is not empty from the file system
+ */
+exports.deleteFolderFileSystem = path => {
+  if (!fs.existsSync(path)) {
+    return
+  }
+
+  fs.rmSync(path, { recursive: true, force: true })
+}
+
 /**
  * Full function definition for below can be found in the utilities.
  */
@@ -334,5 +462,6 @@ exports.upload = upload
 exports.retrieve = retrieve
 exports.retrieveToTmp = retrieveToTmp
 exports.deleteFiles = deleteFiles
+exports.downloadTarballDirect = downloadTarballDirect
 exports.TOP_LEVEL_PATH = TOP_LEVEL_PATH
 exports.NODE_MODULES_PATH = NODE_MODULES_PATH

package/src/utilities/index.js

@@ -162,3 +162,11 @@ exports.convertBookmark = bookmark => {
   }
   return bookmark
 }
+
+exports.isQsTrue = param => {
+  if (typeof param === "string") {
+    return param.toLowerCase() === "true"
+  } else {
+    return param === true
+  }
+}

package/src/utilities/rowProcessor/utils.js

@@ -76,7 +76,7 @@ exports.processDates = (table, rows) => {
     if (schema.type !== FieldTypes.DATETIME) {
       continue
     }
-    if (!schema.ignoreTimezones) {
+    if (!schema.timeOnly && !schema.ignoreTimezones) {
       datesWithTZ.push(column)
     }
   }

package/src/utilities/usageQuota/rows.js

@@ -72,7 +72,8 @@ exports.getUniqueRows = async appIds => {
     // ensure uniqueness on a per app pair basis
     // this can't be done on all rows because app import results in
     // duplicate row ids across apps
-    uniqueRows = uniqueRows.concat(...new Set(appRows))
+    // the array pre-concat is important to avoid stack overflow
+    uniqueRows = uniqueRows.concat([...new Set(appRows)])
   }
 
   return uniqueRows

package/src/watch.ts

@@ -0,0 +1,38 @@
+import path from "path"
+import * as env from "./environment"
+import chokidar from "chokidar"
+import fs from "fs"
+import { tenancy } from "@budibase/backend-core"
+import { DEFAULT_TENANT_ID } from "@budibase/backend-core/constants"
+import { processPlugin } from "./api/controllers/plugin"
+
+export function watch() {
+  const watchPath = path.join(env.PLUGINS_DIR, "./**/*.tar.gz")
+  chokidar
+    .watch(watchPath, {
+      ignored: "**/node_modules",
+      awaitWriteFinish: {
+        pollInterval: 100,
+        stabilityThreshold: 250,
+      },
+      usePolling: true,
+      interval: 250,
+    })
+    .on("all", async (event: string, path: string) => {
+      // Sanity checks
+      if (!path?.endsWith(".tar.gz") || !fs.existsSync(path)) {
+        return
+      }
+      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
+        try {
+          const split = path.split("/")
+          const name = split[split.length - 1]
+          console.log("Importing plugin:", path)
+          await processPlugin({ name, path })
+        } catch (err: any) {
+          const message = err?.message ? err?.message : err
+          console.error("Failed to import plugin:", message)
+        }
+      })
+    })
+}