@budibase/server 1.3.19 → 1.3.21-alpha.0

package/src/threads/query.ts

@@ -2,7 +2,7 @@ import { default as threadUtils } from "./utils"
 threadUtils.threadSetup()
 import { WorkerCallback, QueryEvent, QueryVariable } from "./definitions"
 const ScriptRunner = require("../utilities/scriptRunner")
-const { integrations } = require("../integrations")
+const { getIntegration } = require("../integrations")
 const { processStringSync } = require("@budibase/string-templates")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
 const {
@@ -62,7 +62,7 @@ class QueryRunner {
     let datasourceClone = cloneDeep(datasource)
     let fieldsClone = cloneDeep(fields)
 
-    const Integration = integrations[datasourceClone.source]
+    const Integration = await getIntegration(datasourceClone.source)
     if (!Integration) {
       throw "Integration type does not exist."
     }

package/src/utilities/fileSystem/index.js

@@ -15,9 +15,11 @@ const {
   streamUpload,
   deleteFolder,
   downloadTarball,
+  downloadTarballDirect,
   deleteFiles,
 } = require("./utilities")
 const { updateClientLibrary } = require("./clientLibrary")
+const { checkSlashesInUrl } = require("../")
 const env = require("../../environment")
 const {
   USER_METDATA_PREFIX,
@@ -26,9 +28,12 @@ const {
 } = require("../../db/utils")
 const MemoryStream = require("memorystream")
 const { getAppId } = require("@budibase/backend-core/context")
+const tar = require("tar")
+const fetch = require("node-fetch")
 
 const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
 const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
+const DATASOURCE_PATH = join(budibaseTempDir(), "datasource")
 
 /**
  * The single stack system (Cloud and Builder) should not make use of the file system where possible,
@@ -107,6 +112,13 @@ exports.loadHandlebarsFile = path => {
   return fs.readFileSync(path, "utf8")
 }
 
+/**
+ * Same as above just with a different name.
+ */
+exports.loadJSFile = (directory, name) => {
+  return fs.readFileSync(join(directory, name), "utf8")
+}
+
 /**
  * When return a file from the API need to write the file to the system temporarily so we
  * can create a read stream to send.
@@ -335,6 +347,123 @@ exports.cleanup = appIds => {
   }
 }
 
+const createTempFolder = item => {
+  const path = join(budibaseTempDir(), item)
+  try {
+    // remove old tmp directories automatically - don't combine
+    if (fs.existsSync(path)) {
+      fs.rmSync(path, { recursive: true, force: true })
+    }
+    fs.mkdirSync(path)
+  } catch (err) {
+    throw new Error(`Path cannot be created: ${err.message}`)
+  }
+
+  return path
+}
+exports.createTempFolder = createTempFolder
+
+const extractTarball = async (fromFilePath, toPath) => {
+  await tar.extract({
+    file: fromFilePath,
+    C: toPath,
+  })
+}
+exports.extractTarball = extractTarball
+
+const getPluginMetadata = async path => {
+  let metadata = {}
+  try {
+    const pkg = fs.readFileSync(join(path, "package.json"), "utf8")
+    const schema = fs.readFileSync(join(path, "schema.json"), "utf8")
+
+    metadata.schema = JSON.parse(schema)
+    metadata.package = JSON.parse(pkg)
+
+    if (
+      !metadata.package.name ||
+      !metadata.package.version ||
+      !metadata.package.description
+    ) {
+      throw new Error(
+        "package.json is missing one of 'name', 'version' or 'description'."
+      )
+    }
+  } catch (err) {
+    throw new Error(
+      `Unable to process schema.json/package.json in plugin. ${err.message}`
+    )
+  }
+
+  return { metadata, directory: path }
+}
+exports.getPluginMetadata = getPluginMetadata
+
+exports.getDatasourcePlugin = async (name, url, hash) => {
+  if (!fs.existsSync(DATASOURCE_PATH)) {
+    fs.mkdirSync(DATASOURCE_PATH)
+  }
+  const filename = join(DATASOURCE_PATH, name)
+  const metadataName = `${filename}.bbmetadata`
+  if (fs.existsSync(filename)) {
+    const currentHash = fs.readFileSync(metadataName, "utf8")
+    // if hash is the same return the file, otherwise remove it and re-download
+    if (currentHash === hash) {
+      return require(filename)
+    } else {
+      console.log(`Updating plugin: ${name}`)
+      delete require.cache[require.resolve(filename)]
+      fs.unlinkSync(filename)
+    }
+  }
+  const fullUrl = checkSlashesInUrl(
+    `${env.MINIO_URL}/${ObjectStoreBuckets.PLUGINS}/${url}`
+  )
+  const response = await fetch(fullUrl)
+  if (response.status === 200) {
+    const content = await response.text()
+    fs.writeFileSync(filename, content)
+    fs.writeFileSync(metadataName, hash)
+    return require(filename)
+  } else {
+    throw new Error(
+      `Unable to retrieve plugin - reason: ${await response.text()}`
+    )
+  }
+}
+
+/**
+ * Find for a file recursively from start path applying filter, return first match
+ */
+exports.findFileRec = (startPath, filter) => {
+  if (!fs.existsSync(startPath)) {
+    return
+  }
+
+  const files = fs.readdirSync(startPath)
+  for (let i = 0, len = files.length; i < len; i++) {
+    const filename = join(startPath, files[i])
+    const stat = fs.lstatSync(filename)
+
+    if (stat.isDirectory()) {
+      return exports.findFileRec(filename, filter)
+    } else if (filename.endsWith(filter)) {
+      return filename
+    }
+  }
+}
+
+/**
+ * Remove a folder which is not empty from the file system
+ */
+exports.deleteFolderFileSystem = path => {
+  if (!fs.existsSync(path)) {
+    return
+  }
+
+  fs.rmSync(path, { recursive: true, force: true })
+}
+
 /**
  * Full function definition for below can be found in the utilities.
  */
@@ -342,5 +471,6 @@ exports.upload = upload
 exports.retrieve = retrieve
 exports.retrieveToTmp = retrieveToTmp
 exports.deleteFiles = deleteFiles
+exports.downloadTarballDirect = downloadTarballDirect
 exports.TOP_LEVEL_PATH = TOP_LEVEL_PATH
 exports.NODE_MODULES_PATH = NODE_MODULES_PATH

package/src/utilities/workerRequests.js

@@ -153,3 +153,11 @@ exports.getChecklist = async () => {
   )
   return checkResponse(response, "get checklist")
 }
+
+exports.generateApiKey = async userId => {
+  const response = await fetch(
+    checkSlashesInUrl(env.WORKER_URL + "/api/global/self/api_key"),
+    request(null, { method: "POST", body: { userId } })
+  )
+  return checkResponse(response, "generate API key")
+}

package/src/watch.ts

@@ -0,0 +1,38 @@
+import path from "path"
+import * as env from "./environment"
+import chokidar from "chokidar"
+import fs from "fs"
+import { tenancy } from "@budibase/backend-core"
+import { DEFAULT_TENANT_ID } from "@budibase/backend-core/constants"
+import { processPlugin } from "./api/controllers/plugin"
+
+export function watch() {
+  const watchPath = path.join(env.PLUGINS_DIR, "./**/*.tar.gz")
+  chokidar
+    .watch(watchPath, {
+      ignored: "**/node_modules",
+      awaitWriteFinish: {
+        pollInterval: 100,
+        stabilityThreshold: 250,
+      },
+      usePolling: true,
+      interval: 250,
+    })
+    .on("all", async (event: string, path: string) => {
+      // Sanity checks
+      if (!path?.endsWith(".tar.gz") || !fs.existsSync(path)) {
+        return
+      }
+      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
+        try {
+          const split = path.split("/")
+          const name = split[split.length - 1]
+          console.log("Importing plugin:", path)
+          await processPlugin({ name, path })
+        } catch (err: any) {
+          const message = err?.message ? err?.message : err
+          console.error("Failed to import plugin:", message)
+        }
+      })
+    })
+}

package/src/websocket.ts

@@ -0,0 +1,26 @@
+import { Server } from "socket.io"
+import http from "http"
+
+class Websocket {
+  socketServer: Server
+
+  constructor(server: http.Server, path: string) {
+    this.socketServer = new Server(server, {
+      path,
+    })
+  }
+
+  // Emit an event to all sockets
+  emit(event: string, payload: any) {
+    this.socketServer.sockets.emit(event, payload)
+  }
+}
+
+// Likely to be more socket instances in future
+let ClientAppSocket: Websocket
+
+export const initialise = (server: http.Server) => {
+  ClientAppSocket = new Websocket(server, "/socket/client")
+}
+
+export { ClientAppSocket }